cp qtest.inp /pvfs/da13-test/qtest.F05 unset echo unset echo /usr/src/gamess-hg/ddikick.x /usr/src/gamess-hg/gamess.pvfs32.x qtest -ddi 1 1 localhost -scr /pvfs/da13-test io_init pvfs2_aio_init id 0 sleep 0ms for pvfs ibv_reg_mr temp failure workaround pvfs2_aio_init id 0 wakeup [D 20:00:41.496901] PINT_encode_initialize [D 20:00:41.496961] lebf_initialize [D 20:00:41.496997] check_req_size [D 20:00:41.497023] encode_common [D 20:00:41.497055] lebf_encode_req [D 20:00:41.497087] lebf_encode_rel [D 20:00:41.497119] check_resp_size [D 20:00:41.497147] encode_common [D 20:00:41.497177] lebf_encode_resp [D 20:00:41.497206] lebf_encode_rel [D 20:00:41.497237] check_req_size [D 20:00:41.497262] encode_common [D 20:00:41.497290] lebf_encode_req [D 20:00:41.497315] lebf_encode_rel [D 20:00:41.497344] check_resp_size [D 20:00:41.497369] encode_common [D 20:00:41.497398] lebf_encode_resp [D 20:00:41.497423] lebf_encode_rel [D 20:00:41.497452] check_req_size [D 20:00:41.497477] encode_common [D 20:00:41.497506] lebf_encode_req [D 20:00:41.497543] PINT_do_request_commit: commit node 0xff80dd10 [D 20:00:41.497570] node stored at 0 [D 20:00:41.497593] clearing tree [D 20:00:41.497624] lebf_encode_rel [D 20:00:41.497654] check_resp_size [D 20:00:41.497679] encode_common [D 20:00:41.497708] lebf_encode_resp [D 20:00:41.497733] lebf_encode_rel [D 20:00:41.497762] check_req_size [D 20:00:41.497787] encode_common [D 20:00:41.497816] lebf_encode_req [D 20:00:41.497841] lebf_encode_rel [D 20:00:41.497871] check_resp_size [D 20:00:41.497896] encode_common [D 20:00:41.497923] lebf_encode_resp [D 20:00:41.497953] lebf_encode_rel [D 20:00:41.497983] check_req_size [D 20:00:41.498008] encode_common [D 20:00:41.498037] lebf_encode_req [D 20:00:41.498066] lebf_encode_rel [D 20:00:41.498097] check_resp_size [D 20:00:41.498121] encode_common [D 20:00:41.498150] lebf_encode_resp [D 20:00:41.498175] lebf_encode_rel [D 20:00:41.498205] check_req_size [D 20:00:41.498229] encode_common [D 20:00:41.498257] lebf_encode_req [D 20:00:41.498283] lebf_encode_rel [D 20:00:41.498312] check_resp_size [D 20:00:41.498337] encode_common [D 20:00:41.498366] lebf_encode_resp [D 20:00:41.498392] lebf_encode_rel [D 20:00:41.498421] check_req_size [D 20:00:41.498446] encode_common [D 20:00:41.498475] lebf_encode_req [D 20:00:41.498503] lebf_encode_rel [D 20:00:41.498534] check_resp_size [D 20:00:41.498558] encode_common [D 20:00:41.498589] lebf_encode_resp [D 20:00:41.498618] lebf_encode_rel [D 20:00:41.498652] check_req_size [D 20:00:41.498680] encode_common [D 20:00:41.498711] lebf_encode_req [D 20:00:41.498737] lebf_encode_rel [D 20:00:41.498766] check_resp_size [D 20:00:41.498790] encode_common [D 20:00:41.498819] lebf_encode_resp [D 20:00:41.498845] lebf_encode_rel [D 20:00:41.498874] check_req_size [D 20:00:41.498899] encode_common [D 20:00:41.498928] lebf_encode_req [D 20:00:41.498953] lebf_encode_rel [D 20:00:41.498982] check_resp_size [D 20:00:41.499007] encode_common [D 20:00:41.499035] lebf_encode_resp [D 20:00:41.499060] lebf_encode_rel [D 20:00:41.499090] check_req_size [D 20:00:41.499114] encode_common [D 20:00:41.499142] lebf_encode_req [D 20:00:41.499168] lebf_encode_rel [D 20:00:41.499197] check_resp_size [D 20:00:41.499222] encode_common [D 20:00:41.499250] lebf_encode_resp [D 20:00:41.499275] lebf_encode_rel [D 20:00:41.499305] check_req_size [D 20:00:41.499329] encode_common [D 20:00:41.499358] lebf_encode_req [D 20:00:41.499387] lebf_encode_rel [D 20:00:41.499418] check_resp_size [D 20:00:41.499442] encode_common [D 20:00:41.499470] lebf_encode_resp [D 20:00:41.499496] lebf_encode_rel [D 20:00:41.499526] check_req_size [D 20:00:41.499550] encode_common [D 20:00:41.499579] lebf_encode_req [D 20:00:41.499604] lebf_encode_rel [D 20:00:41.499633] check_resp_size [D 20:00:41.499658] encode_common [D 20:00:41.499687] lebf_encode_resp [D 20:00:41.499712] lebf_encode_rel [D 20:00:41.499741] check_req_size [D 20:00:41.499766] encode_common [D 20:00:41.499794] lebf_encode_req [D 20:00:41.499819] lebf_encode_rel [D 20:00:41.499849] check_resp_size [D 20:00:41.499874] encode_common [D 20:00:41.499905] lebf_encode_resp [D 20:00:41.499930] lebf_encode_rel [D 20:00:41.499959] check_resp_size [D 20:00:41.499985] encode_common [D 20:00:41.500014] lebf_encode_resp [D 20:00:41.500041] lebf_encode_rel [D 20:00:41.500072] check_req_size [D 20:00:41.500097] encode_common [D 20:00:41.500125] lebf_encode_req [D 20:00:41.500150] lebf_encode_rel [D 20:00:41.500180] check_resp_size [D 20:00:41.500205] encode_common [D 20:00:41.500233] lebf_encode_resp [D 20:00:41.500257] lebf_encode_rel [D 20:00:41.500287] check_req_size [D 20:00:41.500311] encode_common [D 20:00:41.500340] lebf_encode_req [D 20:00:41.500365] lebf_encode_rel [D 20:00:41.500394] check_resp_size [D 20:00:41.500419] encode_common [D 20:00:41.500448] lebf_encode_resp [D 20:00:41.500475] lebf_encode_rel [D 20:00:41.500506] check_req_size [D 20:00:41.500531] encode_common [D 20:00:41.500558] lebf_encode_req [D 20:00:41.500587] lebf_encode_rel [D 20:00:41.500617] check_resp_size [D 20:00:41.500642] encode_common [D 20:00:41.500670] lebf_encode_resp [D 20:00:41.500695] lebf_encode_rel [D 20:00:41.500724] check_req_size [D 20:00:41.500749] encode_common [D 20:00:41.500778] lebf_encode_req [D 20:00:41.500802] lebf_encode_rel [D 20:00:41.500832] check_resp_size [D 20:00:41.500857] encode_common [D 20:00:41.500885] lebf_encode_resp [D 20:00:41.500912] lebf_encode_rel [D 20:00:41.500943] check_req_size [D 20:00:41.500967] encode_common [D 20:00:41.500995] lebf_encode_req [D 20:00:41.501020] lebf_encode_rel [D 20:00:41.501049] check_resp_size [D 20:00:41.501074] encode_common [D 20:00:41.501103] lebf_encode_resp [D 20:00:41.501129] lebf_encode_rel [D 20:00:41.501158] check_req_size [D 20:00:41.501183] encode_common [D 20:00:41.501212] lebf_encode_req [D 20:00:41.501236] lebf_encode_rel [D 20:00:41.501266] check_resp_size [D 20:00:41.501290] encode_common [D 20:00:41.501318] lebf_encode_resp [D 20:00:41.501343] lebf_encode_rel [D 20:00:41.501373] check_req_size [D 20:00:41.501398] encode_common [D 20:00:41.501426] lebf_encode_req [D 20:00:41.501454] lebf_encode_rel [D 20:00:41.501484] check_resp_size [D 20:00:41.501508] encode_common [D 20:00:41.501537] lebf_encode_resp [D 20:00:41.501562] lebf_encode_rel [D 20:00:41.501592] check_req_size [D 20:00:41.501616] encode_common [D 20:00:41.501645] lebf_encode_req [D 20:00:41.501670] lebf_encode_rel [D 20:00:41.501700] check_resp_size [D 20:00:41.501724] encode_common [D 20:00:41.501752] lebf_encode_resp [D 20:00:41.501778] lebf_encode_rel [D 20:00:41.501808] check_req_size [D 20:00:41.501832] encode_common [D 20:00:41.501860] lebf_encode_req [D 20:00:41.501886] lebf_encode_rel [D 20:00:41.501915] check_resp_size [D 20:00:41.501939] encode_common [D 20:00:41.501968] lebf_encode_resp [D 20:00:41.501993] lebf_encode_rel [D 20:00:41.502022] check_req_size [D 20:00:41.502046] encode_common [D 20:00:41.502075] lebf_encode_req [D 20:00:41.502100] lebf_encode_rel [D 20:00:41.502130] check_resp_size [D 20:00:41.502155] encode_common [D 20:00:41.502183] lebf_encode_resp [D 20:00:41.502207] lebf_encode_rel [D 20:00:41.502237] check_req_size [D 20:00:41.502262] encode_common [D 20:00:41.502290] lebf_encode_req [D 20:00:41.502316] lebf_encode_rel [D 20:00:41.502345] check_resp_size [D 20:00:41.502369] encode_common [D 20:00:41.502398] lebf_encode_resp [D 20:00:41.502423] lebf_encode_rel [D 20:00:41.502452] check_req_size [D 20:00:41.502477] encode_common [D 20:00:41.502506] lebf_encode_req [D 20:00:41.502531] lebf_encode_rel [D 20:00:41.502560] check_resp_size [D 20:00:41.502587] encode_common [D 20:00:41.502619] lebf_encode_resp [D 20:00:41.502648] lebf_encode_rel [D 20:00:41.502681] check_req_size [D 20:00:41.502709] encode_common [D 20:00:41.502737] lebf_encode_req [D 20:00:41.502762] lebf_encode_rel [D 20:00:41.502792] check_resp_size [D 20:00:41.502817] encode_common [D 20:00:41.502845] lebf_encode_resp [D 20:00:41.502873] lebf_encode_rel [D 20:00:41.502903] check_req_size [D 20:00:41.502927] encode_common [D 20:00:41.502957] lebf_encode_req [D 20:00:41.502983] lebf_encode_rel [D 20:00:41.503012] check_resp_size [D 20:00:41.503039] encode_common [D 20:00:41.503067] lebf_encode_resp [D 20:00:41.503092] lebf_encode_rel [D 20:00:41.503123] check_req_size [D 20:00:41.503148] encode_common [D 20:00:41.503176] lebf_encode_req [D 20:00:41.503201] lebf_encode_rel [D 20:00:41.503232] check_resp_size [D 20:00:41.503257] encode_common [D 20:00:41.503286] lebf_encode_resp [D 20:00:41.503311] lebf_encode_rel [D 20:00:41.503340] check_req_size [D 20:00:41.503364] encode_common [D 20:00:41.503393] lebf_encode_req [D 20:00:41.503418] lebf_encode_rel [D 20:00:41.503448] check_resp_size [D 20:00:41.503473] encode_common [D 20:00:41.503500] lebf_encode_resp [D 20:00:41.503526] lebf_encode_rel [D 20:00:41.503556] check_req_size [D 20:00:41.503581] encode_common [D 20:00:41.503609] lebf_encode_req [D 20:00:41.503635] PINT_do_request_commit: commit node 0xff80dd10 [D 20:00:41.503660] node stored at 0 [D 20:00:41.503684] clearing tree [D 20:00:41.503710] lebf_encode_rel [D 20:00:41.503740] check_resp_size [D 20:00:41.503765] encode_common [D 20:00:41.503793] lebf_encode_resp [D 20:00:41.503818] lebf_encode_rel [D 20:00:41.503849] check_req_size [D 20:00:41.503873] encode_common [D 20:00:41.503901] lebf_encode_req [D 20:00:41.503927] lebf_encode_rel [D 20:00:41.503956] check_resp_size [D 20:00:41.503981] encode_common [D 20:00:41.504010] lebf_encode_resp [D 20:00:41.504037] lebf_encode_rel [D 20:00:41.504248] [SM Frame PUSH]: (0x15e4ea10) frame: 0x15e5fd70 [D 20:00:41.504277] [SM Locating]: (0x15e4ea10) op-id: 300 [D 20:00:41.504304] client_op_state_get_machine 300 [D 20:00:41.504330] [SM Locating]: (0x15e4ea10) located: client_job_timer_sm:do_work [D 20:00:41.504360] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:00:41.504389] PINT_client_state_machine_post smcb 0x15e4ea10, op: PVFS_CLIENT_JOB_TIMER [D 20:00:41.504416] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:00:41.504446] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:00:41.504485] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:00:41.504513] Posted PVFS_CLIENT_JOB_TIMER (-1) (waiting for test)(-1073741839) [D 20:00:41.504689] BMI_ib_initialize: init. [D 20:00:41.504719] openib_ib_initialize: init. [D 20:00:41.505816] openib_ib_initialize: max 16380 completion queue entries. [D 20:00:41.506017] BMI_ib_initialize: done. [D 20:00:41.506075] [SM Frame PUSH]: (0x15e62108) frame: 0x15e621a0 [D 20:00:41.506102] [SM Locating]: (0x15e62108) op-id: 19 [D 20:00:41.506128] client_op_state_get_machine 19 [D 20:00:41.506153] [SM push_state]: (0x15e62108) op-id: 19 stk-ptr: 0 base-frm: 0 [D 20:00:41.506178] [SM Locating]: (0x15e62108) located: server_get_config_nested_sm:setup_msgpair [D 20:00:41.506203] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.506230] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.506256] PINT_client_state_machine_post smcb 0x15e62108, op: PVFS_SYS_FS_ADD [D 20:00:41.506281] [SM Entering]: (0x15e62108) server_get_config_nested_sm:setup_msgpair (status: 0) [D 20:00:41.506307] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.506332] get_config state: server_get_config_setup_msgpair [D 20:00:41.506358] [SM Exiting]: (0x15e62108) server_get_config_nested_sm:setup_msgpair (error code: 0), (action: COMPLETE) [D 20:00:41.506383] [SM push_state]: (0x15e62108) op-id: 19 stk-ptr: 1 base-frm: 0 [D 20:00:41.506409] [SM Entering]: (0x15e62108) msgpairarray_sm:init (status: 0) [D 20:00:41.506441] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.506466] (0x15e62108) msgpairarray state: init (1 msgpair(s)) [D 20:00:41.506491] [SM Exiting]: (0x15e62108) msgpairarray_sm:init (error code: 0), (action: COMPLETE) [D 20:00:41.506517] [SM Entering]: (0x15e62108) msgpairarray_sm:post (status: 0) [D 20:00:41.506542] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.506567] msgpairarray_post: sm 0x15e62108 2 total message(s) with 2 incomplete [D 20:00:41.506601] PINT_encode [D 20:00:41.506632] encode_common [D 20:00:41.506667] lebf_encode_req [D 20:00:41.506697] PINT_encode_calc_max_size [D 20:00:41.506781] msgpairarray_post: sm 0x15e62108 msgpair 0: posting recv [D 20:00:41.506814] BMI_post_recv: addr: 367403208, offset: 0x15e629a8, size: 65568, tag: 1 [E 20:00:41.508256] Warning: openib_new_connection: asked for 150 send WRs on QP, got 0. [D 20:00:41.509110] openib_post_rr: da13:3346 bh 0 wr_id 15e734e8. [D 20:00:41.509182] openib_post_rr: da13:3346 bh 1 wr_id 15e73500. [D 20:00:41.509210] openib_post_rr: da13:3346 bh 2 wr_id 15e73518. [D 20:00:41.509236] openib_post_rr: da13:3346 bh 3 wr_id 15e73530. [D 20:00:41.509262] openib_post_rr: da13:3346 bh 4 wr_id 15e73548. [D 20:00:41.509287] openib_post_rr: da13:3346 bh 5 wr_id 15e73560. [D 20:00:41.509314] openib_post_rr: da13:3346 bh 6 wr_id 15e73578. [D 20:00:41.509340] openib_post_rr: da13:3346 bh 7 wr_id 15e73590. [D 20:00:41.509365] openib_post_rr: da13:3346 bh 8 wr_id 15e735a8. [D 20:00:41.509390] openib_post_rr: da13:3346 bh 9 wr_id 15e735c0. [D 20:00:41.509416] openib_post_rr: da13:3346 bh 10 wr_id 15e735d8. [D 20:00:41.509442] openib_post_rr: da13:3346 bh 11 wr_id 15e735f0. [D 20:00:41.509468] openib_post_rr: da13:3346 bh 12 wr_id 15e73608. [D 20:00:41.509492] openib_post_rr: da13:3346 bh 13 wr_id 15e73620. [D 20:00:41.509518] openib_post_rr: da13:3346 bh 14 wr_id 15e73638. [D 20:00:41.509544] openib_post_rr: da13:3346 bh 15 wr_id 15e73650. [D 20:00:41.509570] openib_post_rr: da13:3346 bh 16 wr_id 15e73668. [D 20:00:41.509598] openib_post_rr: da13:3346 bh 17 wr_id 15e73680. [D 20:00:41.509624] openib_post_rr: da13:3346 bh 18 wr_id 15e73698. [D 20:00:41.509650] openib_post_rr: da13:3346 bh 19 wr_id 15e736b0. [D 20:00:41.509675] openib_post_rr: da13:3346 bh 20 wr_id 15e736c8. [D 20:00:41.509700] openib_post_rr: da13:3346 bh 21 wr_id 15e736e0. [D 20:00:41.509726] openib_post_rr: da13:3346 bh 22 wr_id 15e736f8. [D 20:00:41.509751] openib_post_rr: da13:3346 bh 23 wr_id 15e73710. [D 20:00:41.509777] openib_post_rr: da13:3346 bh 24 wr_id 15e73728. [D 20:00:41.509802] openib_post_rr: da13:3346 bh 25 wr_id 15e73740. [D 20:00:41.509827] openib_post_rr: da13:3346 bh 26 wr_id 15e73758. [D 20:00:41.509853] openib_post_rr: da13:3346 bh 27 wr_id 15e73770. [D 20:00:41.509879] openib_post_rr: da13:3346 bh 28 wr_id 15e73788. [D 20:00:41.509904] openib_post_rr: da13:3346 bh 29 wr_id 15e737a0. [D 20:00:41.509929] openib_post_rr: da13:3346 bh 30 wr_id 15e737b8. [D 20:00:41.509955] openib_post_rr: da13:3346 bh 31 wr_id 15e737d0. [D 20:00:41.509981] openib_post_rr: da13:3346 bh 32 wr_id 15e737e8. [D 20:00:41.510006] openib_post_rr: da13:3346 bh 33 wr_id 15e73800. [D 20:00:41.510031] openib_post_rr: da13:3346 bh 34 wr_id 15e73818. [D 20:00:41.510057] openib_post_rr: da13:3346 bh 35 wr_id 15e73830. [D 20:00:41.510082] openib_post_rr: da13:3346 bh 36 wr_id 15e73848. [D 20:00:41.510107] openib_post_rr: da13:3346 bh 37 wr_id 15e73860. [D 20:00:41.510133] openib_post_rr: da13:3346 bh 38 wr_id 15e73878. [D 20:00:41.510158] openib_post_rr: da13:3346 bh 39 wr_id 15e73890. [D 20:00:41.510184] openib_post_rr: da13:3346 bh 40 wr_id 15e738a8. [D 20:00:41.510209] openib_post_rr: da13:3346 bh 41 wr_id 15e738c0. [D 20:00:41.510234] openib_post_rr: da13:3346 bh 42 wr_id 15e738d8. [D 20:00:41.510260] openib_post_rr: da13:3346 bh 43 wr_id 15e738f0. [D 20:00:41.510285] openib_post_rr: da13:3346 bh 44 wr_id 15e73908. [D 20:00:41.510311] openib_post_rr: da13:3346 bh 45 wr_id 15e73920. [D 20:00:41.510336] openib_post_rr: da13:3346 bh 46 wr_id 15e73938. [D 20:00:41.510361] openib_post_rr: da13:3346 bh 47 wr_id 15e73950. [D 20:00:41.510387] openib_post_rr: da13:3346 bh 48 wr_id 15e73968. [D 20:00:41.510413] openib_post_rr: da13:3346 bh 49 wr_id 15e73980. [D 20:00:41.510437] openib_post_rr: da13:3346 bh 50 wr_id 15e73998. [D 20:00:41.510463] openib_post_rr: da13:3346 bh 51 wr_id 15e739b0. [D 20:00:41.510489] openib_post_rr: da13:3346 bh 52 wr_id 15e739c8. [D 20:00:41.510514] openib_post_rr: da13:3346 bh 53 wr_id 15e739e0. [D 20:00:41.510540] openib_post_rr: da13:3346 bh 54 wr_id 15e739f8. [D 20:00:41.510568] openib_post_rr: da13:3346 bh 55 wr_id 15e73a10. [D 20:00:41.510595] openib_post_rr: da13:3346 bh 56 wr_id 15e73a28. [D 20:00:41.510620] openib_post_rr: da13:3346 bh 57 wr_id 15e73a40. [D 20:00:41.510645] openib_post_rr: da13:3346 bh 58 wr_id 15e73a58. [D 20:00:41.510671] openib_post_rr: da13:3346 bh 59 wr_id 15e73a70. [D 20:00:41.510697] openib_post_rr: da13:3346 bh 60 wr_id 15e73a88. [D 20:00:41.510723] openib_post_rr: da13:3346 bh 61 wr_id 15e73aa0. [D 20:00:41.510747] openib_post_rr: da13:3346 bh 62 wr_id 15e73ab8. [D 20:00:41.510773] openib_post_rr: da13:3346 bh 63 wr_id 15e73ad0. [D 20:00:41.510799] openib_post_rr: da13:3346 bh 64 wr_id 15e73ae8. [D 20:00:41.510825] openib_post_rr: da13:3346 bh 65 wr_id 15e73b00. [D 20:00:41.510849] openib_post_rr: da13:3346 bh 66 wr_id 15e73b18. [D 20:00:41.510875] openib_post_rr: da13:3346 bh 67 wr_id 15e73b30. [D 20:00:41.510900] openib_post_rr: da13:3346 bh 68 wr_id 15e73b48. [D 20:00:41.510926] openib_post_rr: da13:3346 bh 69 wr_id 15e73b60. [D 20:00:41.510951] openib_post_rr: da13:3346 bh 70 wr_id 15e73b78. [D 20:00:41.510976] openib_post_rr: da13:3346 bh 71 wr_id 15e73b90. [D 20:00:41.511002] openib_post_rr: da13:3346 bh 72 wr_id 15e73ba8. [D 20:00:41.511028] openib_post_rr: da13:3346 bh 73 wr_id 15e73bc0. [D 20:00:41.511053] openib_post_rr: da13:3346 bh 74 wr_id 15e73bd8. [D 20:00:41.511078] openib_post_rr: da13:3346 bh 75 wr_id 15e73bf0. [D 20:00:41.511104] openib_post_rr: da13:3346 bh 76 wr_id 15e73c08. [D 20:00:41.511129] openib_post_rr: da13:3346 bh 77 wr_id 15e73c20. [D 20:00:41.511155] openib_post_rr: da13:3346 bh 78 wr_id 15e73c38. [D 20:00:41.511180] openib_post_rr: da13:3346 bh 79 wr_id 15e73c50. [D 20:00:41.511205] openib_post_rr: da13:3346 bh 80 wr_id 15e73c68. [D 20:00:41.511231] openib_post_rr: da13:3346 bh 81 wr_id 15e73c80. [D 20:00:41.511257] openib_post_rr: da13:3346 bh 82 wr_id 15e73c98. [D 20:00:41.511282] openib_post_rr: da13:3346 bh 83 wr_id 15e73cb0. [D 20:00:41.511307] openib_post_rr: da13:3346 bh 84 wr_id 15e73cc8. [D 20:00:41.511333] openib_post_rr: da13:3346 bh 85 wr_id 15e73ce0. [D 20:00:41.511359] openib_post_rr: da13:3346 bh 86 wr_id 15e73cf8. [D 20:00:41.511384] openib_post_rr: da13:3346 bh 87 wr_id 15e73d10. [D 20:00:41.511409] openib_post_rr: da13:3346 bh 88 wr_id 15e73d28. [D 20:00:41.511435] openib_post_rr: da13:3346 bh 89 wr_id 15e73d40. [D 20:00:41.511461] openib_post_rr: da13:3346 bh 90 wr_id 15e73d58. [D 20:00:41.511487] openib_post_rr: da13:3346 bh 91 wr_id 15e73d70. [D 20:00:41.511511] openib_post_rr: da13:3346 bh 92 wr_id 15e73d88. [D 20:00:41.511537] openib_post_rr: da13:3346 bh 93 wr_id 15e73da0. [D 20:00:41.511563] openib_post_rr: da13:3346 bh 94 wr_id 15e73db8. [D 20:00:41.511591] openib_post_rr: da13:3346 bh 95 wr_id 15e73dd0. [D 20:00:41.511621] openib_post_rr: da13:3346 bh 96 wr_id 15e73de8. [D 20:00:41.511651] openib_post_rr: da13:3346 bh 97 wr_id 15e73e00. [D 20:00:41.511680] openib_post_rr: da13:3346 bh 98 wr_id 15e73e18. [D 20:00:41.511709] openib_post_rr: da13:3346 bh 99 wr_id 15e73e30. [D 20:00:41.511941] msgpairarray_post: sm 0x15e62108 msgpair 0: posting send [D 20:00:41.511983] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 24, tag: 1 [D 20:00:41.512010] element 0: offset: 0x15e61e88, size: 24 [D 20:00:41.512041] post_sr: da13:3346 bh 0 len 40 wr_id 15e72b80. [D 20:00:41.512075] [SM Exiting]: (0x15e62108) msgpairarray_sm:post (error code: 0), (action: DEFERRED) [D 20:00:41.512107] Posted PVFS_SYS_FS_ADD (5) (waiting for test)(-1073741839) [D 20:00:41.512135] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.512166] PINT_client_state_machine_test id 5 [D 20:00:41.512218] openib_check_cq: completed send wr_id 15e72b80. [D 20:00:41.512256] sm control op 19 [D 20:00:41.512282] [SM Entering]: (0x15e62108) msgpairarray_sm:complete (status: 1) [D 20:00:41.512307] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.512332] msgpairarray_complete: sm 0x15e62108 status_user_tag 1 msgarray_count 1 [D 20:00:41.512358] msgpairarray: 1 operations remain [D 20:00:41.512387] [SM Exiting]: (0x15e62108) msgpairarray_sm:complete (error code: 0), (action: DEFERRED) [D 20:00:41.512413] PINT_client_state_machine_test id 5 [D 20:00:41.522440] PINT_client_state_machine_test id 5 [D 20:00:41.532470] PINT_client_state_machine_test id 5 -[D 20:00:41.537890] openib_check_cq: completed recv wr_id 15e734e8. [D 20:00:41.537931] openib_post_rr: da13:3346 bh 0 wr_id 15e734e8. [D 20:00:41.537964] sm control op 19 [D 20:00:41.537990] [SM Entering]: (0x15e62108) msgpairarray_sm:complete (status: 0) [D 20:00:41.538016] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.538041] msgpairarray_complete: sm 0x15e62108 status_user_tag 0 msgarray_count 1 [D 20:00:41.538066] msgpairarray: all operations complete [D 20:00:41.538090] [SM Exiting]: (0x15e62108) msgpairarray_sm:complete (error code: 190), (action: COMPLETE) [D 20:00:41.538116] [SM Entering]: (0x15e62108) msgpairarray_sm:completion_fn (status: 0) [D 20:00:41.538142] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.538166] (0x15e62108) msgpairarray state: completion_fn [D 20:00:41.538192] PINT_decode [D 20:00:41.538221] lebf_decode_resp [D 20:00:41.538248] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.538275] PINT_encode_release [D 20:00:41.538299] lebf_encode_rel [D 20:00:41.538325] PINT_decode_release [D 20:00:41.538349] lebf_decode_rel [D 20:00:41.538374] msgpairarray_completion_fn: sm 0x15e62108 msgpair 0 marked complete [D 20:00:41.538399] [SM Exiting]: (0x15e62108) msgpairarray_sm:completion_fn (error code: 0), (action: COMPLETE) [D 20:00:41.538426] [SM pop_state]: (0x15e62108) op-id: 19 stk-ptr: 2 base-frm: 0 [D 20:00:41.538451] [SM Entering]: (0x15e62108) server_get_config_nested_sm:parse (status: 0) [D 20:00:41.538476] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.539155] [SM Exiting]: (0x15e62108) server_get_config_nested_sm:parse (error code: 0), (action: COMPLETE) [D 20:00:41.539182] [SM Entering]: (0x15e62108) server_get_config_nested_sm:cleanup (status: 0) [D 20:00:41.539208] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.539234] [SM Exiting]: (0x15e62108) server_get_config_nested_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:41.539260] [SM pop_state]: (0x15e62108) op-id: 19 stk-ptr: 1 base-frm: 0 [D 20:00:41.539285] [SM Entering]: (0x15e62108) fs_add_sm:parent_cleanup (status: 0) [D 20:00:41.539318] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.539343] [BMI CONTROL]: BMI_set_info: set_info: 0 option: 11 [D 20:00:41.539370] [BMI CONTROL]: BMI_set_info: set_info: 0 option: 12 [D 20:00:41.539410] PINT_server_config_mgr_add_config: adding config 0x15e62918 [D 20:00:41.539438] mapped fs_id 136 => config 0x15e62918 [D 20:00:41.539472] Set min handle recycle time to 360 seconds [D 20:00:41.539498] Reloading handle mappings for fs_id 136 [D 20:00:41.539528] [SM Exiting]: (0x15e62108) fs_add_sm:parent_cleanup (error code: 0), (action: COMPLETE) [D 20:00:41.539554] [SM Entering]: (0x15e62108) fs_add_sm:config_integrity_checks (status: 0) [D 20:00:41.539580] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.539616] [SM Exiting]: (0x15e62108) fs_add_sm:config_integrity_checks (error code: 1), (action: COMPLETE) [D 20:00:41.539643] [SM Entering]: (0x15e62108) fs_add_sm:final_cleanup (status: 0) [D 20:00:41.539669] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.539693] [SM Exiting]: (0x15e62108) fs_add_sm:final_cleanup (error code: 0), (action: TERMINATE) [D 20:00:41.539720] client_state_machine_terminate smcb 0x15e62108 [D 20:00:41.539744] add smcb 0x15e62108 to completion list [D 20:00:41.539770] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 [D 20:00:41.539795] PINT_sys_release: id 5 [D 20:00:41.539821] [SM frame get]: (0x15e62108) op-id: 19 index: 0 base-frm: 0 pvfs2_aio_open opening /pvfs/da13-test/qtest.F08 pvfs2_aio_open id 0 sleep 0ms for pvfs ibv_reg_mr temp failure workaround pvfs2_aio_open id 0 wakeup user_path : /pvfs/da13-test/qtest.F08 , pvfs2_path : /qtest.F08 [D 20:00:41.540615] PVFS_sys_ref_lookup entered [D 20:00:41.540649] PVFS_isys_ref_lookup entered [D 20:00:41.540678] [SM Frame PUSH]: (0x15e755b8) frame: 0x15e62108 [D 20:00:41.540702] [SM Locating]: (0x15e755b8) op-id: 11 [D 20:00:41.540727] client_op_state_get_machine 11 [D 20:00:41.540752] [SM Locating]: (0x15e755b8) located: client_lookup_sm:lookup_segment_start [D 20:00:41.540778] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.540808] initialize_context called [D 20:00:41.540842] original pathname is: qtest.F08 [D 20:00:41.540867] cur_seg_name[0]: qtest.F08 [D 20:00:41.540891] pathname is: qtest.F08 [D 20:00:41.540917] *seg_remaining is: qtest.F08 [D 20:00:41.540942] lookup got: qtest.F08 (parent 138) [D 20:00:41.540966] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.540992] PINT_client_state_machine_post smcb 0x15e755b8, op: PVFS_SYS_LOOKUP [D 20:00:41.541018] [SM Entering]: (0x15e755b8) client_lookup_sm:lookup_segment_start (status: 0) [D 20:00:41.541044] lookup_segment_start [D 20:00:41.541068] [SM Exiting]: (0x15e755b8) client_lookup_sm:lookup_segment_start (error code: 0), (action: COMPLETE) [D 20:00:41.541093] [SM Entering]: (0x15e755b8) client_lookup_sm:lookup_segment_query_ncache (status: 0) [D 20:00:41.541119] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.541144] lookup_segment_query_ncache: segment [qtest.F08] [D 20:00:41.541169] ncache: get_cached_entry(): [qtest.F08] [D 20:00:41.541195] ncache: miss: name=[qtest.F08] [D 20:00:41.541223] *** ncache clean miss on first segment of qtest.F08 [D 20:00:41.541248] [SM Exiting]: (0x15e755b8) client_lookup_sm:lookup_segment_query_ncache (error code: 1), (action: COMPLETE) [D 20:00:41.541275] [SM Entering]: (0x15e755b8) client_lookup_sm:lookup_segment_setup_msgpair (status: 0) [D 20:00:41.541299] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.541325] lookup_segment_setup_msgpair [D 20:00:41.541350] Looking up segment qtest.F08 under handle 138 [D 20:00:41.541378] [SM Exiting]: (0x15e755b8) client_lookup_sm:lookup_segment_setup_msgpair (error code: 0), (action: COMPLETE) [D 20:00:41.541404] [SM push_state]: (0x15e755b8) op-id: 11 stk-ptr: 0 base-frm: 0 [D 20:00:41.541429] [SM Entering]: (0x15e755b8) msgpairarray_sm:init (status: 0) [D 20:00:41.541455] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.541480] (0x15e755b8) msgpairarray state: init (1 msgpair(s)) [D 20:00:41.541504] [SM Exiting]: (0x15e755b8) msgpairarray_sm:init (error code: 0), (action: COMPLETE) [D 20:00:41.541530] [SM Entering]: (0x15e755b8) msgpairarray_sm:post (status: 0) [D 20:00:41.541555] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.541581] msgpairarray_post: sm 0x15e755b8 2 total message(s) with 2 incomplete [D 20:00:41.541607] PINT_encode [D 20:00:41.541632] encode_common [D 20:00:41.541657] lebf_encode_req [D 20:00:41.541683] PINT_encode_calc_max_size [D 20:00:41.541739] msgpairarray_post: sm 0x15e755b8 msgpair 0: posting recv [D 20:00:41.541768] BMI_post_recv: addr: 367403208, offset: 0x15e770c0, size: 8224, tag: 2 [D 20:00:41.541802] msgpairarray_post: sm 0x15e755b8 msgpair 0: posting send [D 20:00:41.541830] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 60, tag: 2 [D 20:00:41.541855] element 0: offset: 0x15e75650, size: 60 [D 20:00:41.541885] post_sr: da13:3346 bh 1 len 76 wr_id 15e72b98. [D 20:00:41.541913] [SM Exiting]: (0x15e755b8) msgpairarray_sm:post (error code: 0), (action: DEFERRED) [D 20:00:41.541940] Posted PVFS_SYS_LOOKUP (8) (waiting for test)(-1073741839) [D 20:00:41.541967] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.541993] PINT_client_state_machine_test id 8 [D 20:00:41.542020] openib_check_cq: completed send wr_id 15e72b98. [D 20:00:41.542051] sm control op 11 [D 20:00:41.542076] [SM Entering]: (0x15e755b8) msgpairarray_sm:complete (status: 1) [D 20:00:41.542101] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.542129] msgpairarray_complete: sm 0x15e755b8 status_user_tag 1 msgarray_count 1 [D 20:00:41.542155] msgpairarray: 1 operations remain [D 20:00:41.542180] [SM Exiting]: (0x15e755b8) msgpairarray_sm:complete (error code: 0), (action: DEFERRED) [D 20:00:41.542205] PINT_client_state_machine_test id 8 [D 20:00:41.552233] PINT_client_state_machine_test id 8 [D 20:00:41.562263] PINT_client_state_machine_test id 8 [D 20:00:41.562363] openib_check_cq: completed recv wr_id 15e73500. [D 20:00:41.562392] openib_post_rr: da13:3346 bh 1 wr_id 15e73500. [D 20:00:41.562422] sm control op 11 [D 20:00:41.562448] [SM Entering]: (0x15e755b8) msgpairarray_sm:complete (status: 0) [D 20:00:41.562473] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.562499] msgpairarray_complete: sm 0x15e755b8 status_user_tag 0 msgarray_count 1 [D 20:00:41.562524] msgpairarray: all operations complete [D 20:00:41.562548] [SM Exiting]: (0x15e755b8) msgpairarray_sm:complete (error code: 190), (action: COMPLETE) [D 20:00:41.562574] [SM Entering]: (0x15e755b8) msgpairarray_sm:completion_fn (status: 0) [D 20:00:41.562603] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.562632] (0x15e755b8) msgpairarray state: completion_fn [D 20:00:41.562661] PINT_decode [D 20:00:41.562689] lebf_decode_resp [D 20:00:41.562718] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.562744] lookup_segment_lookup_comp_fn [D 20:00:41.562773] PINT_encode_release [D 20:00:41.562798] lebf_encode_rel [D 20:00:41.562824] PINT_decode_release [D 20:00:41.562848] lebf_decode_rel [D 20:00:41.562872] msgpairarray_completion_fn: sm 0x15e755b8 msgpair 0 marked complete [D 20:00:41.562898] [SM Exiting]: (0x15e755b8) msgpairarray_sm:completion_fn (error code: -1073741826), (action: COMPLETE) [D 20:00:41.562924] [SM pop_state]: (0x15e755b8) op-id: 11 stk-ptr: 1 base-frm: 0 [D 20:00:41.562949] [SM Entering]: (0x15e755b8) client_lookup_sm:lookup_segment_lookup_failure (status: 0) [D 20:00:41.562974] lookup state: lookup_segment_lookup_failure [D 20:00:41.562998] [SM Exiting]: (0x15e755b8) client_lookup_sm:lookup_segment_lookup_failure (error code: -1073741826), (action: COMPLETE) [D 20:00:41.563025] [SM Entering]: (0x15e755b8) client_lookup_sm:lookup_cleanup (status: 0) [D 20:00:41.563050] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.563076] lookup state: lookup_cleanup [D 20:00:41.563100] Freeing segment qtest.F08 [D 20:00:41.563124] Freeing remaining segment qtest.F08 [D 20:00:41.563152] All contexts finalized [D 20:00:41.563195] Lookup failed to resolve qtest.F08: No such file or directory [D 20:00:41.563220] [SM Exiting]: (0x15e755b8) client_lookup_sm:lookup_cleanup (error code: -1073741826), (action: TERMINATE) [D 20:00:41.563246] client_state_machine_terminate smcb 0x15e755b8 [D 20:00:41.563271] add smcb 0x15e755b8 to completion list [D 20:00:41.563296] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.563322] PINT_sys_release: id 8 [D 20:00:41.563348] [SM frame get]: (0x15e755b8) op-id: 11 index: 0 base-frm: 0 [D 20:00:41.563381] PVFS_sys_create entered [D 20:00:41.563411] PVFS_isys_create entered [D 20:00:41.563441] [SM Frame PUSH]: (0x15e75990) frame: 0x15e62108 [D 20:00:41.563467] [SM Locating]: (0x15e75990) op-id: 2 [D 20:00:41.563490] client_op_state_get_machine 2 [D 20:00:41.563520] [SM Locating]: (0x15e75990) located: client_create_sm:init [D 20:00:41.563547] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.563594] Creating file qtest.F08 under 138, 136 [D 20:00:41.563621] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.563647] PINT_client_state_machine_post smcb 0x15e75990, op: PVFS_SYS_CREATE [D 20:00:41.563673] [SM Entering]: (0x15e75990) client_create_sm:init (status: 0) [D 20:00:41.563698] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.563724] create state: init [D 20:00:41.563749] [SM Exiting]: (0x15e75990) client_create_sm:init (error code: 0), (action: COMPLETE) [D 20:00:41.563776] [SM push_state]: (0x15e75990) op-id: 2 stk-ptr: 0 base-frm: 0 [D 20:00:41.563802] [SM Entering]: (0x15e75990) client_getattr_sm:acache_lookup (status: 0) [D 20:00:41.563831] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.563858] getattr_acache_lookup: handle 138 fsid 136 [D 20:00:41.563889] acache: get_cached_entry(): H=138 [D 20:00:41.563919] acache: miss static: H=138 [D 20:00:41.563943] acache: miss non-static: H=138 [D 20:00:41.563972] acache: clean acache miss: [138] [D 20:00:41.563997] [SM Exiting]: (0x15e75990) client_getattr_sm:acache_lookup (error code: 1), (action: COMPLETE) [D 20:00:41.564023] [SM Entering]: (0x15e75990) client_getattr_sm:object_getattr_setup_msgpair (status: 0) [D 20:00:41.564048] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.564077] (0x15e62108) getattr_object_getattr_setup_msgpair [D 20:00:41.564105] [SM Exiting]: (0x15e75990) client_getattr_sm:object_getattr_setup_msgpair (error code: 0), (action: COMPLETE) [D 20:00:41.564131] [SM push_state]: (0x15e75990) op-id: 2 stk-ptr: 1 base-frm: 0 [D 20:00:41.564157] [SM Entering]: (0x15e75990) msgpairarray_sm:init (status: 0) [D 20:00:41.564182] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.564208] (0x15e75990) msgpairarray state: init (1 msgpair(s)) [D 20:00:41.564234] [SM Exiting]: (0x15e75990) msgpairarray_sm:init (error code: 0), (action: COMPLETE) [D 20:00:41.564263] [SM Entering]: (0x15e75990) msgpairarray_sm:post (status: 0) [D 20:00:41.564289] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.564314] msgpairarray_post: sm 0x15e75990 2 total message(s) with 2 incomplete [D 20:00:41.564341] PINT_encode [D 20:00:41.564366] encode_common [D 20:00:41.564392] lebf_encode_req [D 20:00:41.564417] PINT_encode_calc_max_size [D 20:00:41.564471] msgpairarray_post: sm 0x15e75990 msgpair 0: posting recv [D 20:00:41.564499] BMI_post_recv: addr: 367403208, offset: 0x15e790e8, size: 9280, tag: 3 [D 20:00:41.564534] msgpairarray_post: sm 0x15e75990 msgpair 0: posting send [D 20:00:41.564563] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 40, tag: 3 [D 20:00:41.564595] element 0: offset: 0x15e628e8, size: 40 [D 20:00:41.564625] post_sr: da13:3346 bh 2 len 56 wr_id 15e72bb0. [D 20:00:41.564654] [SM Exiting]: (0x15e75990) msgpairarray_sm:post (error code: 0), (action: DEFERRED) [D 20:00:41.564681] Posted PVFS_SYS_CREATE (11) (waiting for test)(-1073741839) [D 20:00:41.564707] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.564733] PINT_client_state_machine_test id 11 [D 20:00:41.564762] openib_check_cq: completed send wr_id 15e72bb0. [D 20:00:41.564792] sm control op 2 [D 20:00:41.564818] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 1) [D 20:00:41.564844] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.564868] msgpairarray_complete: sm 0x15e75990 status_user_tag 1 msgarray_count 1 [D 20:00:41.564894] msgpairarray: 1 operations remain [D 20:00:41.564919] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 0), (action: DEFERRED) [D 20:00:41.564950] PINT_client_state_machine_test id 11 [D 20:00:41.574978] PINT_client_state_machine_test id 11 [D 20:00:41.585012] PINT_client_state_machine_test id 11 [D 20:00:41.600590] PINT_client_state_machine_test id 11 [D 20:00:41.612589] PINT_client_state_machine_test id 11 [D 20:00:41.624590] PINT_client_state_machine_test id 11 [D 20:00:41.636590] PINT_client_state_machine_test id 11 [D 20:00:41.648590] PINT_client_state_machine_test id 11 [D 20:00:41.660590] PINT_client_state_machine_test id 11 [D 20:00:41.672591] PINT_client_state_machine_test id 11 [D 20:00:41.684591] PINT_client_state_machine_test id 11 [D 20:00:41.696590] PINT_client_state_machine_test id 11 [D 20:00:41.708590] PINT_client_state_machine_test id 11 [D 20:00:41.720590] PINT_client_state_machine_test id 11 [D 20:00:41.732590] PINT_client_state_machine_test id 11 [D 20:00:41.744590] PINT_client_state_machine_test id 11 [D 20:00:41.756590] PINT_client_state_machine_test id 11 [D 20:00:41.768591] PINT_client_state_machine_test id 11 [D 20:00:41.780591] PINT_client_state_machine_test id 11 -[D 20:00:41.789361] openib_check_cq: completed recv wr_id 15e73518. [D 20:00:41.789409] openib_post_rr: da13:3346 bh 2 wr_id 15e73518. [D 20:00:41.789452] sm control op 2 [D 20:00:41.789479] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 0) [D 20:00:41.789505] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.789531] msgpairarray_complete: sm 0x15e75990 status_user_tag 0 msgarray_count 1 [D 20:00:41.789557] msgpairarray: all operations complete [D 20:00:41.789582] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 190), (action: COMPLETE) [D 20:00:41.789608] [SM Entering]: (0x15e75990) msgpairarray_sm:completion_fn (status: 0) [D 20:00:41.789633] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.789659] (0x15e75990) msgpairarray state: completion_fn [D 20:00:41.789685] PINT_decode [D 20:00:41.789710] lebf_decode_resp [D 20:00:41.789750] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.789775] getattr_object_getattr_comp_fn called [D 20:00:41.789801] getattr comp_fn [0x15e62138] dfile_count = 0 dist_name_len = 0 dist_params_len = 0 [D 20:00:41.789827] PINT_encode_release [D 20:00:41.789851] lebf_encode_rel [D 20:00:41.789876] PINT_decode_release [D 20:00:41.789900] lebf_decode_rel [D 20:00:41.789926] msgpairarray_completion_fn: sm 0x15e75990 msgpair 0 marked complete [D 20:00:41.789951] [SM Exiting]: (0x15e75990) msgpairarray_sm:completion_fn (error code: 0), (action: COMPLETE) [D 20:00:41.789977] [SM pop_state]: (0x15e75990) op-id: 2 stk-ptr: 2 base-frm: 0 [D 20:00:41.790002] [SM Entering]: (0x15e75990) client_getattr_sm:acache_insert (status: 0) [D 20:00:41.790028] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.790058] acache: update(): H=138 [D 20:00:41.790091] trying to add object reference to acache [D 20:00:41.790116] [SM Exiting]: (0x15e75990) client_getattr_sm:acache_insert (error code: 0), (action: COMPLETE) [D 20:00:41.790142] [SM Entering]: (0x15e75990) client_getattr_sm:cleanup (status: 0) [D 20:00:41.790167] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.790192] (0x15e62108) getattr state: getattr_cleanup [D 20:00:41.790217] [SM Exiting]: (0x15e75990) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:41.790243] [SM pop_state]: (0x15e75990) op-id: 2 stk-ptr: 1 base-frm: 0 [D 20:00:41.790267] [SM Entering]: (0x15e75990) client_create_sm:parent_getattr_inspect (status: 0) [D 20:00:41.790293] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.790318] create state: parent_getattr_inspect [D 20:00:41.790343] parent owner: 0, group: 0, perms: 511 [D 20:00:41.790367] create_parent_getattr: [0x15e62138] dfile_count = 0 dist_name_len = 0 dist_params_len = 0 [D 20:00:41.790392] Setting number of datafiles to 1 [requested 0] [D 20:00:41.790417] [SM Exiting]: (0x15e75990) client_create_sm:parent_getattr_inspect (error code: 0), (action: COMPLETE) [D 20:00:41.790443] [SM Entering]: (0x15e75990) client_create_sm:dspace_create_setup_msgpair (status: 0) [D 20:00:41.790472] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.790498] create state: dspace_create_setup_msgpair [D 20:00:41.790522] need to create 1 datafiles [D 20:00:41.790546] create: posting create req [D 20:00:41.790588] [SM Exiting]: (0x15e75990) client_create_sm:dspace_create_setup_msgpair (error code: 0), (action: COMPLETE) [D 20:00:41.790615] [SM push_state]: (0x15e75990) op-id: 2 stk-ptr: 0 base-frm: 0 [D 20:00:41.790639] [SM Entering]: (0x15e75990) msgpairarray_sm:init (status: 0) [D 20:00:41.790665] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.790690] (0x15e75990) msgpairarray state: init (1 msgpair(s)) [D 20:00:41.790715] [SM Exiting]: (0x15e75990) msgpairarray_sm:init (error code: 0), (action: COMPLETE) [D 20:00:41.790740] [SM Entering]: (0x15e75990) msgpairarray_sm:post (status: 0) [D 20:00:41.790770] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.790795] msgpairarray_post: sm 0x15e75990 2 total message(s) with 2 incomplete [D 20:00:41.790823] PINT_encode [D 20:00:41.790848] encode_common [D 20:00:41.790914] lebf_encode_req [D 20:00:41.790942] PINT_encode_calc_max_size [D 20:00:41.790967] msgpairarray_post: sm 0x15e75990 msgpair 0: posting recv [D 20:00:41.790995] BMI_post_recv: addr: 367403208, offset: 0x15e628b8, size: 24, tag: 4 [D 20:00:41.791030] msgpairarray_post: sm 0x15e75990 msgpair 0: posting send [D 20:00:41.791058] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 56, tag: 4 [D 20:00:41.791087] element 0: offset: 0x15e7b530, size: 56 [D 20:00:41.791117] post_sr: da13:3346 bh 3 len 72 wr_id 15e72bc8. [D 20:00:41.791146] [SM Exiting]: (0x15e75990) msgpairarray_sm:post (error code: 0), (action: DEFERRED) [D 20:00:41.791172] PINT_client_state_machine_test id 11 [D 20:00:41.791201] openib_check_cq: completed send wr_id 15e72bc8. [D 20:00:41.791231] sm control op 2 [D 20:00:41.791255] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 1) [D 20:00:41.791280] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.791306] msgpairarray_complete: sm 0x15e75990 status_user_tag 1 msgarray_count 1 [D 20:00:41.791331] msgpairarray: 1 operations remain [D 20:00:41.791356] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 0), (action: DEFERRED) [D 20:00:41.791381] PINT_client_state_machine_test id 11 [D 20:00:41.801409] PINT_client_state_machine_test id 11 [D 20:00:41.811438] PINT_client_state_machine_test id 11 -[D 20:00:41.814108] openib_check_cq: completed recv wr_id 15e73530. [D 20:00:41.814141] openib_post_rr: da13:3346 bh 3 wr_id 15e73530. [D 20:00:41.814175] sm control op 2 [D 20:00:41.814200] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 0) [D 20:00:41.814226] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.814250] msgpairarray_complete: sm 0x15e75990 status_user_tag 0 msgarray_count 1 [D 20:00:41.814276] msgpairarray: all operations complete [D 20:00:41.814300] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 190), (action: COMPLETE) [D 20:00:41.814326] [SM Entering]: (0x15e75990) msgpairarray_sm:completion_fn (status: 0) [D 20:00:41.814351] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.814375] (0x15e75990) msgpairarray state: completion_fn [D 20:00:41.814400] PINT_decode [D 20:00:41.814424] lebf_decode_resp [D 20:00:41.814449] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.814475] create_create_comp_fn [D 20:00:41.814498] *** Got newly created handle 146 [D 20:00:41.814523] PINT_encode_release [D 20:00:41.814547] lebf_encode_rel [D 20:00:41.814572] PINT_decode_release [D 20:00:41.814596] lebf_decode_rel [D 20:00:41.814620] msgpairarray_completion_fn: sm 0x15e75990 msgpair 0 marked complete [D 20:00:41.814646] [SM Exiting]: (0x15e75990) msgpairarray_sm:completion_fn (error code: 0), (action: COMPLETE) [D 20:00:41.814672] [SM pop_state]: (0x15e75990) op-id: 2 stk-ptr: 1 base-frm: 0 [D 20:00:41.814697] [SM Entering]: (0x15e75990) client_create_sm:datafiles_setup_msgpair_array (status: 0) [D 20:00:41.814723] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.814747] create state: datafiles_setup_msgpair_array [D 20:00:41.814779] posting datafile[0] create with extents 1431655768-2863311531 [D 20:00:41.814805] [SM Exiting]: (0x15e75990) client_create_sm:datafiles_setup_msgpair_array (error code: 0), (action: COMPLETE) [D 20:00:41.814831] [SM push_state]: (0x15e75990) op-id: 2 stk-ptr: 0 base-frm: 0 [D 20:00:41.814856] [SM Entering]: (0x15e75990) msgpairarray_sm:init (status: 0) [D 20:00:41.814881] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.814907] (0x15e75990) msgpairarray state: init (1 msgpair(s)) [D 20:00:41.814931] [SM Exiting]: (0x15e75990) msgpairarray_sm:init (error code: 0), (action: COMPLETE) [D 20:00:41.814956] [SM Entering]: (0x15e75990) msgpairarray_sm:post (status: 0) [D 20:00:41.814985] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.815010] msgpairarray_post: sm 0x15e75990 2 total message(s) with 2 incomplete [D 20:00:41.815036] PINT_encode [D 20:00:41.815060] encode_common [D 20:00:41.815086] lebf_encode_req [D 20:00:41.815117] PINT_encode_calc_max_size [D 20:00:41.815146] msgpairarray_post: sm 0x15e75990 msgpair 0: posting recv [D 20:00:41.815176] BMI_post_recv: addr: 367403208, offset: 0x15e628b8, size: 24, tag: 5 [D 20:00:41.815214] msgpairarray_post: sm 0x15e75990 msgpair 0: posting send [D 20:00:41.815241] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 56, tag: 5 [D 20:00:41.815267] element 0: offset: 0x15e7b530, size: 56 [D 20:00:41.815296] post_sr: da13:3346 bh 4 len 72 wr_id 15e72be0. [D 20:00:41.815324] [SM Exiting]: (0x15e75990) msgpairarray_sm:post (error code: 0), (action: DEFERRED) [D 20:00:41.815350] PINT_client_state_machine_test id 11 [D 20:00:41.815379] openib_check_cq: completed send wr_id 15e72be0. [D 20:00:41.815408] sm control op 2 [D 20:00:41.815433] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 1) [D 20:00:41.815458] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.815484] msgpairarray_complete: sm 0x15e75990 status_user_tag 1 msgarray_count 1 [D 20:00:41.815508] msgpairarray: 1 operations remain [D 20:00:41.815533] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 0), (action: DEFERRED) [D 20:00:41.815559] PINT_client_state_machine_test id 11 [D 20:00:41.825586] PINT_client_state_machine_test id 11 [D 20:00:41.835615] PINT_client_state_machine_test id 11 -[D 20:00:41.838972] openib_check_cq: completed recv wr_id 15e73548. [D 20:00:41.839011] openib_post_rr: da13:3346 bh 4 wr_id 15e73548. [D 20:00:41.839042] sm control op 2 [D 20:00:41.839068] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 0) [D 20:00:41.839093] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.839118] msgpairarray_complete: sm 0x15e75990 status_user_tag 0 msgarray_count 1 [D 20:00:41.839144] msgpairarray: all operations complete [D 20:00:41.839168] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 190), (action: COMPLETE) [D 20:00:41.839194] [SM Entering]: (0x15e75990) msgpairarray_sm:completion_fn (status: 0) [D 20:00:41.839219] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.839244] (0x15e75990) msgpairarray state: completion_fn [D 20:00:41.839269] PINT_decode [D 20:00:41.839293] lebf_decode_resp [D 20:00:41.839318] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.839343] create_datafiles_comp_fn[0] [D 20:00:41.839367] Datafile handle 0 is 2863311531 [D 20:00:41.839392] PINT_encode_release [D 20:00:41.839416] lebf_encode_rel [D 20:00:41.839441] PINT_decode_release [D 20:00:41.839465] lebf_decode_rel [D 20:00:41.839489] msgpairarray_completion_fn: sm 0x15e75990 msgpair 0 marked complete [D 20:00:41.839515] [SM Exiting]: (0x15e75990) msgpairarray_sm:completion_fn (error code: 0), (action: COMPLETE) [D 20:00:41.839541] [SM pop_state]: (0x15e75990) op-id: 2 stk-ptr: 1 base-frm: 0 [D 20:00:41.839566] [SM Entering]: (0x15e75990) client_create_sm:create_setattr_setup_msgpair (status: 0) [D 20:00:41.839593] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.839619] create state: setattr_setup_msgpair [D 20:00:41.839643] create: posting setattr req [D 20:00:41.839671] [SM Exiting]: (0x15e75990) client_create_sm:create_setattr_setup_msgpair (error code: 0), (action: COMPLETE) [D 20:00:41.839696] [SM push_state]: (0x15e75990) op-id: 2 stk-ptr: 0 base-frm: 0 [D 20:00:41.839721] [SM Entering]: (0x15e75990) msgpairarray_sm:init (status: 0) [D 20:00:41.839747] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.839772] (0x15e75990) msgpairarray state: init (1 msgpair(s)) [D 20:00:41.839797] [SM Exiting]: (0x15e75990) msgpairarray_sm:init (error code: 0), (action: COMPLETE) [D 20:00:41.839821] [SM Entering]: (0x15e75990) msgpairarray_sm:post (status: 0) [D 20:00:41.839849] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.839875] msgpairarray_post: sm 0x15e75990 2 total message(s) with 2 incomplete [D 20:00:41.839901] PINT_encode [D 20:00:41.839925] encode_common [D 20:00:41.839976] lebf_encode_req [D 20:00:41.840006] PINT_encode_calc_max_size [D 20:00:41.840031] msgpairarray_post: sm 0x15e75990 msgpair 0: posting recv [D 20:00:41.840058] BMI_post_recv: addr: 367403208, offset: 0x15e72ae8, size: 16, tag: 6 [D 20:00:41.840091] msgpairarray_post: sm 0x15e75990 msgpair 0: posting send [D 20:00:41.840118] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 144, tag: 6 [D 20:00:41.840144] element 0: offset: 0x15e7f560, size: 144 [D 20:00:41.840173] post_sr: da13:3346 bh 5 len 160 wr_id 15e72bf8. [D 20:00:41.840201] [SM Exiting]: (0x15e75990) msgpairarray_sm:post (error code: 0), (action: DEFERRED) [D 20:00:41.840228] PINT_client_state_machine_test id 11 [D 20:00:41.840256] openib_check_cq: completed send wr_id 15e72bf8. [D 20:00:41.840286] sm control op 2 [D 20:00:41.840311] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 1) [D 20:00:41.840335] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:41.840360] msgpairarray_complete: sm 0x15e75990 status_user_tag 1 msgarray_count 1 [D 20:00:41.840386] msgpairarray: 1 operations remain [D 20:00:41.840410] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 0), (action: DEFERRED) [D 20:00:41.840435] PINT_client_state_machine_test id 11 [D 20:00:41.850463] PINT_client_state_machine_test id 11 [D 20:00:41.860492] PINT_client_state_machine_test id 11 [D 20:00:41.871591] PINT_client_state_machine_test id 11 [D 20:00:41.883591] PINT_client_state_machine_test id 11 [D 20:00:41.895590] PINT_client_state_machine_test id 11 [D 20:00:41.907590] PINT_client_state_machine_test id 11 [D 20:00:41.919590] PINT_client_state_machine_test id 11 [D 20:00:41.931591] PINT_client_state_machine_test id 11 [D 20:00:41.943591] PINT_client_state_machine_test id 11 [D 20:00:41.955591] PINT_client_state_machine_test id 11 [D 20:00:41.967592] PINT_client_state_machine_test id 11 [D 20:00:41.979591] PINT_client_state_machine_test id 11 [D 20:00:41.991602] PINT_client_state_machine_test id 11 [D 20:00:42.005593] PINT_client_state_machine_test id 11 [D 20:00:42.017592] PINT_client_state_machine_test id 11 [D 20:00:42.029592] PINT_client_state_machine_test id 11 [D 20:00:42.041592] PINT_client_state_machine_test id 11 [D 20:00:42.053592] PINT_client_state_machine_test id 11 -[D 20:00:42.063876] openib_check_cq: completed recv wr_id 15e73560. [D 20:00:42.063914] openib_post_rr: da13:3346 bh 5 wr_id 15e73560. [D 20:00:42.063947] sm control op 2 [D 20:00:42.063972] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 0) [D 20:00:42.063998] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.064023] msgpairarray_complete: sm 0x15e75990 status_user_tag 0 msgarray_count 1 [D 20:00:42.064049] msgpairarray: all operations complete [D 20:00:42.064073] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 190), (action: COMPLETE) [D 20:00:42.064099] [SM Entering]: (0x15e75990) msgpairarray_sm:completion_fn (status: 0) [D 20:00:42.064125] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.064149] (0x15e75990) msgpairarray state: completion_fn [D 20:00:42.064174] PINT_decode [D 20:00:42.064199] lebf_decode_resp [D 20:00:42.064224] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.064249] create_setattr_comp_fn [D 20:00:42.064275] PINT_encode_release [D 20:00:42.064300] lebf_encode_rel [D 20:00:42.064325] PINT_decode_release [D 20:00:42.064349] lebf_decode_rel [D 20:00:42.064373] msgpairarray_completion_fn: sm 0x15e75990 msgpair 0 marked complete [D 20:00:42.064399] [SM Exiting]: (0x15e75990) msgpairarray_sm:completion_fn (error code: 0), (action: COMPLETE) [D 20:00:42.064425] [SM pop_state]: (0x15e75990) op-id: 2 stk-ptr: 1 base-frm: 0 [D 20:00:42.064451] [SM Entering]: (0x15e75990) client_create_sm:crdirent_setup_msgpair (status: 0) [D 20:00:42.064479] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.064504] create state: crdirent_setup_msgpair [D 20:00:42.064528] create: create_crdirent_setup_msgpair: posting crdirent req: parent handle: 138, name: qtest.F08, handle: 146 [D 20:00:42.064558] [SM Exiting]: (0x15e75990) client_create_sm:crdirent_setup_msgpair (error code: 0), (action: COMPLETE) [D 20:00:42.064584] [SM push_state]: (0x15e75990) op-id: 2 stk-ptr: 0 base-frm: 0 [D 20:00:42.064616] [SM Entering]: (0x15e75990) msgpairarray_sm:init (status: 0) [D 20:00:42.064644] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.064673] (0x15e75990) msgpairarray state: init (1 msgpair(s)) [D 20:00:42.064702] [SM Exiting]: (0x15e75990) msgpairarray_sm:init (error code: 0), (action: COMPLETE) [D 20:00:42.064727] [SM Entering]: (0x15e75990) msgpairarray_sm:post (status: 0) [D 20:00:42.064753] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.064778] msgpairarray_post: sm 0x15e75990 2 total message(s) with 2 incomplete [D 20:00:42.064805] PINT_encode [D 20:00:42.064829] encode_common [D 20:00:42.064855] lebf_encode_req [D 20:00:42.064880] PINT_encode_calc_max_size [D 20:00:42.064905] msgpairarray_post: sm 0x15e75990 msgpair 0: posting recv [D 20:00:42.064931] BMI_post_recv: addr: 367403208, offset: 0x15e72ae8, size: 16, tag: 7 [D 20:00:42.064965] msgpairarray_post: sm 0x15e75990 msgpair 0: posting send [D 20:00:42.064992] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 60, tag: 7 [D 20:00:42.065018] element 0: offset: 0x15e75b78, size: 60 [D 20:00:42.065048] post_sr: da13:3346 bh 6 len 76 wr_id 15e72c10. [D 20:00:42.065075] [SM Exiting]: (0x15e75990) msgpairarray_sm:post (error code: 0), (action: DEFERRED) [D 20:00:42.065102] PINT_client_state_machine_test id 11 [D 20:00:42.065130] openib_check_cq: completed send wr_id 15e72c10. [D 20:00:42.065159] sm control op 2 [D 20:00:42.065184] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 1) [D 20:00:42.065210] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.065235] msgpairarray_complete: sm 0x15e75990 status_user_tag 1 msgarray_count 1 [D 20:00:42.065260] msgpairarray: 1 operations remain [D 20:00:42.065284] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 0), (action: DEFERRED) [D 20:00:42.065310] PINT_client_state_machine_test id 11 [D 20:00:42.075338] PINT_client_state_machine_test id 11 [D 20:00:42.085368] PINT_client_state_machine_test id 11 -[D 20:00:42.094574] openib_check_cq: completed recv wr_id 15e73578. [D 20:00:42.094613] openib_post_rr: da13:3346 bh 6 wr_id 15e73578. [D 20:00:42.094649] sm control op 2 [D 20:00:42.094677] [SM Entering]: (0x15e75990) msgpairarray_sm:complete (status: 0) [D 20:00:42.094703] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.094729] msgpairarray_complete: sm 0x15e75990 status_user_tag 0 msgarray_count 1 [D 20:00:42.094755] msgpairarray: all operations complete [D 20:00:42.094779] [SM Exiting]: (0x15e75990) msgpairarray_sm:complete (error code: 190), (action: COMPLETE) [D 20:00:42.094804] [SM Entering]: (0x15e75990) msgpairarray_sm:completion_fn (status: 0) [D 20:00:42.094830] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.094855] (0x15e75990) msgpairarray state: completion_fn [D 20:00:42.094880] PINT_decode [D 20:00:42.094904] lebf_decode_resp [D 20:00:42.094929] create_crdirent_comp_fn [D 20:00:42.094953] PINT_encode_release [D 20:00:42.094977] lebf_encode_rel [D 20:00:42.095002] PINT_decode_release [D 20:00:42.095026] lebf_decode_rel [D 20:00:42.095050] msgpairarray_completion_fn: sm 0x15e75990 msgpair 0 marked complete [D 20:00:42.095076] [SM Exiting]: (0x15e75990) msgpairarray_sm:completion_fn (error code: 0), (action: COMPLETE) [D 20:00:42.095102] [SM pop_state]: (0x15e75990) op-id: 2 stk-ptr: 1 base-frm: 0 [D 20:00:42.095127] [SM Entering]: (0x15e75990) client_create_sm:cleanup (status: 0) [D 20:00:42.095152] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.095181] create state: cleanup [D 20:00:42.095207] ncache: update(): name [qtest.F08] [D 20:00:42.095235] ncache: update(): return=0 [D 20:00:42.095260] acache: update(): H=146 [D 20:00:42.095292] [SM Exiting]: (0x15e75990) client_create_sm:cleanup (error code: 0), (action: TERMINATE) [D 20:00:42.095318] client_state_machine_terminate smcb 0x15e75990 [D 20:00:42.095343] add smcb 0x15e75990 to completion list [D 20:00:42.095368] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 [D 20:00:42.095395] PINT_sys_release: id 11 [D 20:00:42.095419] [SM frame get]: (0x15e75990) op-id: 2 index: 0 base-frm: 0 pvfs2_aio_open created new file qtest.F08 opened /pvfs/da13-test/qtest.F08 for bypassio using module: PVFS2 native AIO (pvfs2_isys_*()) pvfs2_aio_rewind nothing yet get_free_buffer id 0 new buf # 1 total buffer count 1 pvfs2_aio_flush id 0 buf 1 offset 0, size 33554432 [D 20:00:43.238503] PVFS_isys_io entered [146] [D 20:00:43.238562] [SM Frame PUSH]: (0x15e75b78) frame: 0x15e62108 [D 20:00:43.238590] [SM Locating]: (0x15e75b78) op-id: 6 [D 20:00:43.238615] client_op_state_get_machine 6 [D 20:00:43.238640] [SM Locating]: (0x15e75b78) located: client_io_sm:init [D 20:00:43.238666] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:43.238694] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:43.238719] PINT_client_state_machine_post smcb 0x15e75b78, op: PVFS_SYS_IO [D 20:00:43.238748] [SM Entering]: (0x15e75b78) client_io_sm:init (status: 0) [D 20:00:43.238774] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:43.238800] (0x15e62108) io state: io_init [D 20:00:43.238824] [SM Exiting]: (0x15e75b78) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:43.238851] [SM push_state]: (0x15e75b78) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:43.238876] [SM Entering]: (0x15e75b78) client_getattr_sm:acache_lookup (status: 0) [D 20:00:43.238903] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:43.238927] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:43.238954] acache: get_cached_entry(): H=146 [D 20:00:43.238984] acache: copying out attr. [D 20:00:43.239009] acache: copying out size. [D 20:00:43.239035] acache: hit: H=146, size_status=0, attr_status=0 [D 20:00:43.239060] acache: acache hit [146] [D 20:00:43.239085] [SM Exiting]: (0x15e75b78) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:43.239110] [SM Entering]: (0x15e75b78) client_getattr_sm:cleanup (status: 0) [D 20:00:43.239136] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:43.239162] (0x15e62108) getattr state: getattr_cleanup [D 20:00:43.239187] [SM Exiting]: (0x15e75b78) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:43.239212] [SM pop_state]: (0x15e75b78) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:43.239237] [SM Entering]: (0x15e75b78) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:43.239263] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:43.239289] (0x15e62108) io state: io_datafile_setup_msgpairs [D 20:00:43.239321] - io_find_target_datafiles called [D 20:00:43.239350] PINT_new_request_states n=1 [D 20:00:43.239377] PINT_new_request_states n=1 [D 20:00:43.239414] ========================================================= [D 20:00:43.239439] PINT_process_request [D 20:00:43.239465] tiling 2 copies [D 20:00:43.239490] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:43.239517] lvl 0 el 0 blk 0 by 0 [D 20:00:43.239542] to 0 ta 0 fi 33554432 [D 20:00:43.239566] mto 0 mta 0 mfi 33554432 [D 20:00:43.239592] basic type or contiguous data [D 20:00:43.239624] Request disp [D 20:00:43.239653] contig_offset = 0 contig_size = 67108864 lvl_flag = 1 [D 20:00:43.239683] calling distribute [D 20:00:43.239713] PINT_distribute [D 20:00:43.239738] of 0 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:43.239768] begin iteration loff: 0 [D 20:00:43.239793] segment exceeds byte limit [D 20:00:43.239818] file being extended [D 20:00:43.239847] client lstof 0 diff 0 sgof 0 [D 20:00:43.239871] **********CALL***PROCESS********* [D 20:00:43.239896] segment of 0 sz 8177 [D 20:00:43.239920] PINT_process_request [D 20:00:43.239945] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:43.239971] lvl 0 el 0 blk 0 by 0 [D 20:00:43.239995] to 0 ta 0 fi 8177 [D 20:00:43.240020] basic type or contiguous data [D 20:00:43.240044] Request disp [D 20:00:43.240068] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:43.240094] process a segment [D 20:00:43.240118] of 0 sz 8177 [D 20:00:43.240141] add a segment 0 [D 20:00:43.240166] retval = 8177 [D 20:00:43.240190] exiting distribute returned less than expected [D 20:00:43.240214] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:43.240241] *****RETURN***FROM***PROCESS***** [D 20:00:43.240265] end iteration [D 20:00:43.240289] done with segments or bytes [D 20:00:43.240313] of 8177 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:43.240339] next loff: 8177 next poff: 8177 [D 20:00:43.240364] return value 8177 [D 20:00:43.240388] finished [D 20:00:43.240412] retval = 8177 [D 20:00:43.240436] exiting distribute returned less than expected [D 20:00:43.240460] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 33554432 eof 0 [D 20:00:43.240487] ========================================================= [D 20:00:43.240511] PINT_encode_calc_max_size [D 20:00:43.240537] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:43.240564] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:43.240592] initializing context[0] 0x15e74cc8 [D 20:00:43.240621] filling I/O request for 2863311531 [D 20:00:43.240647] [SM Exiting]: (0x15e75b78) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:43.240674] [SM Entering]: (0x15e75b78) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:43.240698] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:43.240724] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:43.240750] PINT_encode [D 20:00:43.240775] encode_common [D 20:00:43.240853] lebf_encode_req [D 20:00:43.240881] PINT_do_request_commit: commit node 0x15e75990 [D 20:00:43.240906] node stored at 0 [D 20:00:43.240931] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:43.240956] node stored at 1 [D 20:00:43.240987] clearing tree [D 20:00:43.241016] PINT_encode_calc_max_size [D 20:00:43.241042] posting recv with status_user_tag=0 (max_size 24) [D 20:00:43.241071] BMI_post_recv: addr: 367403208, offset: 0x15e75d98, size: 24, tag: 8 [D 20:00:43.241113] posting send with status_user_tag=1 [D 20:00:43.241141] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 8 [D 20:00:43.241167] element 0: offset: 0x15e819c0, size: 272 [D 20:00:43.241198] post_sr: da13:3346 bh 7 len 288 wr_id 15e72c28. [D 20:00:43.241229] io_datafile_post_msgpairs: completion count is 2 [D 20:00:43.241254] [SM Exiting]: (0x15e75b78) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:43.241281] Posted PVFS_SYS_IO (22) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 2 total buffer count 2 pvfs2_aio_flush id 0 buf 2 offset 33554432, size 33554432 [D 20:00:44.057507] PVFS_isys_io entered [146] [D 20:00:44.057565] [SM Frame PUSH]: (0x15e75f20) frame: 0x15e75fb8 [D 20:00:44.057592] [SM Locating]: (0x15e75f20) op-id: 6 [D 20:00:44.057720] client_op_state_get_machine 6 [D 20:00:44.057745] [SM Locating]: (0x15e75f20) located: client_io_sm:init [D 20:00:44.057771] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.057799] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.057825] PINT_client_state_machine_post smcb 0x15e75f20, op: PVFS_SYS_IO [D 20:00:44.057851] [SM Entering]: (0x15e75f20) client_io_sm:init (status: 0) [D 20:00:44.057877] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.057903] (0x15e75fb8) io state: io_init [D 20:00:44.057928] [SM Exiting]: (0x15e75f20) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:44.057961] [SM push_state]: (0x15e75f20) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:44.057987] [SM Entering]: (0x15e75f20) client_getattr_sm:acache_lookup (status: 0) [D 20:00:44.058013] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.058038] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:44.058064] acache: get_cached_entry(): H=146 [D 20:00:44.058094] acache: copying out attr. [D 20:00:44.058119] acache: copying out size. [D 20:00:44.058145] acache: hit: H=146, size_status=0, attr_status=0 [D 20:00:44.058170] acache: acache hit [146] [D 20:00:44.058195] [SM Exiting]: (0x15e75f20) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:44.058221] [SM Entering]: (0x15e75f20) client_getattr_sm:cleanup (status: 0) [D 20:00:44.058246] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.058272] (0x15e75fb8) getattr state: getattr_cleanup [D 20:00:44.058297] [SM Exiting]: (0x15e75f20) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:44.058322] [SM pop_state]: (0x15e75f20) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:44.058347] [SM Entering]: (0x15e75f20) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:44.058373] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.058399] (0x15e75fb8) io state: io_datafile_setup_msgpairs [D 20:00:44.058426] - io_find_target_datafiles called [D 20:00:44.058451] PINT_new_request_states n=1 [D 20:00:44.058478] PINT_new_request_states n=1 [D 20:00:44.058510] ========================================================= [D 20:00:44.058536] PINT_process_request [D 20:00:44.058561] tiling 3 copies [D 20:00:44.058585] skipping ahead to target_offset [D 20:00:44.058609] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:44.058636] lvl 0 el 0 blk 0 by 0 [D 20:00:44.058661] to 0 ta 33554432 fi 67108864 [D 20:00:44.058688] mto 0 mta 0 mfi 33554432 [D 20:00:44.058713] basic type or contiguous data [D 20:00:44.058737] Request disp [D 20:00:44.058762] contig_offset = 0 contig_size = 100663296 lvl_flag = 1 [D 20:00:44.058787] process logical skip [D 20:00:44.058812] retval = 33554432 [D 20:00:44.058835] exiting logical skip because distribute indicates done [D 20:00:44.058860] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:44.058886] lvl 0 el 0 blk 0 by 33554432 [D 20:00:44.058912] to 33554432 ta 33554432 fi 67108864 [D 20:00:44.058937] mto 0 mta 0 mfi 33554432 [D 20:00:44.058961] basic type or contiguous data [D 20:00:44.058985] Request disp [D 20:00:44.059009] contig_offset = 33554432 contig_size = 67108864 lvl_flag = 1 [D 20:00:44.059035] calling distribute [D 20:00:44.059060] PINT_distribute [D 20:00:44.059084] of 33554432 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:44.059113] begin iteration loff: 33554432 [D 20:00:44.059140] segment exceeds byte limit [D 20:00:44.059164] file being extended [D 20:00:44.059188] client lstof 0 diff 0 sgof 0 [D 20:00:44.059212] **********CALL***PROCESS********* [D 20:00:44.059237] segment of 0 sz 8177 [D 20:00:44.059262] PINT_process_request [D 20:00:44.059286] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:44.059312] lvl 0 el 0 blk 0 by 0 [D 20:00:44.059337] to 0 ta 0 fi 8177 [D 20:00:44.059361] basic type or contiguous data [D 20:00:44.059386] Request disp [D 20:00:44.059410] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:44.059434] process a segment [D 20:00:44.059458] of 0 sz 8177 [D 20:00:44.059483] add a segment 0 [D 20:00:44.059507] retval = 8177 [D 20:00:44.059531] exiting distribute returned less than expected [D 20:00:44.059555] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:44.059581] *****RETURN***FROM***PROCESS***** [D 20:00:44.059610] end iteration [D 20:00:44.059638] done with segments or bytes [D 20:00:44.059666] of 33562609 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:44.059699] next loff: 33562609 next poff: 33562609 [D 20:00:44.059726] return value 8177 [D 20:00:44.059751] finished [D 20:00:44.059774] retval = 8177 [D 20:00:44.059799] exiting distribute returned less than expected [D 20:00:44.059823] done sg 1 sm 1 by 8177 bm 8177 ta 33554432 to 33562609 fo 67108864 eof 0 [D 20:00:44.059849] ========================================================= [D 20:00:44.059874] PINT_encode_calc_max_size [D 20:00:44.059900] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:44.059926] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:44.059954] initializing context[0] 0x15e76730 [D 20:00:44.059980] filling I/O request for 2863311531 [D 20:00:44.060005] [SM Exiting]: (0x15e75f20) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:44.060031] [SM Entering]: (0x15e75f20) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:44.060057] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.060082] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:44.060107] PINT_encode [D 20:00:44.060132] encode_common [D 20:00:44.060210] lebf_encode_req [D 20:00:44.060240] PINT_do_request_commit: commit node 0x15e75ec8 [D 20:00:44.060265] node stored at 0 [D 20:00:44.060289] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:44.060314] node stored at 1 [D 20:00:44.060338] clearing tree [D 20:00:44.060366] PINT_encode_calc_max_size [D 20:00:44.060391] posting recv with status_user_tag=0 (max_size 24) [D 20:00:44.060420] BMI_post_recv: addr: 367403208, offset: 0x15e755d8, size: 24, tag: 9 [D 20:00:44.060449] openib_check_cq: completed send wr_id 15e72c28. [D 20:00:44.060476] openib_check_cq: completed recv wr_id 15e73590. [D 20:00:44.060505] openib_post_rr: da13:3346 bh 7 wr_id 15e73590. [D 20:00:44.060546] posting send with status_user_tag=1 [D 20:00:44.060574] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 9 [D 20:00:44.060601] element 0: offset: 0x15e83ab8, size: 272 [D 20:00:44.060630] post_sr: da13:3346 bh 8 len 288 wr_id 15e72c40. [D 20:00:44.060660] io_datafile_post_msgpairs: completion count is 2 [D 20:00:44.060685] [SM Exiting]: (0x15e75f20) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:44.060712] Posted PVFS_SYS_IO (25) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 3 total buffer count 3 pvfs2_aio_flush id 0 buf 3 offset 67108864, size 33554432 [D 20:00:44.817847] PVFS_isys_io entered [146] [D 20:00:44.817916] [SM Frame PUSH]: (0x15e85bb0) frame: 0x15e85c48 [D 20:00:44.817943] [SM Locating]: (0x15e85bb0) op-id: 6 [D 20:00:44.817969] client_op_state_get_machine 6 [D 20:00:44.817994] [SM Locating]: (0x15e85bb0) located: client_io_sm:init [D 20:00:44.818019] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.818047] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.818073] PINT_client_state_machine_post smcb 0x15e85bb0, op: PVFS_SYS_IO [D 20:00:44.818099] [SM Entering]: (0x15e85bb0) client_io_sm:init (status: 0) [D 20:00:44.818123] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.818149] (0x15e85c48) io state: io_init [D 20:00:44.818175] [SM Exiting]: (0x15e85bb0) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:44.818201] [SM push_state]: (0x15e85bb0) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:44.818225] [SM Entering]: (0x15e85bb0) client_getattr_sm:acache_lookup (status: 0) [D 20:00:44.818251] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.818277] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:44.818303] acache: get_cached_entry(): H=146 [D 20:00:44.818332] acache: copying out attr. [D 20:00:44.818357] acache: copying out size. [D 20:00:44.818383] acache: hit: H=146, size_status=0, attr_status=0 [D 20:00:44.818408] acache: acache hit [146] [D 20:00:44.818432] [SM Exiting]: (0x15e85bb0) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:44.818458] [SM Entering]: (0x15e85bb0) client_getattr_sm:cleanup (status: 0) [D 20:00:44.818489] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.818514] (0x15e85c48) getattr state: getattr_cleanup [D 20:00:44.818539] [SM Exiting]: (0x15e85bb0) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:44.818565] [SM pop_state]: (0x15e85bb0) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:44.818591] [SM Entering]: (0x15e85bb0) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:44.818618] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.818643] (0x15e85c48) io state: io_datafile_setup_msgpairs [D 20:00:44.818672] - io_find_target_datafiles called [D 20:00:44.818696] PINT_new_request_states n=1 [D 20:00:44.818721] PINT_new_request_states n=1 [D 20:00:44.818755] ========================================================= [D 20:00:44.818779] PINT_process_request [D 20:00:44.818804] tiling 4 copies [D 20:00:44.818829] skipping ahead to target_offset [D 20:00:44.818854] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:44.818880] lvl 0 el 0 blk 0 by 0 [D 20:00:44.818904] to 0 ta 67108864 fi 100663296 [D 20:00:44.818930] mto 0 mta 0 mfi 33554432 [D 20:00:44.818955] basic type or contiguous data [D 20:00:44.818980] Request disp [D 20:00:44.819004] contig_offset = 0 contig_size = 134217728 lvl_flag = 1 [D 20:00:44.819028] process logical skip [D 20:00:44.819052] retval = 67108864 [D 20:00:44.819077] exiting logical skip because distribute indicates done [D 20:00:44.819101] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:44.819128] lvl 0 el 0 blk 0 by 67108864 [D 20:00:44.819152] to 67108864 ta 67108864 fi 100663296 [D 20:00:44.819177] mto 0 mta 0 mfi 33554432 [D 20:00:44.819202] basic type or contiguous data [D 20:00:44.819226] Request disp [D 20:00:44.819250] contig_offset = 67108864 contig_size = 67108864 lvl_flag = 1 [D 20:00:44.819274] calling distribute [D 20:00:44.819299] PINT_distribute [D 20:00:44.819323] of 67108864 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:44.819352] begin iteration loff: 67108864 [D 20:00:44.819377] segment exceeds byte limit [D 20:00:44.819401] file being extended [D 20:00:44.819426] client lstof 0 diff 0 sgof 0 [D 20:00:44.819451] **********CALL***PROCESS********* [D 20:00:44.819475] segment of 0 sz 8177 [D 20:00:44.819499] PINT_process_request [D 20:00:44.819523] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:44.819550] lvl 0 el 0 blk 0 by 0 [D 20:00:44.819575] to 0 ta 0 fi 8177 [D 20:00:44.819600] basic type or contiguous data [D 20:00:44.819629] Request disp [D 20:00:44.819657] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:44.819685] process a segment [D 20:00:44.819713] of 0 sz 8177 [D 20:00:44.819738] add a segment 0 [D 20:00:44.819762] retval = 8177 [D 20:00:44.819786] exiting distribute returned less than expected [D 20:00:44.819810] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:44.819836] *****RETURN***FROM***PROCESS***** [D 20:00:44.819861] end iteration [D 20:00:44.819885] done with segments or bytes [D 20:00:44.819909] of 67117041 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:44.819934] next loff: 67117041 next poff: 67117041 [D 20:00:44.819959] return value 8177 [D 20:00:44.819984] finished [D 20:00:44.820007] retval = 8177 [D 20:00:44.820032] exiting distribute returned less than expected [D 20:00:44.820055] done sg 1 sm 1 by 8177 bm 8177 ta 67108864 to 67117041 fo 100663296 eof 0 [D 20:00:44.820081] ========================================================= [D 20:00:44.820106] PINT_encode_calc_max_size [D 20:00:44.820132] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:44.820158] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:44.820187] initializing context[0] 0x15e86408 [D 20:00:44.820213] filling I/O request for 2863311531 [D 20:00:44.820239] [SM Exiting]: (0x15e85bb0) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:44.820267] [SM Entering]: (0x15e85bb0) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:44.820294] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:44.820319] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:44.820344] PINT_encode [D 20:00:44.820368] encode_common [D 20:00:44.820444] lebf_encode_req [D 20:00:44.820472] PINT_do_request_commit: commit node 0x15e75838 [D 20:00:44.820497] node stored at 0 [D 20:00:44.820522] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:44.820546] node stored at 1 [D 20:00:44.820570] clearing tree [D 20:00:44.820597] PINT_encode_calc_max_size [D 20:00:44.820623] posting recv with status_user_tag=0 (max_size 24) [D 20:00:44.820651] BMI_post_recv: addr: 367403208, offset: 0x15e88c68, size: 24, tag: 10 [D 20:00:44.820679] openib_check_cq: completed send wr_id 15e72c40. [D 20:00:44.820705] openib_check_cq: completed recv wr_id 15e735a8. [D 20:00:44.820734] openib_post_rr: da13:3346 bh 8 wr_id 15e735a8. [D 20:00:44.820775] posting send with status_user_tag=1 [D 20:00:44.820802] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 10 [D 20:00:44.820828] element 0: offset: 0x15e86b18, size: 272 [D 20:00:44.820858] post_sr: da13:3346 bh 9 len 288 wr_id 15e72c58. [D 20:00:44.820887] io_datafile_post_msgpairs: completion count is 2 [D 20:00:44.820912] [SM Exiting]: (0x15e85bb0) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:44.820939] Posted PVFS_SYS_IO (28) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 4 total buffer count 4 pvfs2_aio_flush id 0 buf 4 offset 100663296, size 33554432 [D 20:00:45.590295] PVFS_isys_io entered [146] [D 20:00:45.590362] [SM Frame PUSH]: (0x15e88fa0) frame: 0x15e89038 [D 20:00:45.590390] [SM Locating]: (0x15e88fa0) op-id: 6 [D 20:00:45.590415] client_op_state_get_machine 6 [D 20:00:45.590439] [SM Locating]: (0x15e88fa0) located: client_io_sm:init [D 20:00:45.590465] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:45.590493] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:45.590519] PINT_client_state_machine_post smcb 0x15e88fa0, op: PVFS_SYS_IO [D 20:00:45.590545] [SM Entering]: (0x15e88fa0) client_io_sm:init (status: 0) [D 20:00:45.590571] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:45.590597] (0x15e89038) io state: io_init [D 20:00:45.590621] [SM Exiting]: (0x15e88fa0) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:45.590648] [SM push_state]: (0x15e88fa0) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:45.590674] [SM Entering]: (0x15e88fa0) client_getattr_sm:acache_lookup (status: 0) [D 20:00:45.590700] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:45.590724] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:45.590750] acache: get_cached_entry(): H=146 [D 20:00:45.590780] acache: copying out attr. [D 20:00:45.590805] acache: copying out size. [D 20:00:45.590829] acache: hit: H=146, size_status=0, attr_status=0 [D 20:00:45.590855] acache: acache hit [146] [D 20:00:45.590879] [SM Exiting]: (0x15e88fa0) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:45.590906] [SM Entering]: (0x15e88fa0) client_getattr_sm:cleanup (status: 0) [D 20:00:45.590931] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:45.590956] (0x15e89038) getattr state: getattr_cleanup [D 20:00:45.590981] [SM Exiting]: (0x15e88fa0) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:45.591007] [SM pop_state]: (0x15e88fa0) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:45.591032] [SM Entering]: (0x15e88fa0) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:45.591057] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:45.591082] (0x15e89038) io state: io_datafile_setup_msgpairs [D 20:00:45.591111] - io_find_target_datafiles called [D 20:00:45.591135] PINT_new_request_states n=1 [D 20:00:45.591160] PINT_new_request_states n=1 [D 20:00:45.591192] ========================================================= [D 20:00:45.591222] PINT_process_request [D 20:00:45.591247] tiling 5 copies [D 20:00:45.591271] skipping ahead to target_offset [D 20:00:45.591295] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:45.591321] lvl 0 el 0 blk 0 by 0 [D 20:00:45.591346] to 0 ta 100663296 fi 134217728 [D 20:00:45.591371] mto 0 mta 0 mfi 33554432 [D 20:00:45.591396] basic type or contiguous data [D 20:00:45.591421] Request disp [D 20:00:45.591444] contig_offset = 0 contig_size = 167772160 lvl_flag = 1 [D 20:00:45.591469] process logical skip [D 20:00:45.591493] retval = 100663296 [D 20:00:45.591517] exiting logical skip because distribute indicates done [D 20:00:45.591542] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:45.591567] lvl 0 el 0 blk 0 by 100663296 [D 20:00:45.591592] to 100663296 ta 100663296 fi 134217728 [D 20:00:45.591626] mto 0 mta 0 mfi 33554432 [D 20:00:45.591662] basic type or contiguous data [D 20:00:45.591695] Request disp [D 20:00:45.591726] contig_offset = 100663296 contig_size = 67108864 lvl_flag = 1 [D 20:00:45.591754] calling distribute [D 20:00:45.591783] PINT_distribute [D 20:00:45.591812] of 100663296 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:45.591839] begin iteration loff: 100663296 [D 20:00:45.591866] segment exceeds byte limit [D 20:00:45.591890] file being extended [D 20:00:45.591914] client lstof 0 diff 0 sgof 0 [D 20:00:45.591938] **********CALL***PROCESS********* [D 20:00:45.591962] segment of 0 sz 8177 [D 20:00:45.591987] PINT_process_request [D 20:00:45.592011] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:45.592038] lvl 0 el 0 blk 0 by 0 [D 20:00:45.592062] to 0 ta 0 fi 8177 [D 20:00:45.592087] basic type or contiguous data [D 20:00:45.592111] Request disp [D 20:00:45.592135] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:45.592160] process a segment [D 20:00:45.592184] of 0 sz 8177 [D 20:00:45.592207] add a segment 0 [D 20:00:45.592232] retval = 8177 [D 20:00:45.592256] exiting distribute returned less than expected [D 20:00:45.592280] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:45.592307] *****RETURN***FROM***PROCESS***** [D 20:00:45.592331] end iteration [D 20:00:45.592355] done with segments or bytes [D 20:00:45.592379] of 100671473 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:45.592405] next loff: 100671473 next poff: 100671473 [D 20:00:45.592430] return value 8177 [D 20:00:45.592453] finished [D 20:00:45.592477] retval = 8177 [D 20:00:45.592502] exiting distribute returned less than expected [D 20:00:45.592526] done sg 1 sm 1 by 8177 bm 8177 ta 100663296 to 100671473 fo 134217728 eof 0 [D 20:00:45.592552] ========================================================= [D 20:00:45.592576] PINT_encode_calc_max_size [D 20:00:45.592602] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:45.592627] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:45.592657] initializing context[0] 0x15e897e8 [D 20:00:45.592682] filling I/O request for 2863311531 [D 20:00:45.592707] [SM Exiting]: (0x15e88fa0) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:45.592734] [SM Entering]: (0x15e88fa0) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:45.592759] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:45.592784] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:45.592809] PINT_encode [D 20:00:45.592834] encode_common [D 20:00:45.592919] lebf_encode_req [D 20:00:45.592947] PINT_do_request_commit: commit node 0x15e88f48 [D 20:00:45.592972] node stored at 0 [D 20:00:45.592997] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:45.593021] node stored at 1 [D 20:00:45.593046] clearing tree [D 20:00:45.593073] PINT_encode_calc_max_size [D 20:00:45.593099] posting recv with status_user_tag=0 (max_size 24) [D 20:00:45.593128] BMI_post_recv: addr: 367403208, offset: 0x15e8c048, size: 24, tag: 11 [D 20:00:45.593160] openib_check_cq: completed send wr_id 15e72c58. [D 20:00:45.593187] openib_check_cq: completed recv wr_id 15e735c0. [D 20:00:45.593215] openib_post_rr: da13:3346 bh 9 wr_id 15e735c0. [D 20:00:45.593255] posting send with status_user_tag=1 [D 20:00:45.593283] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 11 [D 20:00:45.593309] element 0: offset: 0x15e89ef8, size: 272 [D 20:00:45.593338] post_sr: da13:3346 bh 10 len 288 wr_id 15e72c70. [D 20:00:45.593368] io_datafile_post_msgpairs: completion count is 2 [D 20:00:45.593394] [SM Exiting]: (0x15e88fa0) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:45.593420] Posted PVFS_SYS_IO (31) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 5 total buffer count 5 pvfs2_aio_flush id 0 buf 5 offset 134217728, size 33554432 [D 20:00:46.383642] PVFS_isys_io entered [146] [D 20:00:46.383706] [SM Frame PUSH]: (0x15e8c340) frame: 0x15e8c3d8 [D 20:00:46.383734] [SM Locating]: (0x15e8c340) op-id: 6 [D 20:00:46.383759] client_op_state_get_machine 6 [D 20:00:46.383784] [SM Locating]: (0x15e8c340) located: client_io_sm:init [D 20:00:46.383810] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:46.383837] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:46.383863] PINT_client_state_machine_post smcb 0x15e8c340, op: PVFS_SYS_IO [D 20:00:46.383889] [SM Entering]: (0x15e8c340) client_io_sm:init (status: 0) [D 20:00:46.383914] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:46.383939] (0x15e8c3d8) io state: io_init [D 20:00:46.383965] [SM Exiting]: (0x15e8c340) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:46.383992] [SM push_state]: (0x15e8c340) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:46.384016] [SM Entering]: (0x15e8c340) client_getattr_sm:acache_lookup (status: 0) [D 20:00:46.384042] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:46.384067] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:46.384094] acache: get_cached_entry(): H=146 [D 20:00:46.384122] acache: copying out attr. [D 20:00:46.384147] acache: copying out size. [D 20:00:46.384173] acache: hit: H=146, size_status=0, attr_status=0 [D 20:00:46.384198] acache: acache hit [146] [D 20:00:46.384222] [SM Exiting]: (0x15e8c340) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:46.384249] [SM Entering]: (0x15e8c340) client_getattr_sm:cleanup (status: 0) [D 20:00:46.384275] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:46.384300] (0x15e8c3d8) getattr state: getattr_cleanup [D 20:00:46.384325] [SM Exiting]: (0x15e8c340) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:46.384350] [SM pop_state]: (0x15e8c340) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:46.384375] [SM Entering]: (0x15e8c340) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:46.384401] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:46.384426] (0x15e8c3d8) io state: io_datafile_setup_msgpairs [D 20:00:46.384454] - io_find_target_datafiles called [D 20:00:46.384479] PINT_new_request_states n=1 [D 20:00:46.384505] PINT_new_request_states n=1 [D 20:00:46.384536] ========================================================= [D 20:00:46.384562] PINT_process_request [D 20:00:46.384587] tiling 6 copies [D 20:00:46.384612] skipping ahead to target_offset [D 20:00:46.384635] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:46.384662] lvl 0 el 0 blk 0 by 0 [D 20:00:46.384687] to 0 ta 134217728 fi 167772160 [D 20:00:46.384712] mto 0 mta 0 mfi 33554432 [D 20:00:46.384737] basic type or contiguous data [D 20:00:46.384761] Request disp [D 20:00:46.384786] contig_offset = 0 contig_size = 201326592 lvl_flag = 1 [D 20:00:46.384811] process logical skip [D 20:00:46.384836] retval = 134217728 [D 20:00:46.384860] exiting logical skip because distribute indicates done [D 20:00:46.384883] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:46.384914] lvl 0 el 0 blk 0 by 134217728 [D 20:00:46.384940] to 134217728 ta 134217728 fi 167772160 [D 20:00:46.384964] mto 0 mta 0 mfi 33554432 [D 20:00:46.384989] basic type or contiguous data [D 20:00:46.385013] Request disp [D 20:00:46.385037] contig_offset = 134217728 contig_size = 67108864 lvl_flag = 1 [D 20:00:46.385062] calling distribute [D 20:00:46.385087] PINT_distribute [D 20:00:46.385110] of 134217728 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:46.385138] begin iteration loff: 134217728 [D 20:00:46.385164] segment exceeds byte limit [D 20:00:46.385188] file being extended [D 20:00:46.385211] client lstof 0 diff 0 sgof 0 [D 20:00:46.385236] **********CALL***PROCESS********* [D 20:00:46.385260] segment of 0 sz 8177 [D 20:00:46.385285] PINT_process_request [D 20:00:46.385309] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:46.385335] lvl 0 el 0 blk 0 by 0 [D 20:00:46.385360] to 0 ta 0 fi 8177 [D 20:00:46.385384] basic type or contiguous data [D 20:00:46.385409] Request disp [D 20:00:46.385433] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:46.385457] process a segment [D 20:00:46.385481] of 0 sz 8177 [D 20:00:46.385506] add a segment 0 [D 20:00:46.385530] retval = 8177 [D 20:00:46.385554] exiting distribute returned less than expected [D 20:00:46.385578] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:46.385603] *****RETURN***FROM***PROCESS***** [D 20:00:46.385631] end iteration [D 20:00:46.385655] done with segments or bytes [D 20:00:46.385679] of 134225905 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:46.385704] next loff: 134225905 next poff: 134225905 [D 20:00:46.385730] return value 8177 [D 20:00:46.385755] finished [D 20:00:46.385779] retval = 8177 [D 20:00:46.385803] exiting distribute returned less than expected [D 20:00:46.385827] done sg 1 sm 1 by 8177 bm 8177 ta 134217728 to 134225905 fo 167772160 eof 0 [D 20:00:46.385853] ========================================================= [D 20:00:46.385878] PINT_encode_calc_max_size [D 20:00:46.385904] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:46.385929] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:46.385962] initializing context[0] 0x15e8cbe0 [D 20:00:46.385989] filling I/O request for 2863311531 [D 20:00:46.386013] [SM Exiting]: (0x15e8c340) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:46.386039] [SM Entering]: (0x15e8c340) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:46.386065] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:46.386091] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:46.386116] PINT_encode [D 20:00:46.386140] encode_common [D 20:00:46.386216] lebf_encode_req [D 20:00:46.386244] PINT_do_request_commit: commit node 0x15e8c2e8 [D 20:00:46.386269] node stored at 0 [D 20:00:46.386294] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:46.386319] node stored at 1 [D 20:00:46.386343] clearing tree [D 20:00:46.386370] PINT_encode_calc_max_size [D 20:00:46.386396] posting recv with status_user_tag=0 (max_size 24) [D 20:00:46.386425] BMI_post_recv: addr: 367403208, offset: 0x15e8f440, size: 24, tag: 12 [D 20:00:46.386453] openib_check_cq: completed send wr_id 15e72c70. [D 20:00:46.386480] openib_check_cq: completed recv wr_id 15e735d8. [D 20:00:46.386508] openib_post_rr: da13:3346 bh 10 wr_id 15e735d8. [D 20:00:46.386549] posting send with status_user_tag=1 [D 20:00:46.386577] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 12 [D 20:00:46.386603] element 0: offset: 0x15e8d2f0, size: 272 [D 20:00:46.386633] post_sr: da13:3346 bh 11 len 288 wr_id 15e72c88. [D 20:00:46.386663] io_datafile_post_msgpairs: completion count is 2 [D 20:00:46.386688] [SM Exiting]: (0x15e8c340) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:46.386715] Posted PVFS_SYS_IO (34) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 6 total buffer count 6 pvfs2_aio_flush id 0 buf 6 offset 167772160, size 33554432 [D 20:00:47.172109] PVFS_isys_io entered [146] [D 20:00:47.172170] [SM Frame PUSH]: (0x15e8f790) frame: 0x15e8f828 [D 20:00:47.172197] [SM Locating]: (0x15e8f790) op-id: 6 [D 20:00:47.172223] client_op_state_get_machine 6 [D 20:00:47.172247] [SM Locating]: (0x15e8f790) located: client_io_sm:init [D 20:00:47.172273] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.172301] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.172327] PINT_client_state_machine_post smcb 0x15e8f790, op: PVFS_SYS_IO [D 20:00:47.172353] [SM Entering]: (0x15e8f790) client_io_sm:init (status: 0) [D 20:00:47.172378] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.172404] (0x15e8f828) io state: io_init [D 20:00:47.172429] [SM Exiting]: (0x15e8f790) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:47.172455] [SM push_state]: (0x15e8f790) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:47.172480] [SM Entering]: (0x15e8f790) client_getattr_sm:acache_lookup (status: 0) [D 20:00:47.172506] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.172532] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:47.172557] acache: get_cached_entry(): H=146 [D 20:00:47.172587] acache: miss non-static: H=146 [D 20:00:47.172614] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:47.172638] acache: acache hit [146] [D 20:00:47.172663] [SM Exiting]: (0x15e8f790) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:47.172689] [SM Entering]: (0x15e8f790) client_getattr_sm:cleanup (status: 0) [D 20:00:47.172715] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.172739] (0x15e8f828) getattr state: getattr_cleanup [D 20:00:47.172765] [SM Exiting]: (0x15e8f790) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:47.172790] [SM pop_state]: (0x15e8f790) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:47.172816] [SM Entering]: (0x15e8f790) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:47.172842] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.172866] (0x15e8f828) io state: io_datafile_setup_msgpairs [D 20:00:47.172894] - io_find_target_datafiles called [D 20:00:47.172919] PINT_new_request_states n=1 [D 20:00:47.172949] PINT_new_request_states n=1 [D 20:00:47.172983] ========================================================= [D 20:00:47.173007] PINT_process_request [D 20:00:47.173032] tiling 7 copies [D 20:00:47.173056] skipping ahead to target_offset [D 20:00:47.173081] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:47.173108] lvl 0 el 0 blk 0 by 0 [D 20:00:47.173132] to 0 ta 167772160 fi 201326592 [D 20:00:47.173157] mto 0 mta 0 mfi 33554432 [D 20:00:47.173182] basic type or contiguous data [D 20:00:47.173207] Request disp [D 20:00:47.173231] contig_offset = 0 contig_size = 234881024 lvl_flag = 1 [D 20:00:47.173255] process logical skip [D 20:00:47.173280] retval = 167772160 [D 20:00:47.173304] exiting logical skip because distribute indicates done [D 20:00:47.173329] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:47.173355] lvl 0 el 0 blk 0 by 167772160 [D 20:00:47.173380] to 167772160 ta 167772160 fi 201326592 [D 20:00:47.173405] mto 0 mta 0 mfi 33554432 [D 20:00:47.173430] basic type or contiguous data [D 20:00:47.173454] Request disp [D 20:00:47.173478] contig_offset = 167772160 contig_size = 67108864 lvl_flag = 1 [D 20:00:47.173503] calling distribute [D 20:00:47.173528] PINT_distribute [D 20:00:47.173552] of 167772160 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:47.173579] begin iteration loff: 167772160 [D 20:00:47.173604] segment exceeds byte limit [D 20:00:47.173631] file being extended [D 20:00:47.173655] client lstof 0 diff 0 sgof 0 [D 20:00:47.173680] **********CALL***PROCESS********* [D 20:00:47.173705] segment of 0 sz 8177 [D 20:00:47.173733] PINT_process_request [D 20:00:47.173758] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:47.173784] lvl 0 el 0 blk 0 by 0 [D 20:00:47.173809] to 0 ta 0 fi 8177 [D 20:00:47.173833] basic type or contiguous data [D 20:00:47.173857] Request disp [D 20:00:47.173882] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:47.173907] process a segment [D 20:00:47.173931] of 0 sz 8177 [D 20:00:47.173955] add a segment 0 [D 20:00:47.173979] retval = 8177 [D 20:00:47.174003] exiting distribute returned less than expected [D 20:00:47.174028] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:47.174054] *****RETURN***FROM***PROCESS***** [D 20:00:47.174078] end iteration [D 20:00:47.174102] done with segments or bytes [D 20:00:47.174127] of 167780337 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:47.174153] next loff: 167780337 next poff: 167780337 [D 20:00:47.174178] return value 8177 [D 20:00:47.174202] finished [D 20:00:47.174226] retval = 8177 [D 20:00:47.174250] exiting distribute returned less than expected [D 20:00:47.174274] done sg 1 sm 1 by 8177 bm 8177 ta 167772160 to 167780337 fo 201326592 eof 0 [D 20:00:47.174301] ========================================================= [D 20:00:47.174325] PINT_encode_calc_max_size [D 20:00:47.174350] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:47.174376] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:47.174405] initializing context[0] 0x15e8ffd8 [D 20:00:47.174430] filling I/O request for 2863311531 [D 20:00:47.174455] [SM Exiting]: (0x15e8f790) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:47.174482] [SM Entering]: (0x15e8f790) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:47.174508] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.174532] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:47.174557] PINT_encode [D 20:00:47.174582] encode_common [D 20:00:47.174659] lebf_encode_req [D 20:00:47.174688] PINT_do_request_commit: commit node 0x15e8f738 [D 20:00:47.174713] node stored at 0 [D 20:00:47.174737] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:47.174761] node stored at 1 [D 20:00:47.174786] clearing tree [D 20:00:47.174813] PINT_encode_calc_max_size [D 20:00:47.174838] posting recv with status_user_tag=0 (max_size 24) [D 20:00:47.174866] BMI_post_recv: addr: 367403208, offset: 0x15e92838, size: 24, tag: 13 [D 20:00:47.174896] openib_check_cq: completed send wr_id 15e72c88. [D 20:00:47.174922] openib_check_cq: completed recv wr_id 15e735f0. [D 20:00:47.174951] openib_post_rr: da13:3346 bh 11 wr_id 15e735f0. [D 20:00:47.174991] posting send with status_user_tag=1 [D 20:00:47.175019] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 13 [D 20:00:47.175045] element 0: offset: 0x15e906e8, size: 272 [D 20:00:47.175074] post_sr: da13:3346 bh 12 len 288 wr_id 15e72ca0. [D 20:00:47.175104] io_datafile_post_msgpairs: completion count is 2 [D 20:00:47.175128] [SM Exiting]: (0x15e8f790) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:47.175156] Posted PVFS_SYS_IO (37) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 7 total buffer count 7 pvfs2_aio_flush id 0 buf 7 offset 201326592, size 33554432 [D 20:00:47.865912] PVFS_isys_io entered [146] [D 20:00:47.865987] [SM Frame PUSH]: (0x15e92b88) frame: 0x15e92c20 [D 20:00:47.866015] [SM Locating]: (0x15e92b88) op-id: 6 [D 20:00:47.866040] client_op_state_get_machine 6 [D 20:00:47.866065] [SM Locating]: (0x15e92b88) located: client_io_sm:init [D 20:00:47.866090] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.866118] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.866145] PINT_client_state_machine_post smcb 0x15e92b88, op: PVFS_SYS_IO [D 20:00:47.866170] [SM Entering]: (0x15e92b88) client_io_sm:init (status: 0) [D 20:00:47.866196] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.866228] (0x15e92c20) io state: io_init [D 20:00:47.866253] [SM Exiting]: (0x15e92b88) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:47.866280] [SM push_state]: (0x15e92b88) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:47.866306] [SM Entering]: (0x15e92b88) client_getattr_sm:acache_lookup (status: 0) [D 20:00:47.866332] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.866356] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:47.866383] acache: get_cached_entry(): H=146 [D 20:00:47.866412] acache: miss non-static: H=146 [D 20:00:47.866438] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:47.866463] acache: acache hit [146] [D 20:00:47.866488] [SM Exiting]: (0x15e92b88) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:47.866515] [SM Entering]: (0x15e92b88) client_getattr_sm:cleanup (status: 0) [D 20:00:47.866540] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.866565] (0x15e92c20) getattr state: getattr_cleanup [D 20:00:47.866590] [SM Exiting]: (0x15e92b88) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:47.866616] [SM pop_state]: (0x15e92b88) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:47.866641] [SM Entering]: (0x15e92b88) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:47.866667] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.866692] (0x15e92c20) io state: io_datafile_setup_msgpairs [D 20:00:47.866721] - io_find_target_datafiles called [D 20:00:47.866745] PINT_new_request_states n=1 [D 20:00:47.866771] PINT_new_request_states n=1 [D 20:00:47.866804] ========================================================= [D 20:00:47.866828] PINT_process_request [D 20:00:47.866854] tiling 8 copies [D 20:00:47.866879] skipping ahead to target_offset [D 20:00:47.866903] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:47.866930] lvl 0 el 0 blk 0 by 0 [D 20:00:47.866954] to 0 ta 201326592 fi 234881024 [D 20:00:47.866979] mto 0 mta 0 mfi 33554432 [D 20:00:47.867005] basic type or contiguous data [D 20:00:47.867030] Request disp [D 20:00:47.867054] contig_offset = 0 contig_size = 268435456 lvl_flag = 1 [D 20:00:47.867079] process logical skip [D 20:00:47.867103] retval = 201326592 [D 20:00:47.867128] exiting logical skip because distribute indicates done [D 20:00:47.867152] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:47.867178] lvl 0 el 0 blk 0 by 201326592 [D 20:00:47.867203] to 201326592 ta 201326592 fi 234881024 [D 20:00:47.867228] mto 0 mta 0 mfi 33554432 [D 20:00:47.867253] basic type or contiguous data [D 20:00:47.867278] Request disp [D 20:00:47.867301] contig_offset = 201326592 contig_size = 67108864 lvl_flag = 1 [D 20:00:47.867327] calling distribute [D 20:00:47.867351] PINT_distribute [D 20:00:47.867375] of 201326592 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:47.867403] begin iteration loff: 201326592 [D 20:00:47.867429] segment exceeds byte limit [D 20:00:47.867453] file being extended [D 20:00:47.867478] client lstof 0 diff 0 sgof 0 [D 20:00:47.867503] **********CALL***PROCESS********* [D 20:00:47.867527] segment of 0 sz 8177 [D 20:00:47.867551] PINT_process_request [D 20:00:47.867576] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:47.867602] lvl 0 el 0 blk 0 by 0 [D 20:00:47.867630] to 0 ta 0 fi 8177 [D 20:00:47.867659] basic type or contiguous data [D 20:00:47.867687] Request disp [D 20:00:47.867714] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:47.867742] process a segment [D 20:00:47.867766] of 0 sz 8177 [D 20:00:47.867791] add a segment 0 [D 20:00:47.867815] retval = 8177 [D 20:00:47.867839] exiting distribute returned less than expected [D 20:00:47.867863] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:47.867890] *****RETURN***FROM***PROCESS***** [D 20:00:47.867915] end iteration [D 20:00:47.867939] done with segments or bytes [D 20:00:47.867968] of 201334769 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:47.867995] next loff: 201334769 next poff: 201334769 [D 20:00:47.868021] return value 8177 [D 20:00:47.868045] finished [D 20:00:47.868069] retval = 8177 [D 20:00:47.868093] exiting distribute returned less than expected [D 20:00:47.868118] done sg 1 sm 1 by 8177 bm 8177 ta 201326592 to 201334769 fo 234881024 eof 0 [D 20:00:47.868144] ========================================================= [D 20:00:47.868169] PINT_encode_calc_max_size [D 20:00:47.868194] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:47.868221] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:47.868249] initializing context[0] 0x15e933d0 [D 20:00:47.868276] filling I/O request for 2863311531 [D 20:00:47.868302] [SM Exiting]: (0x15e92b88) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:47.868328] [SM Entering]: (0x15e92b88) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:47.868355] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:47.868380] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:47.868406] PINT_encode [D 20:00:47.868431] encode_common [D 20:00:47.868505] lebf_encode_req [D 20:00:47.868534] PINT_do_request_commit: commit node 0x15e92b30 [D 20:00:47.868558] node stored at 0 [D 20:00:47.868583] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:47.868607] node stored at 1 [D 20:00:47.868632] clearing tree [D 20:00:47.868659] PINT_encode_calc_max_size [D 20:00:47.868684] posting recv with status_user_tag=0 (max_size 24) [D 20:00:47.868713] BMI_post_recv: addr: 367403208, offset: 0x15e95c30, size: 24, tag: 14 [D 20:00:47.868741] openib_check_cq: completed send wr_id 15e72ca0. [D 20:00:47.868768] openib_check_cq: completed recv wr_id 15e73608. [D 20:00:47.868797] openib_post_rr: da13:3346 bh 12 wr_id 15e73608. [D 20:00:47.868836] posting send with status_user_tag=1 [D 20:00:47.868864] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 14 [D 20:00:47.868889] element 0: offset: 0x15e93ae0, size: 272 [D 20:00:47.868919] post_sr: da13:3346 bh 13 len 288 wr_id 15e72cb8. [D 20:00:47.868949] io_datafile_post_msgpairs: completion count is 2 [D 20:00:47.868974] [SM Exiting]: (0x15e92b88) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:47.869001] Posted PVFS_SYS_IO (40) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 8 total buffer count 8 pvfs2_aio_flush id 0 buf 8 offset 234881024, size 33554432 [D 20:00:48.682240] PVFS_isys_io entered [146] [D 20:00:48.682310] [SM Frame PUSH]: (0x15e95f68) frame: 0x15e96000 [D 20:00:48.682336] [SM Locating]: (0x15e95f68) op-id: 6 [D 20:00:48.682362] client_op_state_get_machine 6 [D 20:00:48.682386] [SM Locating]: (0x15e95f68) located: client_io_sm:init [D 20:00:48.682412] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:48.682439] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:48.682465] PINT_client_state_machine_post smcb 0x15e95f68, op: PVFS_SYS_IO [D 20:00:48.682492] [SM Entering]: (0x15e95f68) client_io_sm:init (status: 0) [D 20:00:48.682518] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:48.682543] (0x15e96000) io state: io_init [D 20:00:48.682568] [SM Exiting]: (0x15e95f68) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:48.682595] [SM push_state]: (0x15e95f68) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:48.682620] [SM Entering]: (0x15e95f68) client_getattr_sm:acache_lookup (status: 0) [D 20:00:48.682645] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:48.682670] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:48.682696] acache: get_cached_entry(): H=146 [D 20:00:48.682724] acache: miss non-static: H=146 [D 20:00:48.682751] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:48.682776] acache: acache hit [146] [D 20:00:48.682800] [SM Exiting]: (0x15e95f68) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:48.682831] [SM Entering]: (0x15e95f68) client_getattr_sm:cleanup (status: 0) [D 20:00:48.682857] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:48.682883] (0x15e96000) getattr state: getattr_cleanup [D 20:00:48.682908] [SM Exiting]: (0x15e95f68) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:48.682932] [SM pop_state]: (0x15e95f68) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:48.682958] [SM Entering]: (0x15e95f68) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:48.682984] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:48.683009] (0x15e96000) io state: io_datafile_setup_msgpairs [D 20:00:48.683036] - io_find_target_datafiles called [D 20:00:48.683061] PINT_new_request_states n=1 [D 20:00:48.683087] PINT_new_request_states n=1 [D 20:00:48.683118] ========================================================= [D 20:00:48.683143] PINT_process_request [D 20:00:48.683168] tiling 9 copies [D 20:00:48.683193] skipping ahead to target_offset [D 20:00:48.683217] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:48.683243] lvl 0 el 0 blk 0 by 0 [D 20:00:48.683268] to 0 ta 234881024 fi 268435456 [D 20:00:48.683293] mto 0 mta 0 mfi 33554432 [D 20:00:48.683318] basic type or contiguous data [D 20:00:48.683342] Request disp [D 20:00:48.683366] contig_offset = 0 contig_size = 301989888 lvl_flag = 1 [D 20:00:48.683391] process logical skip [D 20:00:48.683415] retval = 234881024 [D 20:00:48.683440] exiting logical skip because distribute indicates done [D 20:00:48.683464] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:48.683490] lvl 0 el 0 blk 0 by 234881024 [D 20:00:48.683515] to 234881024 ta 234881024 fi 268435456 [D 20:00:48.683540] mto 0 mta 0 mfi 33554432 [D 20:00:48.683565] basic type or contiguous data [D 20:00:48.683588] Request disp [D 20:00:48.683612] contig_offset = 234881024 contig_size = 67108864 lvl_flag = 1 [D 20:00:48.683642] calling distribute [D 20:00:48.683670] PINT_distribute [D 20:00:48.683697] of 234881024 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:48.683730] begin iteration loff: 234881024 [D 20:00:48.683756] segment exceeds byte limit [D 20:00:48.683780] file being extended [D 20:00:48.683805] client lstof 0 diff 0 sgof 0 [D 20:00:48.683830] **********CALL***PROCESS********* [D 20:00:48.683854] segment of 0 sz 8177 [D 20:00:48.683877] PINT_process_request [D 20:00:48.683902] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:48.683929] lvl 0 el 0 blk 0 by 0 [D 20:00:48.683954] to 0 ta 0 fi 8177 [D 20:00:48.683978] basic type or contiguous data [D 20:00:48.684001] Request disp [D 20:00:48.684025] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:48.684050] process a segment [D 20:00:48.684074] of 0 sz 8177 [D 20:00:48.684099] add a segment 0 [D 20:00:48.684123] retval = 8177 [D 20:00:48.684146] exiting distribute returned less than expected [D 20:00:48.684170] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:48.684196] *****RETURN***FROM***PROCESS***** [D 20:00:48.684221] end iteration [D 20:00:48.684245] done with segments or bytes [D 20:00:48.684268] of 234889201 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:48.684294] next loff: 234889201 next poff: 234889201 [D 20:00:48.684319] return value 8177 [D 20:00:48.684344] finished [D 20:00:48.684368] retval = 8177 [D 20:00:48.684391] exiting distribute returned less than expected [D 20:00:48.684415] done sg 1 sm 1 by 8177 bm 8177 ta 234881024 to 234889201 fo 268435456 eof 0 [D 20:00:48.684441] ========================================================= [D 20:00:48.684466] PINT_encode_calc_max_size [D 20:00:48.684492] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:48.684517] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:48.684545] initializing context[0] 0x15e967b0 [D 20:00:48.684571] filling I/O request for 2863311531 [D 20:00:48.684596] [SM Exiting]: (0x15e95f68) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:48.684624] [SM Entering]: (0x15e95f68) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:48.684649] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:48.684675] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:48.684700] PINT_encode [D 20:00:48.684723] encode_common [D 20:00:48.684805] lebf_encode_req [D 20:00:48.684834] PINT_do_request_commit: commit node 0x15e95f10 [D 20:00:48.684859] node stored at 0 [D 20:00:48.684884] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:48.684907] node stored at 1 [D 20:00:48.684932] clearing tree [D 20:00:48.684959] PINT_encode_calc_max_size [D 20:00:48.684985] posting recv with status_user_tag=0 (max_size 24) [D 20:00:48.685013] BMI_post_recv: addr: 367403208, offset: 0x15e99010, size: 24, tag: 15 [D 20:00:48.685042] openib_check_cq: completed send wr_id 15e72cb8. [D 20:00:48.685069] openib_check_cq: completed recv wr_id 15e73620. [D 20:00:48.685097] openib_post_rr: da13:3346 bh 13 wr_id 15e73620. [D 20:00:48.685137] posting send with status_user_tag=1 [D 20:00:48.685165] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 15 [D 20:00:48.685191] element 0: offset: 0x15e96ec0, size: 272 [D 20:00:48.685221] post_sr: da13:3346 bh 14 len 288 wr_id 15e72cd0. [D 20:00:48.685251] io_datafile_post_msgpairs: completion count is 2 [D 20:00:48.685276] [SM Exiting]: (0x15e95f68) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:48.685303] Posted PVFS_SYS_IO (43) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 9 total buffer count 9 pvfs2_aio_flush id 0 buf 9 offset 268435456, size 33554432 [D 20:00:49.353669] PVFS_isys_io entered [146] [D 20:00:49.353732] [SM Frame PUSH]: (0x15e99308) frame: 0x15e993a0 [D 20:00:49.353759] [SM Locating]: (0x15e99308) op-id: 6 [D 20:00:49.353784] client_op_state_get_machine 6 [D 20:00:49.353809] [SM Locating]: (0x15e99308) located: client_io_sm:init [D 20:00:49.353835] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:49.353862] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:49.353889] PINT_client_state_machine_post smcb 0x15e99308, op: PVFS_SYS_IO [D 20:00:49.353915] [SM Entering]: (0x15e99308) client_io_sm:init (status: 0) [D 20:00:49.353940] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:49.353965] (0x15e993a0) io state: io_init [D 20:00:49.353991] [SM Exiting]: (0x15e99308) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:49.354017] [SM push_state]: (0x15e99308) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:49.354042] [SM Entering]: (0x15e99308) client_getattr_sm:acache_lookup (status: 0) [D 20:00:49.354068] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:49.354093] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:49.354119] acache: get_cached_entry(): H=146 [D 20:00:49.354148] acache: miss non-static: H=146 [D 20:00:49.354176] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:49.354201] acache: acache hit [146] [D 20:00:49.354226] [SM Exiting]: (0x15e99308) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:49.354251] [SM Entering]: (0x15e99308) client_getattr_sm:cleanup (status: 0) [D 20:00:49.354277] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:49.354303] (0x15e993a0) getattr state: getattr_cleanup [D 20:00:49.354328] [SM Exiting]: (0x15e99308) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:49.354352] [SM pop_state]: (0x15e99308) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:49.354378] [SM Entering]: (0x15e99308) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:49.354403] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:49.354429] (0x15e993a0) io state: io_datafile_setup_msgpairs [D 20:00:49.354456] - io_find_target_datafiles called [D 20:00:49.354481] PINT_new_request_states n=1 [D 20:00:49.354507] PINT_new_request_states n=1 [D 20:00:49.354544] ========================================================= [D 20:00:49.354570] PINT_process_request [D 20:00:49.354596] tiling 10 copies [D 20:00:49.354619] skipping ahead to target_offset [D 20:00:49.354644] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:49.354670] lvl 0 el 0 blk 0 by 0 [D 20:00:49.354696] to 0 ta 268435456 fi 301989888 [D 20:00:49.354721] mto 0 mta 0 mfi 33554432 [D 20:00:49.354745] basic type or contiguous data [D 20:00:49.354770] Request disp [D 20:00:49.354794] contig_offset = 0 contig_size = 335544320 lvl_flag = 1 [D 20:00:49.354819] process logical skip [D 20:00:49.354844] retval = 268435456 [D 20:00:49.354867] exiting logical skip because distribute indicates done [D 20:00:49.354891] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:49.354917] lvl 0 el 0 blk 0 by 268435456 [D 20:00:49.354943] to 268435456 ta 268435456 fi 301989888 [D 20:00:49.354968] mto 0 mta 0 mfi 33554432 [D 20:00:49.354992] basic type or contiguous data [D 20:00:49.355016] Request disp [D 20:00:49.355040] contig_offset = 268435456 contig_size = 67108864 lvl_flag = 1 [D 20:00:49.355065] calling distribute [D 20:00:49.355090] PINT_distribute [D 20:00:49.355113] of 268435456 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:49.355142] begin iteration loff: 268435456 [D 20:00:49.355168] segment exceeds byte limit [D 20:00:49.355192] file being extended [D 20:00:49.355216] client lstof 0 diff 0 sgof 0 [D 20:00:49.355241] **********CALL***PROCESS********* [D 20:00:49.355265] segment of 0 sz 8177 [D 20:00:49.355290] PINT_process_request [D 20:00:49.355314] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:49.355340] lvl 0 el 0 blk 0 by 0 [D 20:00:49.355365] to 0 ta 0 fi 8177 [D 20:00:49.355390] basic type or contiguous data [D 20:00:49.355414] Request disp [D 20:00:49.355438] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:49.355463] process a segment [D 20:00:49.355487] of 0 sz 8177 [D 20:00:49.355511] add a segment 0 [D 20:00:49.355535] retval = 8177 [D 20:00:49.355560] exiting distribute returned less than expected [D 20:00:49.355584] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:49.355609] *****RETURN***FROM***PROCESS***** [D 20:00:49.355638] end iteration [D 20:00:49.355666] done with segments or bytes [D 20:00:49.355693] of 268443633 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:49.355723] next loff: 268443633 next poff: 268443633 [D 20:00:49.355751] return value 8177 [D 20:00:49.355775] finished [D 20:00:49.355799] retval = 8177 [D 20:00:49.355824] exiting distribute returned less than expected [D 20:00:49.355848] done sg 1 sm 1 by 8177 bm 8177 ta 268435456 to 268443633 fo 301989888 eof 0 [D 20:00:49.355874] ========================================================= [D 20:00:49.355899] PINT_encode_calc_max_size [D 20:00:49.355924] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:49.355950] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:49.355983] initializing context[0] 0x15e99ba8 [D 20:00:49.356009] filling I/O request for 2863311531 [D 20:00:49.356035] [SM Exiting]: (0x15e99308) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:49.356060] [SM Entering]: (0x15e99308) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:49.356085] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:49.356111] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:49.356137] PINT_encode [D 20:00:49.356161] encode_common [D 20:00:49.356238] lebf_encode_req [D 20:00:49.356267] PINT_do_request_commit: commit node 0x15e992b0 [D 20:00:49.356292] node stored at 0 [D 20:00:49.356317] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:49.356341] node stored at 1 [D 20:00:49.356366] clearing tree [D 20:00:49.356393] PINT_encode_calc_max_size [D 20:00:49.356418] posting recv with status_user_tag=0 (max_size 24) [D 20:00:49.356450] BMI_post_recv: addr: 367403208, offset: 0x15e9c408, size: 24, tag: 16 [D 20:00:49.356478] openib_check_cq: completed send wr_id 15e72cd0. [D 20:00:49.356505] openib_check_cq: completed recv wr_id 15e73638. [D 20:00:49.356534] openib_post_rr: da13:3346 bh 14 wr_id 15e73638. [D 20:00:49.356574] posting send with status_user_tag=1 [D 20:00:49.356601] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 16 [D 20:00:49.356627] element 0: offset: 0x15e9a2b8, size: 272 [D 20:00:49.356657] post_sr: da13:3346 bh 15 len 288 wr_id 15e72ce8. [D 20:00:49.356686] io_datafile_post_msgpairs: completion count is 2 [D 20:00:49.356712] [SM Exiting]: (0x15e99308) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:49.356739] Posted PVFS_SYS_IO (46) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 10 total buffer count 10 pvfs2_aio_flush id 0 buf 10 offset 301989888, size 33554432 [D 20:00:50.357942] PVFS_isys_io entered [146] [D 20:00:50.358002] [SM Frame PUSH]: (0x15e9c758) frame: 0x15e9c7f0 [D 20:00:50.358029] [SM Locating]: (0x15e9c758) op-id: 6 [D 20:00:50.358054] client_op_state_get_machine 6 [D 20:00:50.358078] [SM Locating]: (0x15e9c758) located: client_io_sm:init [D 20:00:50.358104] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:50.358132] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:50.358160] PINT_client_state_machine_post smcb 0x15e9c758, op: PVFS_SYS_IO [D 20:00:50.358186] [SM Entering]: (0x15e9c758) client_io_sm:init (status: 0) [D 20:00:50.358211] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:50.358237] (0x15e9c7f0) io state: io_init [D 20:00:50.358262] [SM Exiting]: (0x15e9c758) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:50.358287] [SM push_state]: (0x15e9c758) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:50.358313] [SM Entering]: (0x15e9c758) client_getattr_sm:acache_lookup (status: 0) [D 20:00:50.358339] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:50.358365] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:50.358390] acache: get_cached_entry(): H=146 [D 20:00:50.358419] acache: miss non-static: H=146 [D 20:00:50.358447] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:50.358471] acache: acache hit [146] [D 20:00:50.358496] [SM Exiting]: (0x15e9c758) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:50.358523] [SM Entering]: (0x15e9c758) client_getattr_sm:cleanup (status: 0) [D 20:00:50.358549] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:50.358573] (0x15e9c7f0) getattr state: getattr_cleanup [D 20:00:50.358598] [SM Exiting]: (0x15e9c758) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:50.358624] [SM pop_state]: (0x15e9c758) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:50.358649] [SM Entering]: (0x15e9c758) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:50.358675] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:50.358700] (0x15e9c7f0) io state: io_datafile_setup_msgpairs [D 20:00:50.358728] - io_find_target_datafiles called [D 20:00:50.358753] PINT_new_request_states n=1 [D 20:00:50.358782] PINT_new_request_states n=1 [D 20:00:50.358816] ========================================================= [D 20:00:50.358840] PINT_process_request [D 20:00:50.358866] tiling 11 copies [D 20:00:50.358890] skipping ahead to target_offset [D 20:00:50.358915] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:50.358941] lvl 0 el 0 blk 0 by 0 [D 20:00:50.358966] to 0 ta 301989888 fi 335544320 [D 20:00:50.358991] mto 0 mta 0 mfi 33554432 [D 20:00:50.359016] basic type or contiguous data [D 20:00:50.359041] Request disp [D 20:00:50.359066] contig_offset = 0 contig_size = 369098752 lvl_flag = 1 [D 20:00:50.359090] process logical skip [D 20:00:50.359114] retval = 301989888 [D 20:00:50.359139] exiting logical skip because distribute indicates done [D 20:00:50.359163] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:50.359194] lvl 0 el 0 blk 0 by 301989888 [D 20:00:50.359219] to 301989888 ta 301989888 fi 335544320 [D 20:00:50.359244] mto 0 mta 0 mfi 33554432 [D 20:00:50.359269] basic type or contiguous data [D 20:00:50.359292] Request disp [D 20:00:50.359316] contig_offset = 301989888 contig_size = 67108864 lvl_flag = 1 [D 20:00:50.359341] calling distribute [D 20:00:50.359366] PINT_distribute [D 20:00:50.359390] of 301989888 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:50.359418] begin iteration loff: 301989888 [D 20:00:50.359444] segment exceeds byte limit [D 20:00:50.359468] file being extended [D 20:00:50.359492] client lstof 0 diff 0 sgof 0 [D 20:00:50.359517] **********CALL***PROCESS********* [D 20:00:50.359540] segment of 0 sz 8177 [D 20:00:50.359565] PINT_process_request [D 20:00:50.359589] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:50.359616] lvl 0 el 0 blk 0 by 0 [D 20:00:50.359644] to 0 ta 0 fi 8177 [D 20:00:50.359672] basic type or contiguous data [D 20:00:50.359700] Request disp [D 20:00:50.359727] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:50.359755] process a segment [D 20:00:50.359779] of 0 sz 8177 [D 20:00:50.359803] add a segment 0 [D 20:00:50.359827] retval = 8177 [D 20:00:50.359851] exiting distribute returned less than expected [D 20:00:50.359875] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:50.359902] *****RETURN***FROM***PROCESS***** [D 20:00:50.359927] end iteration [D 20:00:50.359951] done with segments or bytes [D 20:00:50.359974] of 301998065 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:50.360000] next loff: 301998065 next poff: 301998065 [D 20:00:50.360025] return value 8177 [D 20:00:50.360050] finished [D 20:00:50.360074] retval = 8177 [D 20:00:50.360097] exiting distribute returned less than expected [D 20:00:50.360121] done sg 1 sm 1 by 8177 bm 8177 ta 301989888 to 301998065 fo 335544320 eof 0 [D 20:00:50.360148] ========================================================= [D 20:00:50.360173] PINT_encode_calc_max_size [D 20:00:50.360198] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:50.360224] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:50.360253] initializing context[0] 0x15e9cfa0 [D 20:00:50.360279] filling I/O request for 2863311531 [D 20:00:50.360304] [SM Exiting]: (0x15e9c758) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:50.360330] [SM Entering]: (0x15e9c758) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:50.360356] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:50.360381] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:50.360406] PINT_encode [D 20:00:50.360431] encode_common [D 20:00:50.360508] lebf_encode_req [D 20:00:50.360537] PINT_do_request_commit: commit node 0x15e9c700 [D 20:00:50.360563] node stored at 0 [D 20:00:50.360587] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:50.360611] node stored at 1 [D 20:00:50.360635] clearing tree [D 20:00:50.360663] PINT_encode_calc_max_size [D 20:00:50.360688] posting recv with status_user_tag=0 (max_size 24) [D 20:00:50.360716] BMI_post_recv: addr: 367403208, offset: 0x15e9f800, size: 24, tag: 17 [D 20:00:50.360745] openib_check_cq: completed send wr_id 15e72ce8. [D 20:00:50.360771] openib_check_cq: completed recv wr_id 15e73650. [D 20:00:50.360800] openib_post_rr: da13:3346 bh 15 wr_id 15e73650. [D 20:00:50.360840] posting send with status_user_tag=1 [D 20:00:50.360868] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 17 [D 20:00:50.360895] element 0: offset: 0x15e9d6b0, size: 272 [D 20:00:50.360924] post_sr: da13:3346 bh 16 len 288 wr_id 15e72d00. [D 20:00:50.360954] io_datafile_post_msgpairs: completion count is 2 [D 20:00:50.360980] [SM Exiting]: (0x15e9c758) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:50.361006] Posted PVFS_SYS_IO (49) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 11 total buffer count 11 pvfs2_aio_flush id 0 buf 11 offset 335544320, size 33554432 [D 20:00:51.489603] PVFS_isys_io entered [146] [D 20:00:51.489667] [SM Frame PUSH]: (0x15e9fb50) frame: 0x15e9fbe8 [D 20:00:51.489694] [SM Locating]: (0x15e9fb50) op-id: 6 [D 20:00:51.489719] client_op_state_get_machine 6 [D 20:00:51.489744] [SM Locating]: (0x15e9fb50) located: client_io_sm:init [D 20:00:51.489769] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:51.489798] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:51.489824] PINT_client_state_machine_post smcb 0x15e9fb50, op: PVFS_SYS_IO [D 20:00:51.489850] [SM Entering]: (0x15e9fb50) client_io_sm:init (status: 0) [D 20:00:51.489876] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:51.489901] (0x15e9fbe8) io state: io_init [D 20:00:51.489927] [SM Exiting]: (0x15e9fb50) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:51.489953] [SM push_state]: (0x15e9fb50) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:51.489979] [SM Entering]: (0x15e9fb50) client_getattr_sm:acache_lookup (status: 0) [D 20:00:51.490005] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:51.490030] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:51.490056] acache: get_cached_entry(): H=146 [D 20:00:51.490086] acache: miss non-static: H=146 [D 20:00:51.490112] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:51.490138] acache: acache hit [146] [D 20:00:51.490162] [SM Exiting]: (0x15e9fb50) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:51.490188] [SM Entering]: (0x15e9fb50) client_getattr_sm:cleanup (status: 0) [D 20:00:51.490214] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:51.490240] (0x15e9fbe8) getattr state: getattr_cleanup [D 20:00:51.490264] [SM Exiting]: (0x15e9fb50) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:51.490290] [SM pop_state]: (0x15e9fb50) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:51.490315] [SM Entering]: (0x15e9fb50) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:51.490341] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:51.490366] (0x15e9fbe8) io state: io_datafile_setup_msgpairs [D 20:00:51.490393] - io_find_target_datafiles called [D 20:00:51.490418] PINT_new_request_states n=1 [D 20:00:51.490444] PINT_new_request_states n=1 [D 20:00:51.490475] ========================================================= [D 20:00:51.490500] PINT_process_request [D 20:00:51.490525] tiling 12 copies [D 20:00:51.490549] skipping ahead to target_offset [D 20:00:51.490573] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:51.490600] lvl 0 el 0 blk 0 by 0 [D 20:00:51.490625] to 0 ta 335544320 fi 369098752 [D 20:00:51.490650] mto 0 mta 0 mfi 33554432 [D 20:00:51.490675] basic type or contiguous data [D 20:00:51.490699] Request disp [D 20:00:51.490723] contig_offset = 0 contig_size = 402653184 lvl_flag = 1 [D 20:00:51.490749] process logical skip [D 20:00:51.490773] retval = 335544320 [D 20:00:51.490797] exiting logical skip because distribute indicates done [D 20:00:51.490821] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:51.490848] lvl 0 el 0 blk 0 by 335544320 [D 20:00:51.490873] to 335544320 ta 335544320 fi 369098752 [D 20:00:51.490898] mto 0 mta 0 mfi 33554432 [D 20:00:51.490922] basic type or contiguous data [D 20:00:51.490946] Request disp [D 20:00:51.490970] contig_offset = 335544320 contig_size = 67108864 lvl_flag = 1 [D 20:00:51.490996] calling distribute [D 20:00:51.491021] PINT_distribute [D 20:00:51.491044] of 335544320 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:51.491072] begin iteration loff: 335544320 [D 20:00:51.491098] segment exceeds byte limit [D 20:00:51.491122] file being extended [D 20:00:51.491146] client lstof 0 diff 0 sgof 0 [D 20:00:51.491170] **********CALL***PROCESS********* [D 20:00:51.491194] segment of 0 sz 8177 [D 20:00:51.491223] PINT_process_request [D 20:00:51.491248] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:51.491274] lvl 0 el 0 blk 0 by 0 [D 20:00:51.491300] to 0 ta 0 fi 8177 [D 20:00:51.491324] basic type or contiguous data [D 20:00:51.491349] Request disp [D 20:00:51.491373] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:51.491397] process a segment [D 20:00:51.491421] of 0 sz 8177 [D 20:00:51.491446] add a segment 0 [D 20:00:51.491470] retval = 8177 [D 20:00:51.491494] exiting distribute returned less than expected [D 20:00:51.491518] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:51.491544] *****RETURN***FROM***PROCESS***** [D 20:00:51.491568] end iteration [D 20:00:51.491592] done with segments or bytes [D 20:00:51.491617] of 335552497 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:51.491645] next loff: 335552497 next poff: 335552497 [D 20:00:51.491675] return value 8177 [D 20:00:51.491703] finished [D 20:00:51.491731] retval = 8177 [D 20:00:51.491758] exiting distribute returned less than expected [D 20:00:51.491782] done sg 1 sm 1 by 8177 bm 8177 ta 335544320 to 335552497 fo 369098752 eof 0 [D 20:00:51.491808] ========================================================= [D 20:00:51.491833] PINT_encode_calc_max_size [D 20:00:51.491859] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:51.491885] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:51.491913] initializing context[0] 0x15ea0398 [D 20:00:51.491939] filling I/O request for 2863311531 [D 20:00:51.491964] [SM Exiting]: (0x15e9fb50) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:51.491990] [SM Entering]: (0x15e9fb50) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:51.492015] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:51.492040] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:51.492066] PINT_encode [D 20:00:51.492090] encode_common [D 20:00:51.492161] lebf_encode_req [D 20:00:51.492190] PINT_do_request_commit: commit node 0x15e9faf8 [D 20:00:51.492215] node stored at 0 [D 20:00:51.492239] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:51.492264] node stored at 1 [D 20:00:51.492288] clearing tree [D 20:00:51.492316] PINT_encode_calc_max_size [D 20:00:51.492341] posting recv with status_user_tag=0 (max_size 24) [D 20:00:51.492369] BMI_post_recv: addr: 367403208, offset: 0x15ea2bf8, size: 24, tag: 18 [D 20:00:51.492398] openib_check_cq: completed send wr_id 15e72d00. [D 20:00:51.492424] openib_check_cq: completed recv wr_id 15e73668. [D 20:00:51.492453] openib_post_rr: da13:3346 bh 16 wr_id 15e73668. [D 20:00:51.492493] posting send with status_user_tag=1 [D 20:00:51.492520] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 18 [D 20:00:51.492546] element 0: offset: 0x15ea0aa8, size: 272 [D 20:00:51.492576] post_sr: da13:3346 bh 17 len 288 wr_id 15e72d18. [D 20:00:51.492606] io_datafile_post_msgpairs: completion count is 2 [D 20:00:51.492631] [SM Exiting]: (0x15e9fb50) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:51.492658] Posted PVFS_SYS_IO (52) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 12 total buffer count 12 pvfs2_aio_flush id 0 buf 12 offset 369098752, size 33554432 [D 20:00:52.633508] PVFS_isys_io entered [146] [D 20:00:52.633566] [SM Frame PUSH]: (0x15ea2f48) frame: 0x15ea2fe0 [D 20:00:52.633593] [SM Locating]: (0x15ea2f48) op-id: 6 [D 20:00:52.633619] client_op_state_get_machine 6 [D 20:00:52.633646] [SM Locating]: (0x15ea2f48) located: client_io_sm:init [D 20:00:52.633671] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:52.633699] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:52.633726] PINT_client_state_machine_post smcb 0x15ea2f48, op: PVFS_SYS_IO [D 20:00:52.633751] [SM Entering]: (0x15ea2f48) client_io_sm:init (status: 0) [D 20:00:52.633778] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:52.633808] (0x15ea2fe0) io state: io_init [D 20:00:52.633833] [SM Exiting]: (0x15ea2f48) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:52.633861] [SM push_state]: (0x15ea2f48) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:52.633886] [SM Entering]: (0x15ea2f48) client_getattr_sm:acache_lookup (status: 0) [D 20:00:52.633912] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:52.633937] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:52.633963] acache: get_cached_entry(): H=146 [D 20:00:52.633993] acache: miss non-static: H=146 [D 20:00:52.634019] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:52.634044] acache: acache hit [146] [D 20:00:52.634069] [SM Exiting]: (0x15ea2f48) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:52.634095] [SM Entering]: (0x15ea2f48) client_getattr_sm:cleanup (status: 0) [D 20:00:52.634120] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:52.634145] (0x15ea2fe0) getattr state: getattr_cleanup [D 20:00:52.634171] [SM Exiting]: (0x15ea2f48) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:52.634197] [SM pop_state]: (0x15ea2f48) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:52.634222] [SM Entering]: (0x15ea2f48) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:52.634247] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:52.634272] (0x15ea2fe0) io state: io_datafile_setup_msgpairs [D 20:00:52.634300] - io_find_target_datafiles called [D 20:00:52.634325] PINT_new_request_states n=1 [D 20:00:52.634350] PINT_new_request_states n=1 [D 20:00:52.634382] ========================================================= [D 20:00:52.634407] PINT_process_request [D 20:00:52.634433] tiling 13 copies [D 20:00:52.634457] skipping ahead to target_offset [D 20:00:52.634482] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:52.634508] lvl 0 el 0 blk 0 by 0 [D 20:00:52.634532] to 0 ta 369098752 fi 402653184 [D 20:00:52.634557] mto 0 mta 0 mfi 33554432 [D 20:00:52.634582] basic type or contiguous data [D 20:00:52.634607] Request disp [D 20:00:52.634632] contig_offset = 0 contig_size = 436207616 lvl_flag = 1 [D 20:00:52.634656] process logical skip [D 20:00:52.634681] retval = 369098752 [D 20:00:52.634705] exiting logical skip because distribute indicates done [D 20:00:52.634729] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:52.634756] lvl 0 el 0 blk 0 by 369098752 [D 20:00:52.634780] to 369098752 ta 369098752 fi 402653184 [D 20:00:52.634805] mto 0 mta 0 mfi 33554432 [D 20:00:52.634830] basic type or contiguous data [D 20:00:52.634854] Request disp [D 20:00:52.634878] contig_offset = 369098752 contig_size = 67108864 lvl_flag = 1 [D 20:00:52.634903] calling distribute [D 20:00:52.634928] PINT_distribute [D 20:00:52.634952] of 369098752 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:52.634981] begin iteration loff: 369098752 [D 20:00:52.635006] segment exceeds byte limit [D 20:00:52.635030] file being extended [D 20:00:52.635055] client lstof 0 diff 0 sgof 0 [D 20:00:52.635079] **********CALL***PROCESS********* [D 20:00:52.635103] segment of 0 sz 8177 [D 20:00:52.635128] PINT_process_request [D 20:00:52.635152] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:52.635178] lvl 0 el 0 blk 0 by 0 [D 20:00:52.635203] to 0 ta 0 fi 8177 [D 20:00:52.635228] basic type or contiguous data [D 20:00:52.635251] Request disp [D 20:00:52.635275] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:52.635300] process a segment [D 20:00:52.635324] of 0 sz 8177 [D 20:00:52.635349] add a segment 0 [D 20:00:52.635373] retval = 8177 [D 20:00:52.635396] exiting distribute returned less than expected [D 20:00:52.635421] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:52.635447] *****RETURN***FROM***PROCESS***** [D 20:00:52.635472] end iteration [D 20:00:52.635496] done with segments or bytes [D 20:00:52.635522] of 369106929 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:52.635548] next loff: 369106929 next poff: 369106929 [D 20:00:52.635574] return value 8177 [D 20:00:52.635599] finished [D 20:00:52.635622] retval = 8177 [D 20:00:52.635649] exiting distribute returned less than expected [D 20:00:52.635677] done sg 1 sm 1 by 8177 bm 8177 ta 369098752 to 369106929 fo 402653184 eof 0 [D 20:00:52.635707] ========================================================= [D 20:00:52.635736] PINT_encode_calc_max_size [D 20:00:52.635765] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:52.635790] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:52.635820] initializing context[0] 0x15ea3790 [D 20:00:52.635846] filling I/O request for 2863311531 [D 20:00:52.635870] [SM Exiting]: (0x15ea2f48) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:52.635896] [SM Entering]: (0x15ea2f48) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:52.635922] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:52.635947] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:52.635972] PINT_encode [D 20:00:52.635996] encode_common [D 20:00:52.636073] lebf_encode_req [D 20:00:52.636105] PINT_do_request_commit: commit node 0x15ea2ef0 [D 20:00:52.636131] node stored at 0 [D 20:00:52.636155] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:52.636179] node stored at 1 [D 20:00:52.636203] clearing tree [D 20:00:52.636232] PINT_encode_calc_max_size [D 20:00:52.636257] posting recv with status_user_tag=0 (max_size 24) [D 20:00:52.636284] BMI_post_recv: addr: 367403208, offset: 0x15ea5ff0, size: 24, tag: 19 [D 20:00:52.636314] openib_check_cq: completed send wr_id 15e72d18. [D 20:00:52.636341] openib_check_cq: completed recv wr_id 15e73680. [D 20:00:52.636369] openib_post_rr: da13:3346 bh 17 wr_id 15e73680. [D 20:00:52.636412] posting send with status_user_tag=1 [D 20:00:52.636440] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 19 [D 20:00:52.636466] element 0: offset: 0x15ea3ea0, size: 272 [D 20:00:52.636495] post_sr: da13:3346 bh 18 len 288 wr_id 15e72d30. [D 20:00:52.636525] io_datafile_post_msgpairs: completion count is 2 [D 20:00:52.636550] [SM Exiting]: (0x15ea2f48) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:52.636577] Posted PVFS_SYS_IO (55) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 13 total buffer count 13 pvfs2_aio_flush id 0 buf 13 offset 402653184, size 33554432 [D 20:00:53.878408] PVFS_isys_io entered [146] [D 20:00:53.878467] [SM Frame PUSH]: (0x15ea6340) frame: 0x15ea63d8 [D 20:00:53.878494] [SM Locating]: (0x15ea6340) op-id: 6 [D 20:00:53.878519] client_op_state_get_machine 6 [D 20:00:53.878543] [SM Locating]: (0x15ea6340) located: client_io_sm:init [D 20:00:53.878569] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:53.878597] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:53.878623] PINT_client_state_machine_post smcb 0x15ea6340, op: PVFS_SYS_IO [D 20:00:53.878649] [SM Entering]: (0x15ea6340) client_io_sm:init (status: 0) [D 20:00:53.878675] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:53.878700] (0x15ea63d8) io state: io_init [D 20:00:53.878725] [SM Exiting]: (0x15ea6340) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:53.878751] [SM push_state]: (0x15ea6340) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:53.878776] [SM Entering]: (0x15ea6340) client_getattr_sm:acache_lookup (status: 0) [D 20:00:53.878802] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:53.878828] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:53.878853] acache: get_cached_entry(): H=146 [D 20:00:53.878884] acache: miss non-static: H=146 [D 20:00:53.878911] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:53.878935] acache: acache hit [146] [D 20:00:53.878960] [SM Exiting]: (0x15ea6340) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:53.878991] [SM Entering]: (0x15ea6340) client_getattr_sm:cleanup (status: 0) [D 20:00:53.879016] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:53.879042] (0x15ea63d8) getattr state: getattr_cleanup [D 20:00:53.879066] [SM Exiting]: (0x15ea6340) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:53.879092] [SM pop_state]: (0x15ea6340) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:53.879117] [SM Entering]: (0x15ea6340) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:53.879143] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:53.879168] (0x15ea63d8) io state: io_datafile_setup_msgpairs [D 20:00:53.879196] - io_find_target_datafiles called [D 20:00:53.879220] PINT_new_request_states n=1 [D 20:00:53.879246] PINT_new_request_states n=1 [D 20:00:53.879279] ========================================================= [D 20:00:53.879303] PINT_process_request [D 20:00:53.879328] tiling 14 copies [D 20:00:53.879352] skipping ahead to target_offset [D 20:00:53.879376] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:53.879403] lvl 0 el 0 blk 0 by 0 [D 20:00:53.879426] to 0 ta 402653184 fi 436207616 [D 20:00:53.879451] mto 0 mta 0 mfi 33554432 [D 20:00:53.879476] basic type or contiguous data [D 20:00:53.879501] Request disp [D 20:00:53.879525] contig_offset = 0 contig_size = 469762048 lvl_flag = 1 [D 20:00:53.879549] process logical skip [D 20:00:53.879573] retval = 402653184 [D 20:00:53.879597] exiting logical skip because distribute indicates done [D 20:00:53.879622] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:53.879651] lvl 0 el 0 blk 0 by 402653184 [D 20:00:53.879679] to 402653184 ta 402653184 fi 436207616 [D 20:00:53.879708] mto 0 mta 0 mfi 33554432 [D 20:00:53.879736] basic type or contiguous data [D 20:00:53.879764] Request disp [D 20:00:53.879788] contig_offset = 402653184 contig_size = 67108864 lvl_flag = 1 [D 20:00:53.879814] calling distribute [D 20:00:53.879837] PINT_distribute [D 20:00:53.879862] of 402653184 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:53.879889] begin iteration loff: 402653184 [D 20:00:53.879915] segment exceeds byte limit [D 20:00:53.879939] file being extended [D 20:00:53.879962] client lstof 0 diff 0 sgof 0 [D 20:00:53.879987] **********CALL***PROCESS********* [D 20:00:53.880011] segment of 0 sz 8177 [D 20:00:53.880036] PINT_process_request [D 20:00:53.880061] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:53.880086] lvl 0 el 0 blk 0 by 0 [D 20:00:53.880111] to 0 ta 0 fi 8177 [D 20:00:53.880136] basic type or contiguous data [D 20:00:53.880160] Request disp [D 20:00:53.880184] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:53.880208] process a segment [D 20:00:53.880232] of 0 sz 8177 [D 20:00:53.880257] add a segment 0 [D 20:00:53.880281] retval = 8177 [D 20:00:53.880305] exiting distribute returned less than expected [D 20:00:53.880330] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:53.880355] *****RETURN***FROM***PROCESS***** [D 20:00:53.880380] end iteration [D 20:00:53.880404] done with segments or bytes [D 20:00:53.880428] of 402661361 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:53.880454] next loff: 402661361 next poff: 402661361 [D 20:00:53.880478] return value 8177 [D 20:00:53.880503] finished [D 20:00:53.880527] retval = 8177 [D 20:00:53.880551] exiting distribute returned less than expected [D 20:00:53.880575] done sg 1 sm 1 by 8177 bm 8177 ta 402653184 to 402661361 fo 436207616 eof 0 [D 20:00:53.880601] ========================================================= [D 20:00:53.880626] PINT_encode_calc_max_size [D 20:00:53.880651] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:53.880678] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:53.880710] initializing context[0] 0x15ea6b88 [D 20:00:53.880737] filling I/O request for 2863311531 [D 20:00:53.880764] [SM Exiting]: (0x15ea6340) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:53.880791] [SM Entering]: (0x15ea6340) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:53.880817] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:53.880843] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:53.880867] PINT_encode [D 20:00:53.880892] encode_common [D 20:00:53.880967] lebf_encode_req [D 20:00:53.880994] PINT_do_request_commit: commit node 0x15ea62e8 [D 20:00:53.881019] node stored at 0 [D 20:00:53.881044] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:53.881069] node stored at 1 [D 20:00:53.881093] clearing tree [D 20:00:53.881119] PINT_encode_calc_max_size [D 20:00:53.881145] posting recv with status_user_tag=0 (max_size 24) [D 20:00:53.881174] BMI_post_recv: addr: 367403208, offset: 0x15ea93e8, size: 24, tag: 20 [D 20:00:53.881203] openib_check_cq: completed send wr_id 15e72d30. [D 20:00:53.881230] openib_check_cq: completed recv wr_id 15e73698. [D 20:00:53.881258] openib_post_rr: da13:3346 bh 18 wr_id 15e73698. [D 20:00:53.881300] posting send with status_user_tag=1 [D 20:00:53.881328] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 20 [D 20:00:53.881355] element 0: offset: 0x15ea7298, size: 272 [D 20:00:53.881385] post_sr: da13:3346 bh 19 len 288 wr_id 15e72d48. [D 20:00:53.881420] io_datafile_post_msgpairs: completion count is 2 [D 20:00:53.881445] [SM Exiting]: (0x15ea6340) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:53.881475] Posted PVFS_SYS_IO (58) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 14 total buffer count 14 pvfs2_aio_flush id 0 buf 14 offset 436207616, size 33554432 [D 20:00:55.128749] PVFS_isys_io entered [146] [D 20:00:55.128807] [SM Frame PUSH]: (0x15ea9738) frame: 0x15ea97d0 [D 20:00:55.128833] [SM Locating]: (0x15ea9738) op-id: 6 [D 20:00:55.128858] client_op_state_get_machine 6 [D 20:00:55.128883] [SM Locating]: (0x15ea9738) located: client_io_sm:init [D 20:00:55.128909] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:55.128937] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:55.128963] PINT_client_state_machine_post smcb 0x15ea9738, op: PVFS_SYS_IO [D 20:00:55.128990] [SM Entering]: (0x15ea9738) client_io_sm:init (status: 0) [D 20:00:55.129015] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:55.129040] (0x15ea97d0) io state: io_init [D 20:00:55.129066] [SM Exiting]: (0x15ea9738) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:55.129092] [SM push_state]: (0x15ea9738) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:55.129117] [SM Entering]: (0x15ea9738) client_getattr_sm:acache_lookup (status: 0) [D 20:00:55.129143] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:55.129169] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:55.129195] acache: get_cached_entry(): H=146 [D 20:00:55.129223] acache: miss non-static: H=146 [D 20:00:55.129250] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:55.129276] acache: acache hit [146] [D 20:00:55.129300] [SM Exiting]: (0x15ea9738) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:55.129326] [SM Entering]: (0x15ea9738) client_getattr_sm:cleanup (status: 0) [D 20:00:55.129352] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:55.129377] (0x15ea97d0) getattr state: getattr_cleanup [D 20:00:55.129402] [SM Exiting]: (0x15ea9738) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:55.129427] [SM pop_state]: (0x15ea9738) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:55.129452] [SM Entering]: (0x15ea9738) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:55.129478] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:55.129503] (0x15ea97d0) io state: io_datafile_setup_msgpairs [D 20:00:55.129531] - io_find_target_datafiles called [D 20:00:55.129557] PINT_new_request_states n=1 [D 20:00:55.129587] PINT_new_request_states n=1 [D 20:00:55.129628] ========================================================= [D 20:00:55.129655] PINT_process_request [D 20:00:55.129681] tiling 15 copies [D 20:00:55.129706] skipping ahead to target_offset [D 20:00:55.129730] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:55.129756] lvl 0 el 0 blk 0 by 0 [D 20:00:55.129781] to 0 ta 436207616 fi 469762048 [D 20:00:55.129806] mto 0 mta 0 mfi 33554432 [D 20:00:55.129831] basic type or contiguous data [D 20:00:55.129856] Request disp [D 20:00:55.129880] contig_offset = 0 contig_size = 503316480 lvl_flag = 1 [D 20:00:55.129905] process logical skip [D 20:00:55.129929] retval = 436207616 [D 20:00:55.129954] exiting logical skip because distribute indicates done [D 20:00:55.129978] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:55.130003] lvl 0 el 0 blk 0 by 436207616 [D 20:00:55.130028] to 436207616 ta 436207616 fi 469762048 [D 20:00:55.130053] mto 0 mta 0 mfi 33554432 [D 20:00:55.130078] basic type or contiguous data [D 20:00:55.130102] Request disp [D 20:00:55.130125] contig_offset = 436207616 contig_size = 67108864 lvl_flag = 1 [D 20:00:55.130151] calling distribute [D 20:00:55.130176] PINT_distribute [D 20:00:55.130201] of 436207616 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:55.130229] begin iteration loff: 436207616 [D 20:00:55.130255] segment exceeds byte limit [D 20:00:55.130279] file being extended [D 20:00:55.130304] client lstof 0 diff 0 sgof 0 [D 20:00:55.130329] **********CALL***PROCESS********* [D 20:00:55.130353] segment of 0 sz 8177 [D 20:00:55.130377] PINT_process_request [D 20:00:55.130402] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:55.130428] lvl 0 el 0 blk 0 by 0 [D 20:00:55.130454] to 0 ta 0 fi 8177 [D 20:00:55.130477] basic type or contiguous data [D 20:00:55.130501] Request disp [D 20:00:55.130526] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:55.130551] process a segment [D 20:00:55.130575] of 0 sz 8177 [D 20:00:55.130599] add a segment 0 [D 20:00:55.130623] retval = 8177 [D 20:00:55.130647] exiting distribute returned less than expected [D 20:00:55.130671] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:55.130697] *****RETURN***FROM***PROCESS***** [D 20:00:55.130722] end iteration [D 20:00:55.130746] done with segments or bytes [D 20:00:55.130770] of 436215793 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:55.130796] next loff: 436215793 next poff: 436215793 [D 20:00:55.130821] return value 8177 [D 20:00:55.130846] finished [D 20:00:55.130869] retval = 8177 [D 20:00:55.130893] exiting distribute returned less than expected [D 20:00:55.130917] done sg 1 sm 1 by 8177 bm 8177 ta 436207616 to 436215793 fo 469762048 eof 0 [D 20:00:55.130943] ========================================================= [D 20:00:55.130969] PINT_encode_calc_max_size [D 20:00:55.130994] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:55.131020] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:55.131049] initializing context[0] 0x15ea9f80 [D 20:00:55.131073] filling I/O request for 2863311531 [D 20:00:55.131099] [SM Exiting]: (0x15ea9738) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:55.131125] [SM Entering]: (0x15ea9738) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:55.131150] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:55.131175] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:55.131201] PINT_encode [D 20:00:55.131225] encode_common [D 20:00:55.131300] lebf_encode_req [D 20:00:55.131330] PINT_do_request_commit: commit node 0x15ea96e0 [D 20:00:55.131355] node stored at 0 [D 20:00:55.131379] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:55.131403] node stored at 1 [D 20:00:55.131427] clearing tree [D 20:00:55.131455] PINT_encode_calc_max_size [D 20:00:55.131481] posting recv with status_user_tag=0 (max_size 24) [D 20:00:55.131511] BMI_post_recv: addr: 367403208, offset: 0x15eac7e0, size: 24, tag: 21 [D 20:00:55.131541] openib_check_cq: completed send wr_id 15e72d48. [D 20:00:55.131567] openib_check_cq: completed recv wr_id 15e736b0. [D 20:00:55.131597] openib_post_rr: da13:3346 bh 19 wr_id 15e736b0. [D 20:00:55.131640] posting send with status_user_tag=1 [D 20:00:55.131671] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 21 [D 20:00:55.131701] element 0: offset: 0x15eaa690, size: 272 [D 20:00:55.131735] post_sr: da13:3346 bh 20 len 288 wr_id 15e72d60. [D 20:00:55.131769] io_datafile_post_msgpairs: completion count is 2 [D 20:00:55.131794] [SM Exiting]: (0x15ea9738) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:55.131822] Posted PVFS_SYS_IO (61) (waiting for test)(-1073741839) get_free_buffer id 0 new buf # 15 total buffer count 15 rewindfile id 0 current b->size 29404752 pvfs2_aio_flush id 0 buf 15 offset 469762048, size 33554432 [D 20:00:56.317519] PVFS_isys_io entered [146] [D 20:00:56.317565] [SM Frame PUSH]: (0x15eadd18) frame: 0x15eaddb0 [D 20:00:56.317592] [SM Locating]: (0x15eadd18) op-id: 6 [D 20:00:56.317617] client_op_state_get_machine 6 [D 20:00:56.317645] [SM Locating]: (0x15eadd18) located: client_io_sm:init [D 20:00:56.317673] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.317701] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.317727] PINT_client_state_machine_post smcb 0x15eadd18, op: PVFS_SYS_IO [D 20:00:56.317753] [SM Entering]: (0x15eadd18) client_io_sm:init (status: 0) [D 20:00:56.317779] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.317805] (0x15eaddb0) io state: io_init [D 20:00:56.317830] [SM Exiting]: (0x15eadd18) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.317856] [SM push_state]: (0x15eadd18) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.317881] [SM Entering]: (0x15eadd18) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.317907] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.317933] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.317958] acache: get_cached_entry(): H=146 [D 20:00:56.317988] acache: miss non-static: H=146 [D 20:00:56.318014] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.318039] acache: acache hit [146] [D 20:00:56.318063] [SM Exiting]: (0x15eadd18) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.318089] [SM Entering]: (0x15eadd18) client_getattr_sm:cleanup (status: 0) [D 20:00:56.318115] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.318140] (0x15eaddb0) getattr state: getattr_cleanup [D 20:00:56.318164] [SM Exiting]: (0x15eadd18) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.318190] [SM pop_state]: (0x15eadd18) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.318216] [SM Entering]: (0x15eadd18) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.318242] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.318266] (0x15eaddb0) io state: io_datafile_setup_msgpairs [D 20:00:56.318294] - io_find_target_datafiles called [D 20:00:56.318318] PINT_new_request_states n=1 [D 20:00:56.318343] PINT_new_request_states n=1 [D 20:00:56.318375] ========================================================= [D 20:00:56.318400] PINT_process_request [D 20:00:56.318425] tiling 16 copies [D 20:00:56.318449] skipping ahead to target_offset [D 20:00:56.318473] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.318500] lvl 0 el 0 blk 0 by 0 [D 20:00:56.318525] to 0 ta 469762048 fi 503316480 [D 20:00:56.318551] mto 0 mta 0 mfi 33554432 [D 20:00:56.318575] basic type or contiguous data [D 20:00:56.318599] Request disp [D 20:00:56.318624] contig_offset = 0 contig_size = 536870912 lvl_flag = 1 [D 20:00:56.318649] process logical skip [D 20:00:56.318673] retval = 469762048 [D 20:00:56.318697] exiting logical skip because distribute indicates done [D 20:00:56.318729] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.318756] lvl 0 el 0 blk 0 by 469762048 [D 20:00:56.318781] to 469762048 ta 469762048 fi 503316480 [D 20:00:56.318806] mto 0 mta 0 mfi 33554432 [D 20:00:56.318831] basic type or contiguous data [D 20:00:56.318855] Request disp [D 20:00:56.318879] contig_offset = 469762048 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.318904] calling distribute [D 20:00:56.318928] PINT_distribute [D 20:00:56.318952] of 469762048 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.318979] begin iteration loff: 469762048 [D 20:00:56.319005] segment exceeds byte limit [D 20:00:56.319028] file being extended [D 20:00:56.319053] client lstof 0 diff 0 sgof 0 [D 20:00:56.319078] **********CALL***PROCESS********* [D 20:00:56.319102] segment of 0 sz 8177 [D 20:00:56.319127] PINT_process_request [D 20:00:56.319150] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.319177] lvl 0 el 0 blk 0 by 0 [D 20:00:56.319202] to 0 ta 0 fi 8177 [D 20:00:56.319227] basic type or contiguous data [D 20:00:56.319251] Request disp [D 20:00:56.319275] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.319300] process a segment [D 20:00:56.319324] of 0 sz 8177 [D 20:00:56.319349] add a segment 0 [D 20:00:56.319374] retval = 8177 [D 20:00:56.319397] exiting distribute returned less than expected [D 20:00:56.319421] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.319447] *****RETURN***FROM***PROCESS***** [D 20:00:56.319472] end iteration [D 20:00:56.319496] done with segments or bytes [D 20:00:56.319519] of 469770225 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.319545] next loff: 469770225 next poff: 469770225 [D 20:00:56.319570] return value 8177 [D 20:00:56.319595] finished [D 20:00:56.319619] retval = 8177 [D 20:00:56.319643] exiting distribute returned less than expected [D 20:00:56.319670] done sg 1 sm 1 by 8177 bm 8177 ta 469762048 to 469770225 fo 503316480 eof 0 [D 20:00:56.319701] ========================================================= [D 20:00:56.319729] PINT_encode_calc_max_size [D 20:00:56.319758] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.319787] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.319814] initializing context[0] 0x15eae590 [D 20:00:56.319840] filling I/O request for 2863311531 [D 20:00:56.319865] [SM Exiting]: (0x15eadd18) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.319891] [SM Entering]: (0x15eadd18) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.319916] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.319942] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.319967] PINT_encode [D 20:00:56.319992] encode_common [D 20:00:56.320073] lebf_encode_req [D 20:00:56.320101] PINT_do_request_commit: commit node 0x15eadcc0 [D 20:00:56.320126] node stored at 0 [D 20:00:56.320151] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.320175] node stored at 1 [D 20:00:56.320199] clearing tree [D 20:00:56.320227] PINT_encode_calc_max_size [D 20:00:56.320252] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.320281] BMI_post_recv: addr: 367403208, offset: 0x15eb0df0, size: 24, tag: 22 [D 20:00:56.320309] openib_check_cq: completed send wr_id 15e72d60. [D 20:00:56.320336] openib_check_cq: completed recv wr_id 15e736c8. [D 20:00:56.320364] openib_post_rr: da13:3346 bh 20 wr_id 15e736c8. [D 20:00:56.320408] posting send with status_user_tag=1 [D 20:00:56.320436] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 22 [D 20:00:56.320462] element 0: offset: 0x15eaeca0, size: 272 [D 20:00:56.320491] post_sr: da13:3346 bh 21 len 288 wr_id 15e72d78. [D 20:00:56.320521] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.320547] [SM Exiting]: (0x15eadd18) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.320576] Posted PVFS_SYS_IO (64) (waiting for test)(-1073741839) rewindfile id 0 final b->size 33554432 userFileSize 503316480 pvfs2_aio_rewind nothing yet rewindfile id 0 pass 0 elapsed time 13.93 sec 36.13 MiB/sec pid 19921 get_free_buffer id 0 new buf # 16 total buffer count 16 pvfs2_aio_rewind nothing yet get_free_buffer id 0 new buf # 17 total buffer count 17 get_next_full_readbuf: enter userFilePos 0, read_ahead_offset 0 -- buffer 0x15e327d0 curbuf 17 buffer_ahead_offset 0 size 33554432 read_ahead_offset 0 pvfs2_aio_fill buf 17 offset 0, size 33554432 [D 20:00:56.320714] PVFS_isys_io entered [146] [D 20:00:56.320742] [SM Frame PUSH]: (0x15eb1140) frame: 0x15eb11d8 [D 20:00:56.320768] [SM Locating]: (0x15eb1140) op-id: 6 [D 20:00:56.320793] client_op_state_get_machine 6 [D 20:00:56.320817] [SM Locating]: (0x15eb1140) located: client_io_sm:init [D 20:00:56.320842] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.320869] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.320895] PINT_client_state_machine_post smcb 0x15eb1140, op: PVFS_SYS_IO [D 20:00:56.320919] [SM Entering]: (0x15eb1140) client_io_sm:init (status: 0) [D 20:00:56.320945] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.320970] (0x15eb11d8) io state: io_init [D 20:00:56.320995] [SM Exiting]: (0x15eb1140) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.321021] [SM push_state]: (0x15eb1140) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.321046] [SM Entering]: (0x15eb1140) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.321071] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.321097] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.321122] acache: get_cached_entry(): H=146 [D 20:00:56.321148] acache: miss non-static: H=146 [D 20:00:56.321174] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.321199] acache: acache hit [146] [D 20:00:56.321224] [SM Exiting]: (0x15eb1140) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.321249] [SM Entering]: (0x15eb1140) client_getattr_sm:cleanup (status: 0) [D 20:00:56.321274] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.321299] (0x15eb11d8) getattr state: getattr_cleanup [D 20:00:56.321324] [SM Exiting]: (0x15eb1140) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.321350] [SM pop_state]: (0x15eb1140) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.321374] [SM Entering]: (0x15eb1140) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.321400] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.321426] (0x15eb11d8) io state: io_datafile_setup_msgpairs [D 20:00:56.321452] - io_find_target_datafiles called [D 20:00:56.321475] PINT_new_request_states n=1 [D 20:00:56.321500] PINT_new_request_states n=1 [D 20:00:56.321528] ========================================================= [D 20:00:56.321553] PINT_process_request [D 20:00:56.321577] tiling 2 copies [D 20:00:56.321601] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.321628] lvl 0 el 0 blk 0 by 0 [D 20:00:56.321653] to 0 ta 0 fi 33554432 [D 20:00:56.321680] mto 0 mta 0 mfi 33554432 [D 20:00:56.321704] basic type or contiguous data [D 20:00:56.321728] Request disp [D 20:00:56.321752] contig_offset = 0 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.321777] calling distribute [D 20:00:56.321802] PINT_distribute [D 20:00:56.321825] of 0 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.321851] begin iteration loff: 0 [D 20:00:56.321876] segment exceeds byte limit [D 20:00:56.321900] file being extended [D 20:00:56.321924] client lstof 0 diff 0 sgof 0 [D 20:00:56.321949] **********CALL***PROCESS********* [D 20:00:56.321972] segment of 0 sz 8177 [D 20:00:56.321997] PINT_process_request [D 20:00:56.322021] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.322047] lvl 0 el 0 blk 0 by 0 [D 20:00:56.322072] to 0 ta 0 fi 8177 [D 20:00:56.322099] basic type or contiguous data [D 20:00:56.322123] Request disp [D 20:00:56.322147] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.322173] process a segment [D 20:00:56.322196] of 0 sz 8177 [D 20:00:56.322220] add a segment 0 [D 20:00:56.322245] retval = 8177 [D 20:00:56.322268] exiting distribute returned less than expected [D 20:00:56.322293] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.322319] *****RETURN***FROM***PROCESS***** [D 20:00:56.322342] end iteration [D 20:00:56.322366] done with segments or bytes [D 20:00:56.322390] of 8177 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.322416] next loff: 8177 next poff: 8177 [D 20:00:56.322440] return value 8177 [D 20:00:56.322464] finished [D 20:00:56.322488] retval = 8177 [D 20:00:56.322513] exiting distribute returned less than expected [D 20:00:56.322537] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 33554432 eof 0 [D 20:00:56.322563] ========================================================= [D 20:00:56.322587] PINT_encode_calc_max_size [D 20:00:56.322612] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.322638] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.322669] initializing context[0] 0x15eb1988 [D 20:00:56.322695] filling I/O request for 2863311531 [D 20:00:56.322720] [SM Exiting]: (0x15eb1140) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.322746] [SM Entering]: (0x15eb1140) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.322772] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.322797] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.322822] PINT_encode [D 20:00:56.322846] encode_common [D 20:00:56.322898] lebf_encode_req [D 20:00:56.322926] PINT_do_request_commit: commit node 0x15eb10e8 [D 20:00:56.322951] node stored at 0 [D 20:00:56.322975] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.322999] node stored at 1 [D 20:00:56.323024] clearing tree [D 20:00:56.323052] PINT_encode_calc_max_size [D 20:00:56.323077] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.323103] BMI_post_recv: addr: 367403208, offset: 0x15eb41e8, size: 24, tag: 23 [D 20:00:56.323131] openib_check_cq: completed send wr_id 15e72d78. [D 20:00:56.323165] posting send with status_user_tag=1 [D 20:00:56.323192] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 23 [D 20:00:56.323218] element 0: offset: 0x15eb2098, size: 272 [D 20:00:56.323247] post_sr: da13:3346 bh 22 len 288 wr_id 15e72d90. [D 20:00:56.323276] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.323302] [SM Exiting]: (0x15eb1140) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.323328] Posted PVFS_SYS_IO (67) (waiting for test)(-1073741839) -- buffer 0x15e327f8 curbuf 17 buffer_ahead_offset 1 size 33554432 read_ahead_offset 33554432 pvfs2_aio_fill buf 18 offset 33554432, size 33554432 [D 20:00:56.323363] PVFS_isys_io entered [146] [D 20:00:56.323390] [SM Frame PUSH]: (0x15eb45f8) frame: 0x15eb4690 [D 20:00:56.323415] [SM Locating]: (0x15eb45f8) op-id: 6 [D 20:00:56.323439] client_op_state_get_machine 6 [D 20:00:56.323464] [SM Locating]: (0x15eb45f8) located: client_io_sm:init [D 20:00:56.323489] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.323515] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.323541] PINT_client_state_machine_post smcb 0x15eb45f8, op: PVFS_SYS_IO [D 20:00:56.323567] [SM Entering]: (0x15eb45f8) client_io_sm:init (status: 0) [D 20:00:56.323592] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.323616] (0x15eb4690) io state: io_init [D 20:00:56.323641] [SM Exiting]: (0x15eb45f8) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.323671] [SM push_state]: (0x15eb45f8) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.323700] [SM Entering]: (0x15eb45f8) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.323733] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.323762] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.323789] acache: get_cached_entry(): H=146 [D 20:00:56.323816] acache: miss non-static: H=146 [D 20:00:56.323842] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.323866] acache: acache hit [146] [D 20:00:56.323890] [SM Exiting]: (0x15eb45f8) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.323917] [SM Entering]: (0x15eb45f8) client_getattr_sm:cleanup (status: 0) [D 20:00:56.323942] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.323967] (0x15eb4690) getattr state: getattr_cleanup [D 20:00:56.323991] [SM Exiting]: (0x15eb45f8) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.324017] [SM pop_state]: (0x15eb45f8) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.324043] [SM Entering]: (0x15eb45f8) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.324069] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.324093] (0x15eb4690) io state: io_datafile_setup_msgpairs [D 20:00:56.324119] - io_find_target_datafiles called [D 20:00:56.324143] PINT_new_request_states n=1 [D 20:00:56.324168] PINT_new_request_states n=1 [D 20:00:56.324195] ========================================================= [D 20:00:56.324220] PINT_process_request [D 20:00:56.324244] tiling 3 copies [D 20:00:56.324269] skipping ahead to target_offset [D 20:00:56.324293] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.324318] lvl 0 el 0 blk 0 by 0 [D 20:00:56.324344] to 0 ta 33554432 fi 67108864 [D 20:00:56.324369] mto 0 mta 0 mfi 33554432 [D 20:00:56.324394] basic type or contiguous data [D 20:00:56.324418] Request disp [D 20:00:56.324441] contig_offset = 0 contig_size = 100663296 lvl_flag = 1 [D 20:00:56.324466] process logical skip [D 20:00:56.324490] retval = 33554432 [D 20:00:56.324515] exiting logical skip because distribute indicates done [D 20:00:56.324539] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.324564] lvl 0 el 0 blk 0 by 33554432 [D 20:00:56.324589] to 33554432 ta 33554432 fi 67108864 [D 20:00:56.324614] mto 0 mta 0 mfi 33554432 [D 20:00:56.324639] basic type or contiguous data [D 20:00:56.324663] Request disp [D 20:00:56.324687] contig_offset = 33554432 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.324712] calling distribute [D 20:00:56.324736] PINT_distribute [D 20:00:56.324760] of 33554432 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.324786] begin iteration loff: 33554432 [D 20:00:56.324810] segment exceeds byte limit [D 20:00:56.324834] file being extended [D 20:00:56.324858] client lstof 0 diff 0 sgof 0 [D 20:00:56.324883] **********CALL***PROCESS********* [D 20:00:56.324907] segment of 0 sz 8177 [D 20:00:56.324932] PINT_process_request [D 20:00:56.324955] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.324982] lvl 0 el 0 blk 0 by 0 [D 20:00:56.325006] to 0 ta 0 fi 8177 [D 20:00:56.325031] basic type or contiguous data [D 20:00:56.325055] Request disp [D 20:00:56.325079] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.325104] process a segment [D 20:00:56.325128] of 0 sz 8177 [D 20:00:56.325152] add a segment 0 [D 20:00:56.325176] retval = 8177 [D 20:00:56.325201] exiting distribute returned less than expected [D 20:00:56.325224] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.325250] *****RETURN***FROM***PROCESS***** [D 20:00:56.325274] end iteration [D 20:00:56.325298] done with segments or bytes [D 20:00:56.325322] of 33562609 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.325348] next loff: 33562609 next poff: 33562609 [D 20:00:56.325373] return value 8177 [D 20:00:56.325397] finished [D 20:00:56.325421] retval = 8177 [D 20:00:56.325446] exiting distribute returned less than expected [D 20:00:56.325469] done sg 1 sm 1 by 8177 bm 8177 ta 33554432 to 33562609 fo 67108864 eof 0 [D 20:00:56.325498] ========================================================= [D 20:00:56.325524] PINT_encode_calc_max_size [D 20:00:56.325549] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.325574] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.325605] initializing context[0] 0x15eb4e60 [D 20:00:56.325632] filling I/O request for 2863311531 [D 20:00:56.325658] [SM Exiting]: (0x15eb45f8) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.325685] [SM Entering]: (0x15eb45f8) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.325711] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.325736] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.325760] PINT_encode [D 20:00:56.325785] encode_common [D 20:00:56.325834] lebf_encode_req [D 20:00:56.325860] PINT_do_request_commit: commit node 0x15eb45a0 [D 20:00:56.325885] node stored at 0 [D 20:00:56.325910] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.325935] node stored at 1 [D 20:00:56.325959] clearing tree [D 20:00:56.325986] PINT_encode_calc_max_size [D 20:00:56.326011] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.326038] BMI_post_recv: addr: 367403208, offset: 0x15eb76c0, size: 24, tag: 24 [D 20:00:56.326065] openib_check_cq: completed send wr_id 15e72d90. [D 20:00:56.326100] posting send with status_user_tag=1 [D 20:00:56.326127] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 24 [D 20:00:56.326152] element 0: offset: 0x15eb5570, size: 272 [D 20:00:56.326182] post_sr: da13:3346 bh 23 len 288 wr_id 15e72da8. [D 20:00:56.326210] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.326236] [SM Exiting]: (0x15eb45f8) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.326263] Posted PVFS_SYS_IO (70) (waiting for test)(-1073741839) -- buffer 0x15e32820 curbuf 17 buffer_ahead_offset 2 size 33554432 read_ahead_offset 67108864 pvfs2_aio_fill buf 19 offset 67108864, size 33554432 [D 20:00:56.326297] PVFS_isys_io entered [146] [D 20:00:56.326327] [SM Frame PUSH]: (0x15eb7ad0) frame: 0x15eb7b68 [D 20:00:56.326353] [SM Locating]: (0x15eb7ad0) op-id: 6 [D 20:00:56.326377] client_op_state_get_machine 6 [D 20:00:56.326402] [SM Locating]: (0x15eb7ad0) located: client_io_sm:init [D 20:00:56.326427] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.326454] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.326479] PINT_client_state_machine_post smcb 0x15eb7ad0, op: PVFS_SYS_IO [D 20:00:56.326504] [SM Entering]: (0x15eb7ad0) client_io_sm:init (status: 0) [D 20:00:56.326530] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.326555] (0x15eb7b68) io state: io_init [D 20:00:56.326580] [SM Exiting]: (0x15eb7ad0) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.326605] [SM push_state]: (0x15eb7ad0) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.326631] [SM Entering]: (0x15eb7ad0) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.326656] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.326682] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.326706] acache: get_cached_entry(): H=146 [D 20:00:56.326733] acache: miss non-static: H=146 [D 20:00:56.326759] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.326784] acache: acache hit [146] [D 20:00:56.326808] [SM Exiting]: (0x15eb7ad0) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.326834] [SM Entering]: (0x15eb7ad0) client_getattr_sm:cleanup (status: 0) [D 20:00:56.326859] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.326884] (0x15eb7b68) getattr state: getattr_cleanup [D 20:00:56.326909] [SM Exiting]: (0x15eb7ad0) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.326934] [SM pop_state]: (0x15eb7ad0) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.326960] [SM Entering]: (0x15eb7ad0) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.326988] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.327013] (0x15eb7b68) io state: io_datafile_setup_msgpairs [D 20:00:56.327040] - io_find_target_datafiles called [D 20:00:56.327064] PINT_new_request_states n=1 [D 20:00:56.327089] PINT_new_request_states n=1 [D 20:00:56.327116] ========================================================= [D 20:00:56.327140] PINT_process_request [D 20:00:56.327165] tiling 4 copies [D 20:00:56.327189] skipping ahead to target_offset [D 20:00:56.327213] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.327239] lvl 0 el 0 blk 0 by 0 [D 20:00:56.327263] to 0 ta 67108864 fi 100663296 [D 20:00:56.327288] mto 0 mta 0 mfi 33554432 [D 20:00:56.327313] basic type or contiguous data [D 20:00:56.327337] Request disp [D 20:00:56.327361] contig_offset = 0 contig_size = 134217728 lvl_flag = 1 [D 20:00:56.327386] process logical skip [D 20:00:56.327410] retval = 67108864 [D 20:00:56.327434] exiting logical skip because distribute indicates done [D 20:00:56.327458] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.327485] lvl 0 el 0 blk 0 by 67108864 [D 20:00:56.327509] to 67108864 ta 67108864 fi 100663296 [D 20:00:56.327534] mto 0 mta 0 mfi 33554432 [D 20:00:56.327559] basic type or contiguous data [D 20:00:56.327583] Request disp [D 20:00:56.327607] contig_offset = 67108864 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.327633] calling distribute [D 20:00:56.327659] PINT_distribute [D 20:00:56.327688] of 67108864 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.327718] begin iteration loff: 67108864 [D 20:00:56.327746] segment exceeds byte limit [D 20:00:56.327774] file being extended [D 20:00:56.327797] client lstof 0 diff 0 sgof 0 [D 20:00:56.327822] **********CALL***PROCESS********* [D 20:00:56.327846] segment of 0 sz 8177 [D 20:00:56.327870] PINT_process_request [D 20:00:56.327895] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.327920] lvl 0 el 0 blk 0 by 0 [D 20:00:56.327945] to 0 ta 0 fi 8177 [D 20:00:56.327970] basic type or contiguous data [D 20:00:56.327994] Request disp [D 20:00:56.328018] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.328042] process a segment [D 20:00:56.328066] of 0 sz 8177 [D 20:00:56.328090] add a segment 0 [D 20:00:56.328114] retval = 8177 [D 20:00:56.328138] exiting distribute returned less than expected [D 20:00:56.328163] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.328188] *****RETURN***FROM***PROCESS***** [D 20:00:56.328212] end iteration [D 20:00:56.328236] done with segments or bytes [D 20:00:56.328260] of 67117041 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.328286] next loff: 67117041 next poff: 67117041 [D 20:00:56.328310] return value 8177 [D 20:00:56.328335] finished [D 20:00:56.328359] retval = 8177 [D 20:00:56.328383] exiting distribute returned less than expected [D 20:00:56.328407] done sg 1 sm 1 by 8177 bm 8177 ta 67108864 to 67117041 fo 100663296 eof 0 [D 20:00:56.328433] ========================================================= [D 20:00:56.328458] PINT_encode_calc_max_size [D 20:00:56.328483] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.328508] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.328536] initializing context[0] 0x15eb8338 [D 20:00:56.328561] filling I/O request for 2863311531 [D 20:00:56.328586] [SM Exiting]: (0x15eb7ad0) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.328612] [SM Entering]: (0x15eb7ad0) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.328637] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.328662] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.328687] PINT_encode [D 20:00:56.328711] encode_common [D 20:00:56.328758] lebf_encode_req [D 20:00:56.328787] PINT_do_request_commit: commit node 0x15eb7a78 [D 20:00:56.328813] node stored at 0 [D 20:00:56.328838] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.328862] node stored at 1 [D 20:00:56.328886] clearing tree [D 20:00:56.328913] PINT_encode_calc_max_size [D 20:00:56.328938] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.328966] BMI_post_recv: addr: 367403208, offset: 0x15ebab98, size: 24, tag: 25 [D 20:00:56.328992] openib_check_cq: completed send wr_id 15e72da8. [D 20:00:56.329027] posting send with status_user_tag=1 [D 20:00:56.329053] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 25 [D 20:00:56.329079] element 0: offset: 0x15eb8a48, size: 272 [D 20:00:56.329109] post_sr: da13:3346 bh 24 len 288 wr_id 15e72dc0. [D 20:00:56.329138] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.329163] [SM Exiting]: (0x15eb7ad0) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.329190] Posted PVFS_SYS_IO (73) (waiting for test)(-1073741839) -- buffer 0x15e32848 curbuf 17 buffer_ahead_offset 3 size 33554432 read_ahead_offset 100663296 pvfs2_aio_fill buf 20 offset 100663296, size 33554432 [D 20:00:56.329224] PVFS_isys_io entered [146] [D 20:00:56.329254] [SM Frame PUSH]: (0x15ebafa8) frame: 0x15ebb040 [D 20:00:56.329280] [SM Locating]: (0x15ebafa8) op-id: 6 [D 20:00:56.329305] client_op_state_get_machine 6 [D 20:00:56.329329] [SM Locating]: (0x15ebafa8) located: client_io_sm:init [D 20:00:56.329355] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.329381] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.329406] PINT_client_state_machine_post smcb 0x15ebafa8, op: PVFS_SYS_IO [D 20:00:56.329432] [SM Entering]: (0x15ebafa8) client_io_sm:init (status: 0) [D 20:00:56.329457] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.329483] (0x15ebb040) io state: io_init [D 20:00:56.329506] [SM Exiting]: (0x15ebafa8) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.329532] [SM push_state]: (0x15ebafa8) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.329557] [SM Entering]: (0x15ebafa8) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.329583] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.329607] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.329632] acache: get_cached_entry(): H=146 [D 20:00:56.329661] acache: miss non-static: H=146 [D 20:00:56.329687] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.329711] acache: acache hit [146] [D 20:00:56.329736] [SM Exiting]: (0x15ebafa8) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.329762] [SM Entering]: (0x15ebafa8) client_getattr_sm:cleanup (status: 0) [D 20:00:56.329788] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.329813] (0x15ebb040) getattr state: getattr_cleanup [D 20:00:56.329837] [SM Exiting]: (0x15ebafa8) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.329862] [SM pop_state]: (0x15ebafa8) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.329888] [SM Entering]: (0x15ebafa8) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.329913] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.329938] (0x15ebb040) io state: io_datafile_setup_msgpairs [D 20:00:56.329964] - io_find_target_datafiles called [D 20:00:56.329988] PINT_new_request_states n=1 [D 20:00:56.330013] PINT_new_request_states n=1 [D 20:00:56.330040] ========================================================= [D 20:00:56.330065] PINT_process_request [D 20:00:56.330089] tiling 5 copies [D 20:00:56.330113] skipping ahead to target_offset [D 20:00:56.330138] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.330163] lvl 0 el 0 blk 0 by 0 [D 20:00:56.330188] to 0 ta 100663296 fi 134217728 [D 20:00:56.330213] mto 0 mta 0 mfi 33554432 [D 20:00:56.330238] basic type or contiguous data [D 20:00:56.330262] Request disp [D 20:00:56.330286] contig_offset = 0 contig_size = 167772160 lvl_flag = 1 [D 20:00:56.330313] process logical skip [D 20:00:56.330337] retval = 100663296 [D 20:00:56.330362] exiting logical skip because distribute indicates done [D 20:00:56.330386] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.330412] lvl 0 el 0 blk 0 by 100663296 [D 20:00:56.330437] to 100663296 ta 100663296 fi 134217728 [D 20:00:56.330462] mto 0 mta 0 mfi 33554432 [D 20:00:56.330487] basic type or contiguous data [D 20:00:56.330511] Request disp [D 20:00:56.330534] contig_offset = 100663296 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.330559] calling distribute [D 20:00:56.330584] PINT_distribute [D 20:00:56.330608] of 100663296 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.330634] begin iteration loff: 100663296 [D 20:00:56.330658] segment exceeds byte limit [D 20:00:56.330683] file being extended [D 20:00:56.330707] client lstof 0 diff 0 sgof 0 [D 20:00:56.330731] **********CALL***PROCESS********* [D 20:00:56.330755] segment of 0 sz 8177 [D 20:00:56.330780] PINT_process_request [D 20:00:56.330803] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.330829] lvl 0 el 0 blk 0 by 0 [D 20:00:56.330854] to 0 ta 0 fi 8177 [D 20:00:56.330879] basic type or contiguous data [D 20:00:56.330903] Request disp [D 20:00:56.330926] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.330952] process a segment [D 20:00:56.330976] of 0 sz 8177 [D 20:00:56.331000] add a segment 0 [D 20:00:56.331025] retval = 8177 [D 20:00:56.331048] exiting distribute returned less than expected [D 20:00:56.331072] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.331098] *****RETURN***FROM***PROCESS***** [D 20:00:56.331122] end iteration [D 20:00:56.331146] done with segments or bytes [D 20:00:56.331171] of 100671473 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.331196] next loff: 100671473 next poff: 100671473 [D 20:00:56.331221] return value 8177 [D 20:00:56.331245] finished [D 20:00:56.331270] retval = 8177 [D 20:00:56.331294] exiting distribute returned less than expected [D 20:00:56.331317] done sg 1 sm 1 by 8177 bm 8177 ta 100663296 to 100671473 fo 134217728 eof 0 [D 20:00:56.331343] ========================================================= [D 20:00:56.331369] PINT_encode_calc_max_size [D 20:00:56.331394] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.331419] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.331447] initializing context[0] 0x15ebb810 [D 20:00:56.331472] filling I/O request for 2863311531 [D 20:00:56.331497] [SM Exiting]: (0x15ebafa8) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.331522] [SM Entering]: (0x15ebafa8) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.331548] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.331573] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.331598] PINT_encode [D 20:00:56.331623] encode_common [D 20:00:56.331675] lebf_encode_req [D 20:00:56.331706] PINT_do_request_commit: commit node 0x15ebaf50 [D 20:00:56.331734] node stored at 0 [D 20:00:56.331762] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.331789] node stored at 1 [D 20:00:56.331812] clearing tree [D 20:00:56.331840] PINT_encode_calc_max_size [D 20:00:56.331866] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.331892] BMI_post_recv: addr: 367403208, offset: 0x15ebe070, size: 24, tag: 26 [D 20:00:56.331920] openib_check_cq: completed send wr_id 15e72dc0. [D 20:00:56.331954] posting send with status_user_tag=1 [D 20:00:56.331981] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 26 [D 20:00:56.332007] element 0: offset: 0x15ebbf20, size: 272 [D 20:00:56.332036] post_sr: da13:3346 bh 25 len 288 wr_id 15e72dd8. [D 20:00:56.332065] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.332090] [SM Exiting]: (0x15ebafa8) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.332119] Posted PVFS_SYS_IO (76) (waiting for test)(-1073741839) -- buffer 0x15e32870 curbuf 17 buffer_ahead_offset 4 size 33554432 read_ahead_offset 134217728 pvfs2_aio_fill buf 21 offset 134217728, size 33554432 [D 20:00:56.332154] PVFS_isys_io entered [146] [D 20:00:56.332180] [SM Frame PUSH]: (0x15ebe480) frame: 0x15ebe518 [D 20:00:56.332205] [SM Locating]: (0x15ebe480) op-id: 6 [D 20:00:56.332230] client_op_state_get_machine 6 [D 20:00:56.332255] [SM Locating]: (0x15ebe480) located: client_io_sm:init [D 20:00:56.332280] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.332306] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.332332] PINT_client_state_machine_post smcb 0x15ebe480, op: PVFS_SYS_IO [D 20:00:56.332357] [SM Entering]: (0x15ebe480) client_io_sm:init (status: 0) [D 20:00:56.332382] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.332407] (0x15ebe518) io state: io_init [D 20:00:56.332432] [SM Exiting]: (0x15ebe480) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.332458] [SM push_state]: (0x15ebe480) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.332483] [SM Entering]: (0x15ebe480) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.332509] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.332533] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.332558] acache: get_cached_entry(): H=146 [D 20:00:56.332585] acache: miss non-static: H=146 [D 20:00:56.332611] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.332635] acache: acache hit [146] [D 20:00:56.332660] [SM Exiting]: (0x15ebe480) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.332685] [SM Entering]: (0x15ebe480) client_getattr_sm:cleanup (status: 0) [D 20:00:56.332711] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.332736] (0x15ebe518) getattr state: getattr_cleanup [D 20:00:56.332760] [SM Exiting]: (0x15ebe480) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.332786] [SM pop_state]: (0x15ebe480) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.332812] [SM Entering]: (0x15ebe480) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.332837] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.332861] (0x15ebe518) io state: io_datafile_setup_msgpairs [D 20:00:56.332888] - io_find_target_datafiles called [D 20:00:56.332912] PINT_new_request_states n=1 [D 20:00:56.332937] PINT_new_request_states n=1 [D 20:00:56.332964] ========================================================= [D 20:00:56.332988] PINT_process_request [D 20:00:56.333013] tiling 6 copies [D 20:00:56.333037] skipping ahead to target_offset [D 20:00:56.333061] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.333086] lvl 0 el 0 blk 0 by 0 [D 20:00:56.333111] to 0 ta 134217728 fi 167772160 [D 20:00:56.333136] mto 0 mta 0 mfi 33554432 [D 20:00:56.333161] basic type or contiguous data [D 20:00:56.333185] Request disp [D 20:00:56.333209] contig_offset = 0 contig_size = 201326592 lvl_flag = 1 [D 20:00:56.333233] process logical skip [D 20:00:56.333257] retval = 134217728 [D 20:00:56.333281] exiting logical skip because distribute indicates done [D 20:00:56.333306] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.333332] lvl 0 el 0 blk 0 by 134217728 [D 20:00:56.333356] to 134217728 ta 134217728 fi 167772160 [D 20:00:56.333381] mto 0 mta 0 mfi 33554432 [D 20:00:56.333406] basic type or contiguous data [D 20:00:56.333430] Request disp [D 20:00:56.333454] contig_offset = 134217728 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.333478] calling distribute [D 20:00:56.333502] PINT_distribute [D 20:00:56.333526] of 134217728 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.333553] begin iteration loff: 134217728 [D 20:00:56.333578] segment exceeds byte limit [D 20:00:56.333601] file being extended [D 20:00:56.333628] client lstof 0 diff 0 sgof 0 [D 20:00:56.333653] **********CALL***PROCESS********* [D 20:00:56.333680] segment of 0 sz 8177 [D 20:00:56.333704] PINT_process_request [D 20:00:56.333728] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.333755] lvl 0 el 0 blk 0 by 0 [D 20:00:56.333779] to 0 ta 0 fi 8177 [D 20:00:56.333804] basic type or contiguous data [D 20:00:56.333827] Request disp [D 20:00:56.333851] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.333876] process a segment [D 20:00:56.333900] of 0 sz 8177 [D 20:00:56.333925] add a segment 0 [D 20:00:56.333949] retval = 8177 [D 20:00:56.333972] exiting distribute returned less than expected [D 20:00:56.333996] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.334022] *****RETURN***FROM***PROCESS***** [D 20:00:56.334046] end iteration [D 20:00:56.334070] done with segments or bytes [D 20:00:56.334094] of 134225905 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.334119] next loff: 134225905 next poff: 134225905 [D 20:00:56.334144] return value 8177 [D 20:00:56.334169] finished [D 20:00:56.334193] retval = 8177 [D 20:00:56.334217] exiting distribute returned less than expected [D 20:00:56.334240] done sg 1 sm 1 by 8177 bm 8177 ta 134217728 to 134225905 fo 167772160 eof 0 [D 20:00:56.334266] ========================================================= [D 20:00:56.334291] PINT_encode_calc_max_size [D 20:00:56.334316] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.334342] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.334372] initializing context[0] 0x15ebece8 [D 20:00:56.334398] filling I/O request for 2863311531 [D 20:00:56.334423] [SM Exiting]: (0x15ebe480) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.334449] [SM Entering]: (0x15ebe480) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.334474] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.334500] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.334525] PINT_encode [D 20:00:56.334548] encode_common [D 20:00:56.334595] lebf_encode_req [D 20:00:56.334622] PINT_do_request_commit: commit node 0x15ebe428 [D 20:00:56.334648] node stored at 0 [D 20:00:56.334671] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.334696] node stored at 1 [D 20:00:56.334720] clearing tree [D 20:00:56.334747] PINT_encode_calc_max_size [D 20:00:56.334773] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.334799] BMI_post_recv: addr: 367403208, offset: 0x15ec1548, size: 24, tag: 27 [D 20:00:56.334826] openib_check_cq: completed send wr_id 15e72dd8. [D 20:00:56.334852] openib_check_cq: completed recv wr_id 15e736e0. [D 20:00:56.334879] openib_post_rr: da13:3346 bh 21 wr_id 15e736e0. [D 20:00:56.334916] posting send with status_user_tag=1 [D 20:00:56.334943] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 27 [D 20:00:56.334969] element 0: offset: 0x15ebf3f8, size: 272 [D 20:00:56.334998] post_sr: da13:3346 bh 26 len 288 wr_id 15e72df0. [D 20:00:56.335027] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.335052] [SM Exiting]: (0x15ebe480) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.335079] Posted PVFS_SYS_IO (79) (waiting for test)(-1073741839) -- buffer 0x15e32898 curbuf 17 buffer_ahead_offset 5 size 33554432 read_ahead_offset 167772160 pvfs2_aio_fill buf 22 offset 167772160, size 33554432 [D 20:00:56.335112] PVFS_isys_io entered [146] [D 20:00:56.335143] [SM Frame PUSH]: (0x15ec1880) frame: 0x15ec1918 [D 20:00:56.335168] [SM Locating]: (0x15ec1880) op-id: 6 [D 20:00:56.335193] client_op_state_get_machine 6 [D 20:00:56.335217] [SM Locating]: (0x15ec1880) located: client_io_sm:init [D 20:00:56.335243] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.335269] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.335294] PINT_client_state_machine_post smcb 0x15ec1880, op: PVFS_SYS_IO [D 20:00:56.335323] [SM Entering]: (0x15ec1880) client_io_sm:init (status: 0) [D 20:00:56.335348] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.335373] (0x15ec1918) io state: io_init [D 20:00:56.335398] [SM Exiting]: (0x15ec1880) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.335423] [SM push_state]: (0x15ec1880) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.335449] [SM Entering]: (0x15ec1880) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.335474] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.335499] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.335524] acache: get_cached_entry(): H=146 [D 20:00:56.335550] acache: miss non-static: H=146 [D 20:00:56.335576] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.335601] acache: acache hit [146] [D 20:00:56.335625] [SM Exiting]: (0x15ec1880) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.335652] [SM Entering]: (0x15ec1880) client_getattr_sm:cleanup (status: 0) [D 20:00:56.335681] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.335710] (0x15ec1918) getattr state: getattr_cleanup [D 20:00:56.335738] [SM Exiting]: (0x15ec1880) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.335767] [SM pop_state]: (0x15ec1880) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.335793] [SM Entering]: (0x15ec1880) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.335819] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.335844] (0x15ec1918) io state: io_datafile_setup_msgpairs [D 20:00:56.335870] - io_find_target_datafiles called [D 20:00:56.335894] PINT_new_request_states n=1 [D 20:00:56.335919] PINT_new_request_states n=1 [D 20:00:56.335947] ========================================================= [D 20:00:56.335970] PINT_process_request [D 20:00:56.335995] tiling 7 copies [D 20:00:56.336019] skipping ahead to target_offset [D 20:00:56.336043] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.336070] lvl 0 el 0 blk 0 by 0 [D 20:00:56.336094] to 0 ta 167772160 fi 201326592 [D 20:00:56.336119] mto 0 mta 0 mfi 33554432 [D 20:00:56.336144] basic type or contiguous data [D 20:00:56.336168] Request disp [D 20:00:56.336192] contig_offset = 0 contig_size = 234881024 lvl_flag = 1 [D 20:00:56.336216] process logical skip [D 20:00:56.336240] retval = 167772160 [D 20:00:56.336265] exiting logical skip because distribute indicates done [D 20:00:56.336289] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.336315] lvl 0 el 0 blk 0 by 167772160 [D 20:00:56.336339] to 167772160 ta 167772160 fi 201326592 [D 20:00:56.336365] mto 0 mta 0 mfi 33554432 [D 20:00:56.336390] basic type or contiguous data [D 20:00:56.336414] Request disp [D 20:00:56.336438] contig_offset = 167772160 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.336462] calling distribute [D 20:00:56.336486] PINT_distribute [D 20:00:56.336510] of 167772160 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.336537] begin iteration loff: 167772160 [D 20:00:56.336562] segment exceeds byte limit [D 20:00:56.336585] file being extended [D 20:00:56.336609] client lstof 0 diff 0 sgof 0 [D 20:00:56.336634] **********CALL***PROCESS********* [D 20:00:56.336658] segment of 0 sz 8177 [D 20:00:56.336682] PINT_process_request [D 20:00:56.336706] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.336732] lvl 0 el 0 blk 0 by 0 [D 20:00:56.336757] to 0 ta 0 fi 8177 [D 20:00:56.336782] basic type or contiguous data [D 20:00:56.336806] Request disp [D 20:00:56.336830] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.336854] process a segment [D 20:00:56.336878] of 0 sz 8177 [D 20:00:56.336903] add a segment 0 [D 20:00:56.336927] retval = 8177 [D 20:00:56.336951] exiting distribute returned less than expected [D 20:00:56.336975] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.337003] *****RETURN***FROM***PROCESS***** [D 20:00:56.337027] end iteration [D 20:00:56.337051] done with segments or bytes [D 20:00:56.337076] of 167780337 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.337101] next loff: 167780337 next poff: 167780337 [D 20:00:56.337126] return value 8177 [D 20:00:56.337150] finished [D 20:00:56.337174] retval = 8177 [D 20:00:56.337199] exiting distribute returned less than expected [D 20:00:56.337223] done sg 1 sm 1 by 8177 bm 8177 ta 167772160 to 167780337 fo 201326592 eof 0 [D 20:00:56.337248] ========================================================= [D 20:00:56.337273] PINT_encode_calc_max_size [D 20:00:56.337298] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.337324] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.337351] initializing context[0] 0x15ec20c8 [D 20:00:56.337377] filling I/O request for 2863311531 [D 20:00:56.337402] [SM Exiting]: (0x15ec1880) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.337428] [SM Entering]: (0x15ec1880) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.337453] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.337478] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.337503] PINT_encode [D 20:00:56.337527] encode_common [D 20:00:56.337574] lebf_encode_req [D 20:00:56.337601] PINT_do_request_commit: commit node 0x15ec1828 [D 20:00:56.337626] node stored at 0 [D 20:00:56.337653] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.337679] node stored at 1 [D 20:00:56.337704] clearing tree [D 20:00:56.337731] PINT_encode_calc_max_size [D 20:00:56.337756] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.337782] BMI_post_recv: addr: 367403208, offset: 0x15ec4928, size: 24, tag: 28 [D 20:00:56.337809] openib_check_cq: completed send wr_id 15e72df0. [D 20:00:56.337843] posting send with status_user_tag=1 [D 20:00:56.337871] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 28 [D 20:00:56.337897] element 0: offset: 0x15ec27d8, size: 272 [D 20:00:56.337925] post_sr: da13:3346 bh 27 len 288 wr_id 15e72e08. [D 20:00:56.337954] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.337980] [SM Exiting]: (0x15ec1880) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.338006] Posted PVFS_SYS_IO (82) (waiting for test)(-1073741839) -- buffer 0x15e328c0 curbuf 17 buffer_ahead_offset 6 size 33554432 read_ahead_offset 201326592 pvfs2_aio_fill buf 23 offset 201326592, size 33554432 [D 20:00:56.338041] PVFS_isys_io entered [146] [D 20:00:56.338070] [SM Frame PUSH]: (0x15ec4d38) frame: 0x15ec4dd0 [D 20:00:56.338096] [SM Locating]: (0x15ec4d38) op-id: 6 [D 20:00:56.338121] client_op_state_get_machine 6 [D 20:00:56.338145] [SM Locating]: (0x15ec4d38) located: client_io_sm:init [D 20:00:56.338171] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.338197] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.338223] PINT_client_state_machine_post smcb 0x15ec4d38, op: PVFS_SYS_IO [D 20:00:56.338248] [SM Entering]: (0x15ec4d38) client_io_sm:init (status: 0) [D 20:00:56.338274] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.338298] (0x15ec4dd0) io state: io_init [D 20:00:56.338323] [SM Exiting]: (0x15ec4d38) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.338349] [SM push_state]: (0x15ec4d38) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.338375] [SM Entering]: (0x15ec4d38) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.338399] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.338424] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.338450] acache: get_cached_entry(): H=146 [D 20:00:56.338476] acache: miss non-static: H=146 [D 20:00:56.338501] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.338527] acache: acache hit [146] [D 20:00:56.338554] [SM Exiting]: (0x15ec4d38) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.338580] [SM Entering]: (0x15ec4d38) client_getattr_sm:cleanup (status: 0) [D 20:00:56.338605] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.338631] (0x15ec4dd0) getattr state: getattr_cleanup [D 20:00:56.338655] [SM Exiting]: (0x15ec4d38) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.338681] [SM pop_state]: (0x15ec4d38) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.338706] [SM Entering]: (0x15ec4d38) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.338731] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.338756] (0x15ec4dd0) io state: io_datafile_setup_msgpairs [D 20:00:56.338782] - io_find_target_datafiles called [D 20:00:56.338806] PINT_new_request_states n=1 [D 20:00:56.338830] PINT_new_request_states n=1 [D 20:00:56.338858] ========================================================= [D 20:00:56.338883] PINT_process_request [D 20:00:56.338907] tiling 8 copies [D 20:00:56.338931] skipping ahead to target_offset [D 20:00:56.338955] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.338981] lvl 0 el 0 blk 0 by 0 [D 20:00:56.339006] to 0 ta 201326592 fi 234881024 [D 20:00:56.339031] mto 0 mta 0 mfi 33554432 [D 20:00:56.339056] basic type or contiguous data [D 20:00:56.339079] Request disp [D 20:00:56.339103] contig_offset = 0 contig_size = 268435456 lvl_flag = 1 [D 20:00:56.339128] process logical skip [D 20:00:56.339152] retval = 201326592 [D 20:00:56.339176] exiting logical skip because distribute indicates done [D 20:00:56.339200] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.339226] lvl 0 el 0 blk 0 by 201326592 [D 20:00:56.339251] to 201326592 ta 201326592 fi 234881024 [D 20:00:56.339276] mto 0 mta 0 mfi 33554432 [D 20:00:56.339301] basic type or contiguous data [D 20:00:56.339324] Request disp [D 20:00:56.339348] contig_offset = 201326592 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.339374] calling distribute [D 20:00:56.339398] PINT_distribute [D 20:00:56.339422] of 201326592 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.339447] begin iteration loff: 201326592 [D 20:00:56.339472] segment exceeds byte limit [D 20:00:56.339496] file being extended [D 20:00:56.339520] client lstof 0 diff 0 sgof 0 [D 20:00:56.339545] **********CALL***PROCESS********* [D 20:00:56.339569] segment of 0 sz 8177 [D 20:00:56.339593] PINT_process_request [D 20:00:56.339617] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.339643] lvl 0 el 0 blk 0 by 0 [D 20:00:56.339672] to 0 ta 0 fi 8177 [D 20:00:56.339699] basic type or contiguous data [D 20:00:56.339727] Request disp [D 20:00:56.339754] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.339782] process a segment [D 20:00:56.339806] of 0 sz 8177 [D 20:00:56.339830] add a segment 0 [D 20:00:56.339855] retval = 8177 [D 20:00:56.339879] exiting distribute returned less than expected [D 20:00:56.339902] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.339928] *****RETURN***FROM***PROCESS***** [D 20:00:56.339953] end iteration [D 20:00:56.339977] done with segments or bytes [D 20:00:56.340001] of 201334769 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.340025] next loff: 201334769 next poff: 201334769 [D 20:00:56.340051] return value 8177 [D 20:00:56.340075] finished [D 20:00:56.340099] retval = 8177 [D 20:00:56.340124] exiting distribute returned less than expected [D 20:00:56.340147] done sg 1 sm 1 by 8177 bm 8177 ta 201326592 to 201334769 fo 234881024 eof 0 [D 20:00:56.340173] ========================================================= [D 20:00:56.340199] PINT_encode_calc_max_size [D 20:00:56.340224] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.340249] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.340277] initializing context[0] 0x15ec55a0 [D 20:00:56.340305] filling I/O request for 2863311531 [D 20:00:56.340330] [SM Exiting]: (0x15ec4d38) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.340355] [SM Entering]: (0x15ec4d38) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.340381] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.340407] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.340432] PINT_encode [D 20:00:56.340456] encode_common [D 20:00:56.340502] lebf_encode_req [D 20:00:56.340529] PINT_do_request_commit: commit node 0x15ec4ce0 [D 20:00:56.340554] node stored at 0 [D 20:00:56.340579] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.340603] node stored at 1 [D 20:00:56.340627] clearing tree [D 20:00:56.340654] PINT_encode_calc_max_size [D 20:00:56.340680] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.340706] BMI_post_recv: addr: 367403208, offset: 0x15ec7e00, size: 24, tag: 29 [D 20:00:56.340733] openib_check_cq: completed send wr_id 15e72e08. [D 20:00:56.340770] posting send with status_user_tag=1 [D 20:00:56.340798] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 29 [D 20:00:56.340824] element 0: offset: 0x15ec5cb0, size: 272 [D 20:00:56.340852] post_sr: da13:3346 bh 28 len 288 wr_id 15e72e20. [D 20:00:56.340882] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.340907] [SM Exiting]: (0x15ec4d38) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.340933] Posted PVFS_SYS_IO (85) (waiting for test)(-1073741839) -- buffer 0x15e328e8 curbuf 17 buffer_ahead_offset 7 size 33554432 read_ahead_offset 234881024 pvfs2_aio_fill buf 24 offset 234881024, size 33554432 [D 20:00:56.340968] PVFS_isys_io entered [146] [D 20:00:56.340994] [SM Frame PUSH]: (0x15ec8210) frame: 0x15ec82a8 [D 20:00:56.341020] [SM Locating]: (0x15ec8210) op-id: 6 [D 20:00:56.341045] client_op_state_get_machine 6 [D 20:00:56.341070] [SM Locating]: (0x15ec8210) located: client_io_sm:init [D 20:00:56.341094] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.341121] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.341147] PINT_client_state_machine_post smcb 0x15ec8210, op: PVFS_SYS_IO [D 20:00:56.341172] [SM Entering]: (0x15ec8210) client_io_sm:init (status: 0) [D 20:00:56.341197] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.341222] (0x15ec82a8) io state: io_init [D 20:00:56.341247] [SM Exiting]: (0x15ec8210) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.341273] [SM push_state]: (0x15ec8210) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.341298] [SM Entering]: (0x15ec8210) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.341323] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.341348] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.341373] acache: get_cached_entry(): H=146 [D 20:00:56.341399] acache: miss non-static: H=146 [D 20:00:56.341424] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.341449] acache: acache hit [146] [D 20:00:56.341474] [SM Exiting]: (0x15ec8210) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.341500] [SM Entering]: (0x15ec8210) client_getattr_sm:cleanup (status: 0) [D 20:00:56.341525] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.341550] (0x15ec82a8) getattr state: getattr_cleanup [D 20:00:56.341575] [SM Exiting]: (0x15ec8210) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.341600] [SM pop_state]: (0x15ec8210) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.341626] [SM Entering]: (0x15ec8210) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.341651] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.341679] (0x15ec82a8) io state: io_datafile_setup_msgpairs [D 20:00:56.341705] - io_find_target_datafiles called [D 20:00:56.341729] PINT_new_request_states n=1 [D 20:00:56.341753] PINT_new_request_states n=1 [D 20:00:56.341785] ========================================================= [D 20:00:56.341811] PINT_process_request [D 20:00:56.341835] tiling 9 copies [D 20:00:56.341859] skipping ahead to target_offset [D 20:00:56.341883] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.341909] lvl 0 el 0 blk 0 by 0 [D 20:00:56.341934] to 0 ta 234881024 fi 268435456 [D 20:00:56.341958] mto 0 mta 0 mfi 33554432 [D 20:00:56.341983] basic type or contiguous data [D 20:00:56.342007] Request disp [D 20:00:56.342031] contig_offset = 0 contig_size = 301989888 lvl_flag = 1 [D 20:00:56.342057] process logical skip [D 20:00:56.342081] retval = 234881024 [D 20:00:56.342104] exiting logical skip because distribute indicates done [D 20:00:56.342128] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.342154] lvl 0 el 0 blk 0 by 234881024 [D 20:00:56.342180] to 234881024 ta 234881024 fi 268435456 [D 20:00:56.342205] mto 0 mta 0 mfi 33554432 [D 20:00:56.342229] basic type or contiguous data [D 20:00:56.342253] Request disp [D 20:00:56.342277] contig_offset = 234881024 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.342302] calling distribute [D 20:00:56.342327] PINT_distribute [D 20:00:56.342350] of 234881024 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.342376] begin iteration loff: 234881024 [D 20:00:56.342401] segment exceeds byte limit [D 20:00:56.342426] file being extended [D 20:00:56.342450] client lstof 0 diff 0 sgof 0 [D 20:00:56.342473] **********CALL***PROCESS********* [D 20:00:56.342497] segment of 0 sz 8177 [D 20:00:56.342522] PINT_process_request [D 20:00:56.342546] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.342573] lvl 0 el 0 blk 0 by 0 [D 20:00:56.342597] to 0 ta 0 fi 8177 [D 20:00:56.342622] basic type or contiguous data [D 20:00:56.342646] Request disp [D 20:00:56.342670] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.342695] process a segment [D 20:00:56.342719] of 0 sz 8177 [D 20:00:56.342743] add a segment 0 [D 20:00:56.342767] retval = 8177 [D 20:00:56.342791] exiting distribute returned less than expected [D 20:00:56.342815] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.342841] *****RETURN***FROM***PROCESS***** [D 20:00:56.342865] end iteration [D 20:00:56.342888] done with segments or bytes [D 20:00:56.342912] of 234889201 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.342938] next loff: 234889201 next poff: 234889201 [D 20:00:56.342963] return value 8177 [D 20:00:56.342988] finished [D 20:00:56.343011] retval = 8177 [D 20:00:56.343035] exiting distribute returned less than expected [D 20:00:56.343060] done sg 1 sm 1 by 8177 bm 8177 ta 234881024 to 234889201 fo 268435456 eof 0 [D 20:00:56.343086] ========================================================= [D 20:00:56.343111] PINT_encode_calc_max_size [D 20:00:56.343135] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.343161] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.343192] initializing context[0] 0x15ec8a78 [D 20:00:56.343217] filling I/O request for 2863311531 [D 20:00:56.343243] [SM Exiting]: (0x15ec8210) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.343269] [SM Entering]: (0x15ec8210) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.343295] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.343320] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.343344] PINT_encode [D 20:00:56.343369] encode_common [D 20:00:56.343419] lebf_encode_req [D 20:00:56.343445] PINT_do_request_commit: commit node 0x15ec81b8 [D 20:00:56.343470] node stored at 0 [D 20:00:56.343495] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.343520] node stored at 1 [D 20:00:56.343544] clearing tree [D 20:00:56.343570] PINT_encode_calc_max_size [D 20:00:56.343596] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.343625] BMI_post_recv: addr: 367403208, offset: 0x15ecb2d8, size: 24, tag: 30 [D 20:00:56.343652] openib_check_cq: completed send wr_id 15e72e20. [D 20:00:56.343692] posting send with status_user_tag=1 [D 20:00:56.343723] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 30 [D 20:00:56.343752] element 0: offset: 0x15ec9188, size: 272 [D 20:00:56.343785] post_sr: da13:3346 bh 29 len 288 wr_id 15e72e38. [D 20:00:56.343813] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.343838] [SM Exiting]: (0x15ec8210) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.343866] Posted PVFS_SYS_IO (88) (waiting for test)(-1073741839) -- buffer 0x15e32910 curbuf 17 buffer_ahead_offset 8 size 33554432 read_ahead_offset 268435456 pvfs2_aio_fill buf 25 offset 268435456, size 33554432 [D 20:00:56.343900] PVFS_isys_io entered [146] [D 20:00:56.343927] [SM Frame PUSH]: (0x15ecb6e8) frame: 0x15ecb780 [D 20:00:56.343952] [SM Locating]: (0x15ecb6e8) op-id: 6 [D 20:00:56.343976] client_op_state_get_machine 6 [D 20:00:56.344001] [SM Locating]: (0x15ecb6e8) located: client_io_sm:init [D 20:00:56.344026] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.344053] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.344078] PINT_client_state_machine_post smcb 0x15ecb6e8, op: PVFS_SYS_IO [D 20:00:56.344102] [SM Entering]: (0x15ecb6e8) client_io_sm:init (status: 0) [D 20:00:56.344128] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.344153] (0x15ecb780) io state: io_init [D 20:00:56.344178] [SM Exiting]: (0x15ecb6e8) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.344209] [SM push_state]: (0x15ecb6e8) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.344234] [SM Entering]: (0x15ecb6e8) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.344260] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.344285] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.344309] acache: get_cached_entry(): H=146 [D 20:00:56.344336] acache: miss non-static: H=146 [D 20:00:56.344361] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.344387] acache: acache hit [146] [D 20:00:56.344410] [SM Exiting]: (0x15ecb6e8) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.344436] [SM Entering]: (0x15ecb6e8) client_getattr_sm:cleanup (status: 0) [D 20:00:56.344461] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.344487] (0x15ecb780) getattr state: getattr_cleanup [D 20:00:56.344511] [SM Exiting]: (0x15ecb6e8) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.344536] [SM pop_state]: (0x15ecb6e8) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.344562] [SM Entering]: (0x15ecb6e8) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.344588] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.344613] (0x15ecb780) io state: io_datafile_setup_msgpairs [D 20:00:56.344638] - io_find_target_datafiles called [D 20:00:56.344663] PINT_new_request_states n=1 [D 20:00:56.344688] PINT_new_request_states n=1 [D 20:00:56.344718] ========================================================= [D 20:00:56.344743] PINT_process_request [D 20:00:56.344768] tiling 10 copies [D 20:00:56.344792] skipping ahead to target_offset [D 20:00:56.344817] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.344842] lvl 0 el 0 blk 0 by 0 [D 20:00:56.344867] to 0 ta 268435456 fi 301989888 [D 20:00:56.344892] mto 0 mta 0 mfi 33554432 [D 20:00:56.344917] basic type or contiguous data [D 20:00:56.344941] Request disp [D 20:00:56.344965] contig_offset = 0 contig_size = 335544320 lvl_flag = 1 [D 20:00:56.344990] process logical skip [D 20:00:56.345014] retval = 268435456 [D 20:00:56.345039] exiting logical skip because distribute indicates done [D 20:00:56.345063] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.345088] lvl 0 el 0 blk 0 by 268435456 [D 20:00:56.345116] to 268435456 ta 268435456 fi 301989888 [D 20:00:56.345141] mto 0 mta 0 mfi 33554432 [D 20:00:56.345166] basic type or contiguous data [D 20:00:56.345190] Request disp [D 20:00:56.345213] contig_offset = 268435456 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.345238] calling distribute [D 20:00:56.345262] PINT_distribute [D 20:00:56.345286] of 268435456 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.345313] begin iteration loff: 268435456 [D 20:00:56.345337] segment exceeds byte limit [D 20:00:56.345361] file being extended [D 20:00:56.345385] client lstof 0 diff 0 sgof 0 [D 20:00:56.345410] **********CALL***PROCESS********* [D 20:00:56.345434] segment of 0 sz 8177 [D 20:00:56.345458] PINT_process_request [D 20:00:56.345481] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.345508] lvl 0 el 0 blk 0 by 0 [D 20:00:56.345533] to 0 ta 0 fi 8177 [D 20:00:56.345558] basic type or contiguous data [D 20:00:56.345582] Request disp [D 20:00:56.345605] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.345630] process a segment [D 20:00:56.345654] of 0 sz 8177 [D 20:00:56.345681] add a segment 0 [D 20:00:56.345705] retval = 8177 [D 20:00:56.345728] exiting distribute returned less than expected [D 20:00:56.345753] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.345778] *****RETURN***FROM***PROCESS***** [D 20:00:56.345803] end iteration [D 20:00:56.345827] done with segments or bytes [D 20:00:56.345850] of 268443633 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.345875] next loff: 268443633 next poff: 268443633 [D 20:00:56.345900] return value 8177 [D 20:00:56.345925] finished [D 20:00:56.345949] retval = 8177 [D 20:00:56.345973] exiting distribute returned less than expected [D 20:00:56.345996] done sg 1 sm 1 by 8177 bm 8177 ta 268435456 to 268443633 fo 301989888 eof 0 [D 20:00:56.346023] ========================================================= [D 20:00:56.346048] PINT_encode_calc_max_size [D 20:00:56.346073] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.346097] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.346125] initializing context[0] 0x15ecbf50 [D 20:00:56.346150] filling I/O request for 2863311531 [D 20:00:56.346177] [SM Exiting]: (0x15ecb6e8) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.346202] [SM Entering]: (0x15ecb6e8) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.346227] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.346253] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.346277] PINT_encode [D 20:00:56.346301] encode_common [D 20:00:56.346347] lebf_encode_req [D 20:00:56.346374] PINT_do_request_commit: commit node 0x15ecb690 [D 20:00:56.346399] node stored at 0 [D 20:00:56.346424] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.346448] node stored at 1 [D 20:00:56.346472] clearing tree [D 20:00:56.346500] PINT_encode_calc_max_size [D 20:00:56.346525] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.346551] BMI_post_recv: addr: 367403208, offset: 0x15ece7b0, size: 24, tag: 31 [D 20:00:56.346578] openib_check_cq: completed send wr_id 15e72e38. [D 20:00:56.346613] posting send with status_user_tag=1 [D 20:00:56.346640] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 31 [D 20:00:56.346666] element 0: offset: 0x15ecc660, size: 272 [D 20:00:56.346695] post_sr: da13:3346 bh 30 len 288 wr_id 15e72e50. [D 20:00:56.346724] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.346750] [SM Exiting]: (0x15ecb6e8) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.346776] Posted PVFS_SYS_IO (91) (waiting for test)(-1073741839) -- buffer 0x15e32938 curbuf 17 buffer_ahead_offset 9 size 33554432 read_ahead_offset 301989888 pvfs2_aio_fill buf 26 offset 301989888, size 33554432 [D 20:00:56.346811] PVFS_isys_io entered [146] [D 20:00:56.346843] [SM Frame PUSH]: (0x15ecebc0) frame: 0x15ecec58 [D 20:00:56.346869] [SM Locating]: (0x15ecebc0) op-id: 6 [D 20:00:56.346895] client_op_state_get_machine 6 [D 20:00:56.346919] [SM Locating]: (0x15ecebc0) located: client_io_sm:init [D 20:00:56.346944] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.346971] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.346996] PINT_client_state_machine_post smcb 0x15ecebc0, op: PVFS_SYS_IO [D 20:00:56.347022] [SM Entering]: (0x15ecebc0) client_io_sm:init (status: 0) [D 20:00:56.347047] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.347072] (0x15ecec58) io state: io_init [D 20:00:56.347097] [SM Exiting]: (0x15ecebc0) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.347122] [SM push_state]: (0x15ecebc0) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.347147] [SM Entering]: (0x15ecebc0) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.347173] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.347198] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.347223] acache: get_cached_entry(): H=146 [D 20:00:56.347249] acache: miss non-static: H=146 [D 20:00:56.347274] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.347300] acache: acache hit [146] [D 20:00:56.347324] [SM Exiting]: (0x15ecebc0) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.347350] [SM Entering]: (0x15ecebc0) client_getattr_sm:cleanup (status: 0) [D 20:00:56.347374] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.347399] (0x15ecec58) getattr state: getattr_cleanup [D 20:00:56.347424] [SM Exiting]: (0x15ecebc0) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.347450] [SM pop_state]: (0x15ecebc0) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.347476] [SM Entering]: (0x15ecebc0) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.347500] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.347525] (0x15ecec58) io state: io_datafile_setup_msgpairs [D 20:00:56.347551] - io_find_target_datafiles called [D 20:00:56.347576] PINT_new_request_states n=1 [D 20:00:56.347599] PINT_new_request_states n=1 [D 20:00:56.347627] ========================================================= [D 20:00:56.347654] PINT_process_request [D 20:00:56.347682] tiling 11 copies [D 20:00:56.347710] skipping ahead to target_offset [D 20:00:56.347738] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.347768] lvl 0 el 0 blk 0 by 0 [D 20:00:56.347793] to 0 ta 301989888 fi 335544320 [D 20:00:56.347818] mto 0 mta 0 mfi 33554432 [D 20:00:56.347843] basic type or contiguous data [D 20:00:56.347867] Request disp [D 20:00:56.347891] contig_offset = 0 contig_size = 369098752 lvl_flag = 1 [D 20:00:56.347916] process logical skip [D 20:00:56.347940] retval = 301989888 [D 20:00:56.347964] exiting logical skip because distribute indicates done [D 20:00:56.347989] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.348014] lvl 0 el 0 blk 0 by 301989888 [D 20:00:56.348039] to 301989888 ta 301989888 fi 335544320 [D 20:00:56.348064] mto 0 mta 0 mfi 33554432 [D 20:00:56.348089] basic type or contiguous data [D 20:00:56.348113] Request disp [D 20:00:56.348137] contig_offset = 301989888 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.348162] calling distribute [D 20:00:56.348186] PINT_distribute [D 20:00:56.348210] of 301989888 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.348237] begin iteration loff: 301989888 [D 20:00:56.348261] segment exceeds byte limit [D 20:00:56.348285] file being extended [D 20:00:56.348309] client lstof 0 diff 0 sgof 0 [D 20:00:56.348334] **********CALL***PROCESS********* [D 20:00:56.348358] segment of 0 sz 8177 [D 20:00:56.348381] PINT_process_request [D 20:00:56.348406] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.348432] lvl 0 el 0 blk 0 by 0 [D 20:00:56.348460] to 0 ta 0 fi 8177 [D 20:00:56.348484] basic type or contiguous data [D 20:00:56.348508] Request disp [D 20:00:56.348532] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.348558] process a segment [D 20:00:56.348582] of 0 sz 8177 [D 20:00:56.348606] add a segment 0 [D 20:00:56.348629] retval = 8177 [D 20:00:56.348653] exiting distribute returned less than expected [D 20:00:56.348678] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.348704] *****RETURN***FROM***PROCESS***** [D 20:00:56.348728] end iteration [D 20:00:56.348752] done with segments or bytes [D 20:00:56.348775] of 301998065 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.348801] next loff: 301998065 next poff: 301998065 [D 20:00:56.348826] return value 8177 [D 20:00:56.348851] finished [D 20:00:56.348875] retval = 8177 [D 20:00:56.348898] exiting distribute returned less than expected [D 20:00:56.348922] done sg 1 sm 1 by 8177 bm 8177 ta 301989888 to 301998065 fo 335544320 eof 0 [D 20:00:56.348949] ========================================================= [D 20:00:56.348974] PINT_encode_calc_max_size [D 20:00:56.348999] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.349023] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.349051] initializing context[0] 0x15ecf428 [D 20:00:56.349076] filling I/O request for 2863311531 [D 20:00:56.349102] [SM Exiting]: (0x15ecebc0) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.349127] [SM Entering]: (0x15ecebc0) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.349153] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.349178] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.349203] PINT_encode [D 20:00:56.349226] encode_common [D 20:00:56.349273] lebf_encode_req [D 20:00:56.349301] PINT_do_request_commit: commit node 0x15eceb68 [D 20:00:56.349326] node stored at 0 [D 20:00:56.349349] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.349374] node stored at 1 [D 20:00:56.349399] clearing tree [D 20:00:56.349426] PINT_encode_calc_max_size [D 20:00:56.349451] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.349478] BMI_post_recv: addr: 367403208, offset: 0x15ed1c88, size: 24, tag: 32 [D 20:00:56.349505] openib_check_cq: completed send wr_id 15e72e50. [D 20:00:56.349540] posting send with status_user_tag=1 [D 20:00:56.349567] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 32 [D 20:00:56.349593] element 0: offset: 0x15ecfb38, size: 272 [D 20:00:56.349621] post_sr: da13:3346 bh 31 len 288 wr_id 15e72e68. [D 20:00:56.349651] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.349678] [SM Exiting]: (0x15ecebc0) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.349705] Posted PVFS_SYS_IO (94) (waiting for test)(-1073741839) -- buffer 0x15e32960 curbuf 17 buffer_ahead_offset 10 size 33554432 read_ahead_offset 335544320 pvfs2_aio_fill buf 27 offset 335544320, size 33554432 [D 20:00:56.349742] PVFS_isys_io entered [146] [D 20:00:56.349770] [SM Frame PUSH]: (0x15ed2098) frame: 0x15ed2130 [D 20:00:56.349795] [SM Locating]: (0x15ed2098) op-id: 6 [D 20:00:56.349820] client_op_state_get_machine 6 [D 20:00:56.349844] [SM Locating]: (0x15ed2098) located: client_io_sm:init [D 20:00:56.349870] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.349897] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.349922] PINT_client_state_machine_post smcb 0x15ed2098, op: PVFS_SYS_IO [D 20:00:56.349947] [SM Entering]: (0x15ed2098) client_io_sm:init (status: 0) [D 20:00:56.349972] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.349997] (0x15ed2130) io state: io_init [D 20:00:56.350022] [SM Exiting]: (0x15ed2098) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.350048] [SM push_state]: (0x15ed2098) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.350073] [SM Entering]: (0x15ed2098) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.350100] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.350126] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.350152] acache: get_cached_entry(): H=146 [D 20:00:56.350177] acache: miss non-static: H=146 [D 20:00:56.350203] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.350229] acache: acache hit [146] [D 20:00:56.350254] [SM Exiting]: (0x15ed2098) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.350279] [SM Entering]: (0x15ed2098) client_getattr_sm:cleanup (status: 0) [D 20:00:56.350304] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.350330] (0x15ed2130) getattr state: getattr_cleanup [D 20:00:56.350354] [SM Exiting]: (0x15ed2098) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.350379] [SM pop_state]: (0x15ed2098) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.350405] [SM Entering]: (0x15ed2098) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.350430] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.350456] (0x15ed2130) io state: io_datafile_setup_msgpairs [D 20:00:56.350481] - io_find_target_datafiles called [D 20:00:56.350505] PINT_new_request_states n=1 [D 20:00:56.350530] PINT_new_request_states n=1 [D 20:00:56.350558] ========================================================= [D 20:00:56.350582] PINT_process_request [D 20:00:56.350606] tiling 12 copies [D 20:00:56.350630] skipping ahead to target_offset [D 20:00:56.350654] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.350681] lvl 0 el 0 blk 0 by 0 [D 20:00:56.350706] to 0 ta 335544320 fi 369098752 [D 20:00:56.350730] mto 0 mta 0 mfi 33554432 [D 20:00:56.350755] basic type or contiguous data [D 20:00:56.350779] Request disp [D 20:00:56.350803] contig_offset = 0 contig_size = 402653184 lvl_flag = 1 [D 20:00:56.350828] process logical skip [D 20:00:56.350852] retval = 335544320 [D 20:00:56.350876] exiting logical skip because distribute indicates done [D 20:00:56.350900] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.350926] lvl 0 el 0 blk 0 by 335544320 [D 20:00:56.350951] to 335544320 ta 335544320 fi 369098752 [D 20:00:56.350975] mto 0 mta 0 mfi 33554432 [D 20:00:56.351000] basic type or contiguous data [D 20:00:56.351024] Request disp [D 20:00:56.351048] contig_offset = 335544320 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.351074] calling distribute [D 20:00:56.351098] PINT_distribute [D 20:00:56.351121] of 335544320 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.351147] begin iteration loff: 335544320 [D 20:00:56.351172] segment exceeds byte limit [D 20:00:56.351196] file being extended [D 20:00:56.351220] client lstof 0 diff 0 sgof 0 [D 20:00:56.351244] **********CALL***PROCESS********* [D 20:00:56.351268] segment of 0 sz 8177 [D 20:00:56.351292] PINT_process_request [D 20:00:56.351317] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.351343] lvl 0 el 0 blk 0 by 0 [D 20:00:56.351367] to 0 ta 0 fi 8177 [D 20:00:56.351391] basic type or contiguous data [D 20:00:56.351415] Request disp [D 20:00:56.351439] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.351465] process a segment [D 20:00:56.351489] of 0 sz 8177 [D 20:00:56.351512] add a segment 0 [D 20:00:56.351536] retval = 8177 [D 20:00:56.351560] exiting distribute returned less than expected [D 20:00:56.351585] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.351610] *****RETURN***FROM***PROCESS***** [D 20:00:56.351634] end iteration [D 20:00:56.351659] done with segments or bytes [D 20:00:56.351688] of 335552497 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.351717] next loff: 335552497 next poff: 335552497 [D 20:00:56.351746] return value 8177 [D 20:00:56.351774] finished [D 20:00:56.351798] retval = 8177 [D 20:00:56.351821] exiting distribute returned less than expected [D 20:00:56.351848] done sg 1 sm 1 by 8177 bm 8177 ta 335544320 to 335552497 fo 369098752 eof 0 [D 20:00:56.351874] ========================================================= [D 20:00:56.351900] PINT_encode_calc_max_size [D 20:00:56.351924] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.351950] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.351981] initializing context[0] 0x15ed2900 [D 20:00:56.352006] filling I/O request for 2863311531 [D 20:00:56.352031] [SM Exiting]: (0x15ed2098) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.352057] [SM Entering]: (0x15ed2098) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.352083] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.352108] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.352133] PINT_encode [D 20:00:56.352157] encode_common [D 20:00:56.352204] lebf_encode_req [D 20:00:56.352231] PINT_do_request_commit: commit node 0x15ed2040 [D 20:00:56.352256] node stored at 0 [D 20:00:56.352280] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.352305] node stored at 1 [D 20:00:56.352329] clearing tree [D 20:00:56.352355] PINT_encode_calc_max_size [D 20:00:56.352381] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.352408] BMI_post_recv: addr: 367403208, offset: 0x15ed5160, size: 24, tag: 33 [D 20:00:56.352435] openib_check_cq: completed send wr_id 15e72e68. [D 20:00:56.352470] posting send with status_user_tag=1 [D 20:00:56.352497] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 33 [D 20:00:56.352522] element 0: offset: 0x15ed3010, size: 272 [D 20:00:56.352552] post_sr: da13:3346 bh 32 len 288 wr_id 15e72e80. [D 20:00:56.352581] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.352606] [SM Exiting]: (0x15ed2098) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.352633] Posted PVFS_SYS_IO (97) (waiting for test)(-1073741839) -- buffer 0x15e32988 curbuf 17 buffer_ahead_offset 11 size 33554432 read_ahead_offset 369098752 pvfs2_aio_fill buf 28 offset 369098752, size 33554432 [D 20:00:56.352667] PVFS_isys_io entered [146] [D 20:00:56.352694] [SM Frame PUSH]: (0x15ed5570) frame: 0x15ed5608 [D 20:00:56.352719] [SM Locating]: (0x15ed5570) op-id: 6 [D 20:00:56.352744] client_op_state_get_machine 6 [D 20:00:56.352768] [SM Locating]: (0x15ed5570) located: client_io_sm:init [D 20:00:56.352793] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.352820] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.352845] PINT_client_state_machine_post smcb 0x15ed5570, op: PVFS_SYS_IO [D 20:00:56.352870] [SM Entering]: (0x15ed5570) client_io_sm:init (status: 0) [D 20:00:56.352895] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.352921] (0x15ed5608) io state: io_init [D 20:00:56.352946] [SM Exiting]: (0x15ed5570) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.352972] [SM push_state]: (0x15ed5570) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.352996] [SM Entering]: (0x15ed5570) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.353022] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.353047] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.353072] acache: get_cached_entry(): H=146 [D 20:00:56.353097] acache: miss non-static: H=146 [D 20:00:56.353123] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.353148] acache: acache hit [146] [D 20:00:56.353173] [SM Exiting]: (0x15ed5570) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.353199] [SM Entering]: (0x15ed5570) client_getattr_sm:cleanup (status: 0) [D 20:00:56.353223] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.353248] (0x15ed5608) getattr state: getattr_cleanup [D 20:00:56.353273] [SM Exiting]: (0x15ed5570) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.353298] [SM pop_state]: (0x15ed5570) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.353326] [SM Entering]: (0x15ed5570) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.353352] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.353377] (0x15ed5608) io state: io_datafile_setup_msgpairs [D 20:00:56.353404] - io_find_target_datafiles called [D 20:00:56.353427] PINT_new_request_states n=1 [D 20:00:56.353452] PINT_new_request_states n=1 [D 20:00:56.353480] ========================================================= [D 20:00:56.353504] PINT_process_request [D 20:00:56.353528] tiling 13 copies [D 20:00:56.353552] skipping ahead to target_offset [D 20:00:56.353576] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.353602] lvl 0 el 0 blk 0 by 0 [D 20:00:56.353627] to 0 ta 369098752 fi 402653184 [D 20:00:56.353652] mto 0 mta 0 mfi 33554432 [D 20:00:56.353679] basic type or contiguous data [D 20:00:56.353704] Request disp [D 20:00:56.353728] contig_offset = 0 contig_size = 436207616 lvl_flag = 1 [D 20:00:56.353753] process logical skip [D 20:00:56.353776] retval = 369098752 [D 20:00:56.353800] exiting logical skip because distribute indicates done [D 20:00:56.353824] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.353851] lvl 0 el 0 blk 0 by 369098752 [D 20:00:56.353876] to 369098752 ta 369098752 fi 402653184 [D 20:00:56.353900] mto 0 mta 0 mfi 33554432 [D 20:00:56.353924] basic type or contiguous data [D 20:00:56.353949] Request disp [D 20:00:56.353973] contig_offset = 369098752 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.353998] calling distribute [D 20:00:56.354022] PINT_distribute [D 20:00:56.354045] of 369098752 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.354071] begin iteration loff: 369098752 [D 20:00:56.354097] segment exceeds byte limit [D 20:00:56.354121] file being extended [D 20:00:56.354145] client lstof 0 diff 0 sgof 0 [D 20:00:56.354170] **********CALL***PROCESS********* [D 20:00:56.354194] segment of 0 sz 8177 [D 20:00:56.354218] PINT_process_request [D 20:00:56.354242] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.354268] lvl 0 el 0 blk 0 by 0 [D 20:00:56.354292] to 0 ta 0 fi 8177 [D 20:00:56.354317] basic type or contiguous data [D 20:00:56.354341] Request disp [D 20:00:56.354365] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.354390] process a segment [D 20:00:56.354413] of 0 sz 8177 [D 20:00:56.354438] add a segment 0 [D 20:00:56.354462] retval = 8177 [D 20:00:56.354486] exiting distribute returned less than expected [D 20:00:56.354510] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.354536] *****RETURN***FROM***PROCESS***** [D 20:00:56.354559] end iteration [D 20:00:56.354583] done with segments or bytes [D 20:00:56.354607] of 369106929 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.354633] next loff: 369106929 next poff: 369106929 [D 20:00:56.354658] return value 8177 [D 20:00:56.354682] finished [D 20:00:56.354706] retval = 8177 [D 20:00:56.354730] exiting distribute returned less than expected [D 20:00:56.354754] done sg 1 sm 1 by 8177 bm 8177 ta 369098752 to 369106929 fo 402653184 eof 0 [D 20:00:56.354780] ========================================================= [D 20:00:56.354805] PINT_encode_calc_max_size [D 20:00:56.354829] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.354855] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.354887] initializing context[0] 0x15ed5dd8 [D 20:00:56.354912] filling I/O request for 2863311531 [D 20:00:56.354937] [SM Exiting]: (0x15ed5570) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.354963] [SM Entering]: (0x15ed5570) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.354989] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.355013] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.355038] PINT_encode [D 20:00:56.355065] encode_common [D 20:00:56.355111] lebf_encode_req [D 20:00:56.355138] PINT_do_request_commit: commit node 0x15ed5518 [D 20:00:56.355163] node stored at 0 [D 20:00:56.355187] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.355212] node stored at 1 [D 20:00:56.355236] clearing tree [D 20:00:56.355263] PINT_encode_calc_max_size [D 20:00:56.355288] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.355315] BMI_post_recv: addr: 367403208, offset: 0x15ed8638, size: 24, tag: 34 [D 20:00:56.355341] openib_check_cq: completed send wr_id 15e72e80. [D 20:00:56.355377] posting send with status_user_tag=1 [D 20:00:56.355403] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 34 [D 20:00:56.355429] element 0: offset: 0x15ed64e8, size: 272 [D 20:00:56.355458] post_sr: da13:3346 bh 33 len 288 wr_id 15e72e98. [D 20:00:56.355487] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.355512] [SM Exiting]: (0x15ed5570) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.355539] Posted PVFS_SYS_IO (100) (waiting for test)(-1073741839) -- buffer 0x15e329b0 curbuf 17 buffer_ahead_offset 12 size 33554432 read_ahead_offset 402653184 pvfs2_aio_fill buf 29 offset 402653184, size 33554432 [D 20:00:56.355573] PVFS_isys_io entered [146] [D 20:00:56.355605] [SM Frame PUSH]: (0x15ed8a48) frame: 0x15ed8ae0 [D 20:00:56.355630] [SM Locating]: (0x15ed8a48) op-id: 6 [D 20:00:56.355655] client_op_state_get_machine 6 [D 20:00:56.355686] [SM Locating]: (0x15ed8a48) located: client_io_sm:init [D 20:00:56.355714] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.355745] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.355773] PINT_client_state_machine_post smcb 0x15ed8a48, op: PVFS_SYS_IO [D 20:00:56.355799] [SM Entering]: (0x15ed8a48) client_io_sm:init (status: 0) [D 20:00:56.355824] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.355850] (0x15ed8ae0) io state: io_init [D 20:00:56.355875] [SM Exiting]: (0x15ed8a48) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.355900] [SM push_state]: (0x15ed8a48) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.355925] [SM Entering]: (0x15ed8a48) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.355951] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.355976] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.356000] acache: get_cached_entry(): H=146 [D 20:00:56.356027] acache: miss non-static: H=146 [D 20:00:56.356053] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.356078] acache: acache hit [146] [D 20:00:56.356101] [SM Exiting]: (0x15ed8a48) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.356127] [SM Entering]: (0x15ed8a48) client_getattr_sm:cleanup (status: 0) [D 20:00:56.356153] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.356178] (0x15ed8ae0) getattr state: getattr_cleanup [D 20:00:56.356203] [SM Exiting]: (0x15ed8a48) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.356228] [SM pop_state]: (0x15ed8a48) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.356253] [SM Entering]: (0x15ed8a48) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.356279] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.356304] (0x15ed8ae0) io state: io_datafile_setup_msgpairs [D 20:00:56.356330] - io_find_target_datafiles called [D 20:00:56.356354] PINT_new_request_states n=1 [D 20:00:56.356379] PINT_new_request_states n=1 [D 20:00:56.356407] ========================================================= [D 20:00:56.356431] PINT_process_request [D 20:00:56.356455] tiling 14 copies [D 20:00:56.356479] skipping ahead to target_offset [D 20:00:56.356503] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.356529] lvl 0 el 0 blk 0 by 0 [D 20:00:56.356554] to 0 ta 402653184 fi 436207616 [D 20:00:56.356578] mto 0 mta 0 mfi 33554432 [D 20:00:56.356603] basic type or contiguous data [D 20:00:56.356630] Request disp [D 20:00:56.356654] contig_offset = 0 contig_size = 469762048 lvl_flag = 1 [D 20:00:56.356678] process logical skip [D 20:00:56.356703] retval = 402653184 [D 20:00:56.356727] exiting logical skip because distribute indicates done [D 20:00:56.356751] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.356778] lvl 0 el 0 blk 0 by 402653184 [D 20:00:56.356802] to 402653184 ta 402653184 fi 436207616 [D 20:00:56.356826] mto 0 mta 0 mfi 33554432 [D 20:00:56.356851] basic type or contiguous data [D 20:00:56.356876] Request disp [D 20:00:56.356900] contig_offset = 402653184 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.356925] calling distribute [D 20:00:56.356948] PINT_distribute [D 20:00:56.356972] of 402653184 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.356999] begin iteration loff: 402653184 [D 20:00:56.357024] segment exceeds byte limit [D 20:00:56.357048] file being extended [D 20:00:56.357071] client lstof 0 diff 0 sgof 0 [D 20:00:56.357096] **********CALL***PROCESS********* [D 20:00:56.357120] segment of 0 sz 8177 [D 20:00:56.357144] PINT_process_request [D 20:00:56.357169] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.357194] lvl 0 el 0 blk 0 by 0 [D 20:00:56.357219] to 0 ta 0 fi 8177 [D 20:00:56.357244] basic type or contiguous data [D 20:00:56.357268] Request disp [D 20:00:56.357292] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.357316] process a segment [D 20:00:56.357340] of 0 sz 8177 [D 20:00:56.357365] add a segment 0 [D 20:00:56.357389] retval = 8177 [D 20:00:56.357413] exiting distribute returned less than expected [D 20:00:56.357437] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.357462] *****RETURN***FROM***PROCESS***** [D 20:00:56.357487] end iteration [D 20:00:56.357511] done with segments or bytes [D 20:00:56.357535] of 402661361 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.357561] next loff: 402661361 next poff: 402661361 [D 20:00:56.357585] return value 8177 [D 20:00:56.357609] finished [D 20:00:56.357636] retval = 8177 [D 20:00:56.357662] exiting distribute returned less than expected [D 20:00:56.357686] done sg 1 sm 1 by 8177 bm 8177 ta 402653184 to 402661361 fo 436207616 eof 0 [D 20:00:56.357713] ========================================================= [D 20:00:56.357738] PINT_encode_calc_max_size [D 20:00:56.357763] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.357788] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.357815] initializing context[0] 0x15ed92b0 [D 20:00:56.357841] filling I/O request for 2863311531 [D 20:00:56.357866] [SM Exiting]: (0x15ed8a48) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.357892] [SM Entering]: (0x15ed8a48) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.357916] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.357942] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.357967] PINT_encode [D 20:00:56.357991] encode_common [D 20:00:56.358037] lebf_encode_req [D 20:00:56.358064] PINT_do_request_commit: commit node 0x15ed89f0 [D 20:00:56.358089] node stored at 0 [D 20:00:56.358113] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.358138] node stored at 1 [D 20:00:56.358162] clearing tree [D 20:00:56.358189] PINT_encode_calc_max_size [D 20:00:56.358214] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.358242] BMI_post_recv: addr: 367403208, offset: 0x15edbb10, size: 24, tag: 35 [D 20:00:56.358268] openib_check_cq: completed send wr_id 15e72e98. [D 20:00:56.358304] posting send with status_user_tag=1 [D 20:00:56.358330] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 35 [D 20:00:56.358356] element 0: offset: 0x15ed99c0, size: 272 [D 20:00:56.358386] post_sr: da13:3346 bh 34 len 288 wr_id 15e72eb0. [D 20:00:56.358414] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.358446] [SM Exiting]: (0x15ed8a48) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.358472] Posted PVFS_SYS_IO (103) (waiting for test)(-1073741839) -- buffer 0x15e329d8 curbuf 17 buffer_ahead_offset 13 size 33554432 read_ahead_offset 436207616 pvfs2_aio_fill buf 30 offset 436207616, size 33554432 [D 20:00:56.358507] PVFS_isys_io entered [146] [D 20:00:56.358536] [SM Frame PUSH]: (0x15edbf20) frame: 0x15edbfb8 [D 20:00:56.358563] [SM Locating]: (0x15edbf20) op-id: 6 [D 20:00:56.358588] client_op_state_get_machine 6 [D 20:00:56.358612] [SM Locating]: (0x15edbf20) located: client_io_sm:init [D 20:00:56.358636] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.358664] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.358689] PINT_client_state_machine_post smcb 0x15edbf20, op: PVFS_SYS_IO [D 20:00:56.358715] [SM Entering]: (0x15edbf20) client_io_sm:init (status: 0) [D 20:00:56.358739] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.358765] (0x15edbfb8) io state: io_init [D 20:00:56.358790] [SM Exiting]: (0x15edbf20) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.358816] [SM push_state]: (0x15edbf20) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.358841] [SM Entering]: (0x15edbf20) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.358866] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.358891] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.358916] acache: get_cached_entry(): H=146 [D 20:00:56.358942] acache: miss non-static: H=146 [D 20:00:56.358968] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.358993] acache: acache hit [146] [D 20:00:56.359018] [SM Exiting]: (0x15edbf20) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.359044] [SM Entering]: (0x15edbf20) client_getattr_sm:cleanup (status: 0) [D 20:00:56.359068] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.359093] (0x15edbfb8) getattr state: getattr_cleanup [D 20:00:56.359118] [SM Exiting]: (0x15edbf20) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.359144] [SM pop_state]: (0x15edbf20) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.359169] [SM Entering]: (0x15edbf20) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.359194] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.359219] (0x15edbfb8) io state: io_datafile_setup_msgpairs [D 20:00:56.359246] - io_find_target_datafiles called [D 20:00:56.359270] PINT_new_request_states n=1 [D 20:00:56.359294] PINT_new_request_states n=1 [D 20:00:56.359321] ========================================================= [D 20:00:56.359346] PINT_process_request [D 20:00:56.359370] tiling 15 copies [D 20:00:56.359394] skipping ahead to target_offset [D 20:00:56.359418] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.359444] lvl 0 el 0 blk 0 by 0 [D 20:00:56.359469] to 0 ta 436207616 fi 469762048 [D 20:00:56.359494] mto 0 mta 0 mfi 33554432 [D 20:00:56.359519] basic type or contiguous data [D 20:00:56.359542] Request disp [D 20:00:56.359566] contig_offset = 0 contig_size = 503316480 lvl_flag = 1 [D 20:00:56.359591] process logical skip [D 20:00:56.359615] retval = 436207616 [D 20:00:56.359640] exiting logical skip because distribute indicates done [D 20:00:56.359666] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.359697] lvl 0 el 0 blk 0 by 436207616 [D 20:00:56.359725] to 436207616 ta 436207616 fi 469762048 [D 20:00:56.359753] mto 0 mta 0 mfi 33554432 [D 20:00:56.359781] basic type or contiguous data [D 20:00:56.359806] Request disp [D 20:00:56.359829] contig_offset = 436207616 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.359855] calling distribute [D 20:00:56.359879] PINT_distribute [D 20:00:56.359903] of 436207616 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.359930] begin iteration loff: 436207616 [D 20:00:56.359957] segment exceeds byte limit [D 20:00:56.359981] file being extended [D 20:00:56.360005] client lstof 0 diff 0 sgof 0 [D 20:00:56.360030] **********CALL***PROCESS********* [D 20:00:56.360054] segment of 0 sz 8177 [D 20:00:56.360078] PINT_process_request [D 20:00:56.360102] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.360129] lvl 0 el 0 blk 0 by 0 [D 20:00:56.360154] to 0 ta 0 fi 8177 [D 20:00:56.360177] basic type or contiguous data [D 20:00:56.360202] Request disp [D 20:00:56.360226] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.360251] process a segment [D 20:00:56.360275] of 0 sz 8177 [D 20:00:56.360299] add a segment 0 [D 20:00:56.360323] retval = 8177 [D 20:00:56.360347] exiting distribute returned less than expected [D 20:00:56.360371] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.360397] *****RETURN***FROM***PROCESS***** [D 20:00:56.360422] end iteration [D 20:00:56.360445] done with segments or bytes [D 20:00:56.360469] of 436215793 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.360495] next loff: 436215793 next poff: 436215793 [D 20:00:56.360520] return value 8177 [D 20:00:56.360544] finished [D 20:00:56.360569] retval = 8177 [D 20:00:56.360592] exiting distribute returned less than expected [D 20:00:56.360616] done sg 1 sm 1 by 8177 bm 8177 ta 436207616 to 436215793 fo 469762048 eof 0 [D 20:00:56.360643] ========================================================= [D 20:00:56.360668] PINT_encode_calc_max_size [D 20:00:56.360692] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.360717] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.360745] initializing context[0] 0x15edc788 [D 20:00:56.360771] filling I/O request for 2863311531 [D 20:00:56.360795] [SM Exiting]: (0x15edbf20) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.360821] [SM Entering]: (0x15edbf20) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.360847] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.360872] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.360897] PINT_encode [D 20:00:56.360920] encode_common [D 20:00:56.360967] lebf_encode_req [D 20:00:56.360998] PINT_do_request_commit: commit node 0x15edbec8 [D 20:00:56.361022] node stored at 0 [D 20:00:56.361046] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.361072] node stored at 1 [D 20:00:56.361096] clearing tree [D 20:00:56.361123] PINT_encode_calc_max_size [D 20:00:56.361147] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.361174] BMI_post_recv: addr: 367403208, offset: 0x15edefe8, size: 24, tag: 36 [D 20:00:56.361202] openib_check_cq: completed send wr_id 15e72eb0. [D 20:00:56.361237] posting send with status_user_tag=1 [D 20:00:56.361264] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 36 [D 20:00:56.361289] element 0: offset: 0x15edce98, size: 272 [D 20:00:56.361318] post_sr: da13:3346 bh 35 len 288 wr_id 15e72ec8. [D 20:00:56.361348] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.361372] [SM Exiting]: (0x15edbf20) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.361399] Posted PVFS_SYS_IO (106) (waiting for test)(-1073741839) -- buffer 0x15e32a00 curbuf 17 buffer_ahead_offset 14 size 33554432 read_ahead_offset 469762048 pvfs2_aio_fill buf 31 offset 469762048, size 33554432 [D 20:00:56.361433] PVFS_isys_io entered [146] [D 20:00:56.361460] [SM Frame PUSH]: (0x15edf3f8) frame: 0x15edf490 [D 20:00:56.361486] [SM Locating]: (0x15edf3f8) op-id: 6 [D 20:00:56.361511] client_op_state_get_machine 6 [D 20:00:56.361535] [SM Locating]: (0x15edf3f8) located: client_io_sm:init [D 20:00:56.361560] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.361587] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.361613] PINT_client_state_machine_post smcb 0x15edf3f8, op: PVFS_SYS_IO [D 20:00:56.361642] [SM Entering]: (0x15edf3f8) client_io_sm:init (status: 0) [D 20:00:56.361670] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.361696] (0x15edf490) io state: io_init [D 20:00:56.361721] [SM Exiting]: (0x15edf3f8) client_io_sm:init (error code: 0), (action: COMPLETE) [D 20:00:56.361746] [SM push_state]: (0x15edf3f8) op-id: 6 stk-ptr: 0 base-frm: 0 [D 20:00:56.361771] [SM Entering]: (0x15edf3f8) client_getattr_sm:acache_lookup (status: 0) [D 20:00:56.361796] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.361822] getattr_acache_lookup: handle 146 fsid 136 [D 20:00:56.361846] acache: get_cached_entry(): H=146 [D 20:00:56.361873] acache: miss non-static: H=146 [D 20:00:56.361899] acache: hit: H=146, size_status=-1073741858, attr_status=0 [D 20:00:56.361924] acache: acache hit [146] [D 20:00:56.361948] [SM Exiting]: (0x15edf3f8) client_getattr_sm:acache_lookup (error code: 0), (action: COMPLETE) [D 20:00:56.361974] [SM Entering]: (0x15edf3f8) client_getattr_sm:cleanup (status: 0) [D 20:00:56.361999] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.362024] (0x15edf490) getattr state: getattr_cleanup [D 20:00:56.362049] [SM Exiting]: (0x15edf3f8) client_getattr_sm:cleanup (error code: 0), (action: COMPLETE) [D 20:00:56.362074] [SM pop_state]: (0x15edf3f8) op-id: 6 stk-ptr: 1 base-frm: 0 [D 20:00:56.362100] [SM Entering]: (0x15edf3f8) client_io_sm:io_datafile_setup_msgpairs (status: 0) [D 20:00:56.362125] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.362150] (0x15edf490) io state: io_datafile_setup_msgpairs [D 20:00:56.362176] - io_find_target_datafiles called [D 20:00:56.362200] PINT_new_request_states n=1 [D 20:00:56.362225] PINT_new_request_states n=1 [D 20:00:56.362253] ========================================================= [D 20:00:56.362277] PINT_process_request [D 20:00:56.362301] tiling 16 copies [D 20:00:56.362325] skipping ahead to target_offset [D 20:00:56.362350] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.362376] lvl 0 el 0 blk 0 by 0 [D 20:00:56.362401] to 0 ta 469762048 fi 503316480 [D 20:00:56.362425] mto 0 mta 0 mfi 33554432 [D 20:00:56.362449] basic type or contiguous data [D 20:00:56.362473] Request disp [D 20:00:56.362498] contig_offset = 0 contig_size = 536870912 lvl_flag = 1 [D 20:00:56.362523] process logical skip [D 20:00:56.362546] retval = 469762048 [D 20:00:56.362570] exiting logical skip because distribute indicates done [D 20:00:56.362594] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.362621] lvl 0 el 0 blk 0 by 469762048 [D 20:00:56.362646] to 469762048 ta 469762048 fi 503316480 [D 20:00:56.362670] mto 0 mta 0 mfi 33554432 [D 20:00:56.362694] basic type or contiguous data [D 20:00:56.362719] Request disp [D 20:00:56.362742] contig_offset = 469762048 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.362768] calling distribute [D 20:00:56.362792] PINT_distribute [D 20:00:56.362815] of 469762048 sz 33554432 ix 0 sm 1 by 0 bm 8177 fsz 0 exfl 1 [D 20:00:56.362841] begin iteration loff: 469762048 [D 20:00:56.362866] segment exceeds byte limit [D 20:00:56.362891] file being extended [D 20:00:56.362915] client lstof 0 diff 0 sgof 0 [D 20:00:56.362939] **********CALL***PROCESS********* [D 20:00:56.362962] segment of 0 sz 8177 [D 20:00:56.362987] PINT_process_request [D 20:00:56.363011] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.363038] lvl 0 el 0 blk 0 by 0 [D 20:00:56.363061] to 0 ta 0 fi 8177 [D 20:00:56.363086] basic type or contiguous data [D 20:00:56.363110] Request disp [D 20:00:56.363134] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.363159] process a segment [D 20:00:56.363183] of 0 sz 8177 [D 20:00:56.363207] add a segment 0 [D 20:00:56.363231] retval = 8177 [D 20:00:56.363255] exiting distribute returned less than expected [D 20:00:56.363279] done sg 1 sm 1 by 8177 bm 8177 ta 0 to 8177 fo 8177 eof 0 [D 20:00:56.363307] *****RETURN***FROM***PROCESS***** [D 20:00:56.363332] end iteration [D 20:00:56.363356] done with segments or bytes [D 20:00:56.363380] of 469770225 sz 33546255 sg 1 sm 1 by 8177 bm 8177 [D 20:00:56.363406] next loff: 469770225 next poff: 469770225 [D 20:00:56.363430] return value 8177 [D 20:00:56.363454] finished [D 20:00:56.363479] retval = 8177 [D 20:00:56.363503] exiting distribute returned less than expected [D 20:00:56.363527] done sg 1 sm 1 by 8177 bm 8177 ta 469762048 to 469770225 fo 503316480 eof 0 [D 20:00:56.363554] ========================================================= [D 20:00:56.363578] PINT_encode_calc_max_size [D 20:00:56.363602] io_find_target_datafiles: datafile[0] might have data (out=1) [D 20:00:56.363628] io_datafile_setup_msgpairs: 1 datafiles might have data [D 20:00:56.363660] initializing context[0] 0x15edfc60 [D 20:00:56.363690] filling I/O request for 2863311531 [D 20:00:56.363719] [SM Exiting]: (0x15edf3f8) client_io_sm:io_datafile_setup_msgpairs (error code: 0), (action: COMPLETE) [D 20:00:56.363749] [SM Entering]: (0x15edf3f8) client_io_sm:io_datafile_post_msgpairs (status: 0) [D 20:00:56.363778] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.363803] io_datafile_post_msgpairs state: post (1 message(s)) [D 20:00:56.363828] PINT_encode [D 20:00:56.363852] encode_common [D 20:00:56.363899] lebf_encode_req [D 20:00:56.363925] PINT_do_request_commit: commit node 0x15edf3a0 [D 20:00:56.363950] node stored at 0 [D 20:00:56.363975] PINT_do_request_commit: commit node 0x112f0198 [D 20:00:56.364000] node stored at 1 [D 20:00:56.364024] clearing tree [D 20:00:56.364050] PINT_encode_calc_max_size [D 20:00:56.364075] posting recv with status_user_tag=0 (max_size 24) [D 20:00:56.364103] BMI_post_recv: addr: 367403208, offset: 0x15ee24c0, size: 24, tag: 37 [D 20:00:56.364130] openib_check_cq: completed send wr_id 15e72ec8. [D 20:00:56.364165] posting send with status_user_tag=1 [D 20:00:56.364192] BMI_post_sendunexpected_list: addr: 367403208, count: 1, total_size: 272, tag: 37 [D 20:00:56.364218] element 0: offset: 0x15ee0370, size: 272 [D 20:00:56.364247] post_sr: da13:3346 bh 36 len 288 wr_id 15e72ee0. [D 20:00:56.364275] io_datafile_post_msgpairs: completion count is 2 [D 20:00:56.364300] [SM Exiting]: (0x15edf3f8) client_io_sm:io_datafile_post_msgpairs (error code: 0), (action: DEFERRED) [D 20:00:56.364328] Posted PVFS_SYS_IO (109) (waiting for test)(-1073741839) -- buffer 0x15e32528 curbuf 17 buffer_ahead_offset 15 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 1 aio_req 0x15e61e30 b->offset 0 b->size 33554432 [D 20:00:56.364365] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.364393] PINT_client_state_machine_test id 22 [D 20:00:56.364482] sm control op 300 [D 20:00:56.364507] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:00:56.364534] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:00:56.364565] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:00:56.364592] sm control op 6 [D 20:00:56.364617] [SM Entering]: (0x15e75b78) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.364647] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.364673] (0x15e62108) io_datafile_complete_operations (tag 1) [D 20:00:56.364702] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.364727] - complete_context_send_or_recv called [D 20:00:56.364752] got a send completion with context index 0 [D 20:00:56.364777] matched send in context 0; continuing. [D 20:00:56.364805] [SM Exiting]: (0x15e75b78) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.364831] sm control op 6 [D 20:00:56.364856] [SM Entering]: (0x15e75b78) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.364881] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.364911] (0x15e62108) io_datafile_complete_operations (tag 0) [D 20:00:56.364936] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.364961] - complete_context_send_or_recv called [D 20:00:56.364986] got a recv completion with context index 0 [D 20:00:56.365011] - io_process_context_recv called [D 20:00:56.365036] - io_process_context_recv called [D 20:00:56.365061] PINT_decode [D 20:00:56.365087] lebf_decode_resp [D 20:00:56.365114] PINT_encode_release [D 20:00:56.365139] lebf_encode_rel [D 20:00:56.365164] PINT_decode_release [D 20:00:56.365188] lebf_decode_rel [D 20:00:56.365215] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.365241] io_post_flow: entry [D 20:00:56.365265] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.365291] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.369178] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.369215] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 0 [D 20:00:56.369245] PINT_new_request_states n=1 [D 20:00:56.369271] PINT_new_request_states n=1 [D 20:00:56.369297] flowproto posting 0x15e75278 [D 20:00:56.369340] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:56.369370] ========================================================= [D 20:00:56.369396] PINT_process_request [D 20:00:56.369421] tiling 2 copies [D 20:00:56.369444] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.369471] lvl 0 el 0 blk 0 by 0 [D 20:00:56.369496] to 0 ta 0 fi 33554432 [D 20:00:56.369521] mto 0 mta 0 mfi 33554432 [D 20:00:56.369544] basic type or contiguous data [D 20:00:56.369569] Request disp [D 20:00:56.369593] contig_offset = 0 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.369618] calling distribute [D 20:00:56.369643] PINT_distribute [D 20:00:56.369669] of 0 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.369695] begin iteration loff: 0 [D 20:00:56.369721] segment exceeds byte limit [D 20:00:56.369745] file being extended [D 20:00:56.369769] client lstof 0 diff 0 sgof 0 [D 20:00:56.369793] **********CALL***PROCESS********* [D 20:00:56.369817] segment of 0 sz 2097152 [D 20:00:56.369842] PINT_process_request [D 20:00:56.369866] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.369893] lvl 0 el 0 blk 0 by 0 [D 20:00:56.369916] to 0 ta 0 fi 2097152 [D 20:00:56.369941] basic type or contiguous data [D 20:00:56.369965] Request disp [D 20:00:56.369990] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.370015] process a segment [D 20:00:56.370038] of 0 sz 2097152 [D 20:00:56.370062] add a segment 0 [D 20:00:56.370086] retval = 2097152 [D 20:00:56.370111] exiting distribute returned less than expected [D 20:00:56.370135] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.370162] *****RETURN***FROM***PROCESS***** [D 20:00:56.370186] end iteration [D 20:00:56.370210] done with segments or bytes [D 20:00:56.370234] of 2097152 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.370260] next loff: 2097152 next poff: 2097152 [D 20:00:56.370284] return value 2097152 [D 20:00:56.370308] finished [D 20:00:56.370332] retval = 2097152 [D 20:00:56.370356] exiting distribute returned less than expected [D 20:00:56.370381] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 33554432 eof 0 [D 20:00:56.370407] ========================================================= [D 20:00:56.370431] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:56.370456] element 0: offset: 0xc2471000, size: 2097152 [D 20:00:56.370487] post_sr: da13:3346 bh 37 len 32 wr_id 15e72ef8. [D 20:00:56.370516] flowproto posted 0x15e75278 [D 20:00:56.370542] Job flows in progress (post time): 1 [D 20:00:56.370570] posted flow for context 0x15e74cc8 [D 20:00:56.370594] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.370620] preposting write ack for context 0x15e74cc8. [D 20:00:56.370649] PINT_encode_calc_max_size [D 20:00:56.370676] BMI_post_recv: addr: 367403208, offset: 0x15ec1750, size: 24, tag: 8 [D 20:00:56.370702] openib_check_cq: completed send wr_id 15e72ee0. [D 20:00:56.370728] openib_check_cq: completed send wr_id 15e72ef8. [D 20:00:56.370754] openib_check_cq: completed recv wr_id 15e736f8. [D 20:00:56.370783] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaaac6bb010 rkey 48002400 nic_wr_credit 1. [D 20:00:56.370811] openib_post_rr: da13:3346 bh 22 wr_id 15e736f8. [D 20:00:56.370841] *** all msgpairs complete (all flows posted) [D 20:00:56.370866] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.370891] [SM Exiting]: (0x15e75b78) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.370918] sm control op 6 [D 20:00:56.370942] [SM Entering]: (0x15e75f20) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.370967] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.370993] (0x15e75fb8) io_datafile_complete_operations (tag 1) [D 20:00:56.371018] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.371043] - complete_context_send_or_recv called [D 20:00:56.371068] got a send completion with context index 0 [D 20:00:56.371092] matched send in context 0; continuing. [D 20:00:56.371117] [SM Exiting]: (0x15e75f20) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.371143] sm control op 6 [D 20:00:56.371167] [SM Entering]: (0x15e75f20) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.371192] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.371218] (0x15e75fb8) io_datafile_complete_operations (tag 0) [D 20:00:56.371243] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.371268] - complete_context_send_or_recv called [D 20:00:56.371292] got a recv completion with context index 0 [D 20:00:56.371316] - io_process_context_recv called [D 20:00:56.371340] - io_process_context_recv called [D 20:00:56.371365] PINT_decode [D 20:00:56.371390] lebf_decode_resp [D 20:00:56.371414] PINT_encode_release [D 20:00:56.371438] lebf_encode_rel [D 20:00:56.371463] PINT_decode_release [D 20:00:56.371488] lebf_decode_rel [D 20:00:56.371514] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.371540] io_post_flow: entry [D 20:00:56.371563] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.371589] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.374873] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.374911] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 33554432 [D 20:00:56.374939] PINT_new_request_states n=1 [D 20:00:56.374966] PINT_new_request_states n=1 [D 20:00:56.374991] flowproto posting 0x15e76ce0 [D 20:00:56.375031] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:56.375058] ========================================================= [D 20:00:56.375082] PINT_process_request [D 20:00:56.375106] tiling 3 copies [D 20:00:56.375130] skipping ahead to target_offset [D 20:00:56.375155] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.375181] lvl 0 el 0 blk 0 by 0 [D 20:00:56.375205] to 0 ta 33554432 fi 67108864 [D 20:00:56.375230] mto 0 mta 0 mfi 33554432 [D 20:00:56.375255] basic type or contiguous data [D 20:00:56.375280] Request disp [D 20:00:56.375304] contig_offset = 0 contig_size = 100663296 lvl_flag = 1 [D 20:00:56.375329] process logical skip [D 20:00:56.375352] retval = 33554432 [D 20:00:56.375377] exiting logical skip because distribute indicates done [D 20:00:56.375401] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.375428] lvl 0 el 0 blk 0 by 33554432 [D 20:00:56.375452] to 33554432 ta 33554432 fi 67108864 [D 20:00:56.375477] mto 0 mta 0 mfi 33554432 [D 20:00:56.375501] basic type or contiguous data [D 20:00:56.375525] Request disp [D 20:00:56.375553] contig_offset = 33554432 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.375578] calling distribute [D 20:00:56.375602] PINT_distribute [D 20:00:56.375626] of 33554432 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.375653] begin iteration loff: 33554432 [D 20:00:56.375682] segment exceeds byte limit [D 20:00:56.375710] file being extended [D 20:00:56.375738] client lstof 0 diff 0 sgof 0 [D 20:00:56.375766] **********CALL***PROCESS********* [D 20:00:56.375791] segment of 0 sz 2097152 [D 20:00:56.375815] PINT_process_request [D 20:00:56.375839] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.375865] lvl 0 el 0 blk 0 by 0 [D 20:00:56.375890] to 0 ta 0 fi 2097152 [D 20:00:56.375914] basic type or contiguous data [D 20:00:56.375939] Request disp [D 20:00:56.375963] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.375988] process a segment [D 20:00:56.376011] of 0 sz 2097152 [D 20:00:56.376035] add a segment 0 [D 20:00:56.376059] retval = 2097152 [D 20:00:56.376084] exiting distribute returned less than expected [D 20:00:56.376108] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.376133] *****RETURN***FROM***PROCESS***** [D 20:00:56.376158] end iteration [D 20:00:56.376182] done with segments or bytes [D 20:00:56.376206] of 35651584 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.376232] next loff: 35651584 next poff: 35651584 [D 20:00:56.376256] return value 2097152 [D 20:00:56.376281] finished [D 20:00:56.376304] retval = 2097152 [D 20:00:56.376329] exiting distribute returned less than expected [D 20:00:56.376353] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 35651584 fo 67108864 eof 0 [D 20:00:56.376379] ========================================================= [D 20:00:56.376403] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:56.376429] element 0: offset: 0xc046f000, size: 2097152 [D 20:00:56.376459] post_sr: da13:3346 bh 38 len 32 wr_id 15e72f10. [D 20:00:56.376489] flowproto posted 0x15e76ce0 [D 20:00:56.376514] Job flows in progress (post time): 2 [D 20:00:56.376542] posted flow for context 0x15e76730 [D 20:00:56.376566] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.376592] preposting write ack for context 0x15e76730. [D 20:00:56.376616] PINT_encode_calc_max_size [D 20:00:56.376643] BMI_post_recv: addr: 367403208, offset: 0x15ea6210, size: 24, tag: 9 [D 20:00:56.376670] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 0. [D 20:00:56.376699] post_sr: da13:3346 bh 39 len 16 wr_id 15e72f28. [D 20:00:56.376727] openib_check_cq: completed send wr_id 15e72f10. [D 20:00:56.376751] openib_check_cq: completed recv wr_id 15e73710. [D 20:00:56.376780] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaaac8bc010 rkey 48002500 nic_wr_credit 1. [D 20:00:56.376807] openib_post_rr: da13:3346 bh 23 wr_id 15e73710. [D 20:00:56.376833] openib_check_cq: completed send wr_id 15e72f28. [D 20:00:56.376862] *** all msgpairs complete (all flows posted) [D 20:00:56.376886] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.376912] [SM Exiting]: (0x15e75f20) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.376938] sm control op 6 [D 20:00:56.376962] [SM Entering]: (0x15e85bb0) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.376988] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.377014] (0x15e85c48) io_datafile_complete_operations (tag 1) [D 20:00:56.377038] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.377063] - complete_context_send_or_recv called [D 20:00:56.377088] got a send completion with context index 0 [D 20:00:56.377113] matched send in context 0; continuing. [D 20:00:56.377138] [SM Exiting]: (0x15e85bb0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.377163] sm control op 6 [D 20:00:56.377191] [SM Entering]: (0x15e85bb0) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.377217] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.377242] (0x15e85c48) io_datafile_complete_operations (tag 0) [D 20:00:56.377267] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.377292] - complete_context_send_or_recv called [D 20:00:56.377317] got a recv completion with context index 0 [D 20:00:56.377342] - io_process_context_recv called [D 20:00:56.377366] - io_process_context_recv called [D 20:00:56.377389] PINT_decode [D 20:00:56.377414] lebf_decode_resp [D 20:00:56.377439] PINT_encode_release [D 20:00:56.377464] lebf_encode_rel [D 20:00:56.377489] PINT_decode_release [D 20:00:56.377512] lebf_decode_rel [D 20:00:56.377538] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.377564] io_post_flow: entry [D 20:00:56.377588] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.377614] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.380646] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.380683] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 67108864 [D 20:00:56.380713] PINT_new_request_states n=1 [D 20:00:56.380739] PINT_new_request_states n=1 [D 20:00:56.380764] flowproto posting 0x15e869b8 [D 20:00:56.380800] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:56.380826] ========================================================= [D 20:00:56.380851] PINT_process_request [D 20:00:56.380875] tiling 4 copies [D 20:00:56.380900] skipping ahead to target_offset [D 20:00:56.380924] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.380950] lvl 0 el 0 blk 0 by 0 [D 20:00:56.380975] to 0 ta 67108864 fi 100663296 [D 20:00:56.381000] mto 0 mta 0 mfi 33554432 [D 20:00:56.381024] basic type or contiguous data [D 20:00:56.381049] Request disp [D 20:00:56.381072] contig_offset = 0 contig_size = 134217728 lvl_flag = 1 [D 20:00:56.381097] process logical skip [D 20:00:56.381121] retval = 67108864 [D 20:00:56.381145] exiting logical skip because distribute indicates done [D 20:00:56.381170] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.381196] lvl 0 el 0 blk 0 by 67108864 [D 20:00:56.381220] to 67108864 ta 67108864 fi 100663296 [D 20:00:56.381245] mto 0 mta 0 mfi 33554432 [D 20:00:56.381270] basic type or contiguous data [D 20:00:56.381294] Request disp [D 20:00:56.381318] contig_offset = 67108864 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.381343] calling distribute [D 20:00:56.381367] PINT_distribute [D 20:00:56.381391] of 67108864 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.381418] begin iteration loff: 67108864 [D 20:00:56.381443] segment exceeds byte limit [D 20:00:56.381466] file being extended [D 20:00:56.381490] client lstof 0 diff 0 sgof 0 [D 20:00:56.381515] **********CALL***PROCESS********* [D 20:00:56.381539] segment of 0 sz 2097152 [D 20:00:56.381563] PINT_process_request [D 20:00:56.381587] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.381613] lvl 0 el 0 blk 0 by 0 [D 20:00:56.381638] to 0 ta 0 fi 2097152 [D 20:00:56.381666] basic type or contiguous data [D 20:00:56.381690] Request disp [D 20:00:56.381714] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.381739] process a segment [D 20:00:56.381763] of 0 sz 2097152 [D 20:00:56.381788] add a segment 0 [D 20:00:56.381812] retval = 2097152 [D 20:00:56.381835] exiting distribute returned less than expected [D 20:00:56.381860] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.381886] *****RETURN***FROM***PROCESS***** [D 20:00:56.381911] end iteration [D 20:00:56.381935] done with segments or bytes [D 20:00:56.381958] of 69206016 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.381984] next loff: 69206016 next poff: 69206016 [D 20:00:56.382013] return value 2097152 [D 20:00:56.382038] finished [D 20:00:56.382061] retval = 2097152 [D 20:00:56.382085] exiting distribute returned less than expected [D 20:00:56.382109] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 69206016 fo 100663296 eof 0 [D 20:00:56.382136] ========================================================= [D 20:00:56.382160] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:56.382185] element 0: offset: 0xbe46d000, size: 2097152 [D 20:00:56.382215] post_sr: da13:3346 bh 40 len 32 wr_id 15e72f40. [D 20:00:56.382246] flowproto posted 0x15e869b8 [D 20:00:56.382271] Job flows in progress (post time): 3 [D 20:00:56.382298] posted flow for context 0x15e86408 [D 20:00:56.382323] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.382349] preposting write ack for context 0x15e86408. [D 20:00:56.382373] PINT_encode_calc_max_size [D 20:00:56.382400] BMI_post_recv: addr: 367403208, offset: 0x15e9c628, size: 24, tag: 10 [D 20:00:56.382428] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 0. [D 20:00:56.382456] post_sr: da13:3346 bh 41 len 16 wr_id 15e72f58. [D 20:00:56.382483] openib_check_cq: completed send wr_id 15e72f40. [D 20:00:56.382509] openib_check_cq: completed recv wr_id 15e73728. [D 20:00:56.382536] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaaacabd010 rkey 48002600 nic_wr_credit 1. [D 20:00:56.382564] openib_post_rr: da13:3346 bh 24 wr_id 15e73728. [D 20:00:56.382591] openib_check_cq: completed send wr_id 15e72f58. [D 20:00:56.382619] *** all msgpairs complete (all flows posted) [D 20:00:56.382644] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.382669] [SM Exiting]: (0x15e85bb0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.382696] sm control op 6 [D 20:00:56.382720] [SM Entering]: (0x15e88fa0) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.382746] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.382771] (0x15e89038) io_datafile_complete_operations (tag 1) [D 20:00:56.382797] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.382822] - complete_context_send_or_recv called [D 20:00:56.382846] got a send completion with context index 0 [D 20:00:56.382871] matched send in context 0; continuing. [D 20:00:56.382895] [SM Exiting]: (0x15e88fa0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.382922] sm control op 6 [D 20:00:56.382946] [SM Entering]: (0x15e88fa0) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.382971] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.382997] (0x15e89038) io_datafile_complete_operations (tag 0) [D 20:00:56.383022] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.383047] - complete_context_send_or_recv called [D 20:00:56.383071] got a recv completion with context index 0 [D 20:00:56.383095] - io_process_context_recv called [D 20:00:56.383120] - io_process_context_recv called [D 20:00:56.383144] PINT_decode [D 20:00:56.383169] lebf_decode_resp [D 20:00:56.383193] PINT_encode_release [D 20:00:56.383217] lebf_encode_rel [D 20:00:56.383242] PINT_decode_release [D 20:00:56.383267] lebf_decode_rel [D 20:00:56.383293] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.383318] io_post_flow: entry [D 20:00:56.383342] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.383368] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.386288] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.386325] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 100663296 [D 20:00:56.386355] PINT_new_request_states n=1 [D 20:00:56.386381] PINT_new_request_states n=1 [D 20:00:56.386406] flowproto posting 0x15e89d98 [D 20:00:56.386442] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:56.386467] ========================================================= [D 20:00:56.386496] PINT_process_request [D 20:00:56.386521] tiling 5 copies [D 20:00:56.386546] skipping ahead to target_offset [D 20:00:56.386569] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.386595] lvl 0 el 0 blk 0 by 0 [D 20:00:56.386620] to 0 ta 100663296 fi 134217728 [D 20:00:56.386645] mto 0 mta 0 mfi 33554432 [D 20:00:56.386670] basic type or contiguous data [D 20:00:56.386693] Request disp [D 20:00:56.386717] contig_offset = 0 contig_size = 167772160 lvl_flag = 1 [D 20:00:56.386743] process logical skip [D 20:00:56.386767] retval = 100663296 [D 20:00:56.386791] exiting logical skip because distribute indicates done [D 20:00:56.386816] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.386841] lvl 0 el 0 blk 0 by 100663296 [D 20:00:56.386866] to 100663296 ta 100663296 fi 134217728 [D 20:00:56.386891] mto 0 mta 0 mfi 33554432 [D 20:00:56.386916] basic type or contiguous data [D 20:00:56.386940] Request disp [D 20:00:56.386963] contig_offset = 100663296 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.386988] calling distribute [D 20:00:56.387013] PINT_distribute [D 20:00:56.387037] of 100663296 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.387063] begin iteration loff: 100663296 [D 20:00:56.387088] segment exceeds byte limit [D 20:00:56.387112] file being extended [D 20:00:56.387136] client lstof 0 diff 0 sgof 0 [D 20:00:56.387161] **********CALL***PROCESS********* [D 20:00:56.387185] segment of 0 sz 2097152 [D 20:00:56.387209] PINT_process_request [D 20:00:56.387233] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.387259] lvl 0 el 0 blk 0 by 0 [D 20:00:56.387284] to 0 ta 0 fi 2097152 [D 20:00:56.387309] basic type or contiguous data [D 20:00:56.387332] Request disp [D 20:00:56.387356] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.387381] process a segment [D 20:00:56.387405] of 0 sz 2097152 [D 20:00:56.387430] add a segment 0 [D 20:00:56.387454] retval = 2097152 [D 20:00:56.387477] exiting distribute returned less than expected [D 20:00:56.387501] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.387528] *****RETURN***FROM***PROCESS***** [D 20:00:56.387552] end iteration [D 20:00:56.387576] done with segments or bytes [D 20:00:56.387599] of 102760448 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.387626] next loff: 102760448 next poff: 102760448 [D 20:00:56.387653] return value 2097152 [D 20:00:56.387681] finished [D 20:00:56.387709] retval = 2097152 [D 20:00:56.387737] exiting distribute returned less than expected [D 20:00:56.387765] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 102760448 fo 134217728 eof 0 [D 20:00:56.387792] ========================================================= [D 20:00:56.387817] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:56.387842] element 0: offset: 0xbc46b000, size: 2097152 [D 20:00:56.387872] post_sr: da13:3346 bh 42 len 32 wr_id 15e72f70. [D 20:00:56.387902] flowproto posted 0x15e89d98 [D 20:00:56.387928] Job flows in progress (post time): 4 [D 20:00:56.387954] posted flow for context 0x15e897e8 [D 20:00:56.387979] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.388005] preposting write ack for context 0x15e897e8. [D 20:00:56.388030] PINT_encode_calc_max_size [D 20:00:56.388056] BMI_post_recv: addr: 367403208, offset: 0x15e8f660, size: 24, tag: 11 [D 20:00:56.388083] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 0. [D 20:00:56.388112] post_sr: da13:3346 bh 43 len 16 wr_id 15e72f88. [D 20:00:56.388139] openib_check_cq: completed send wr_id 15e72f70. [D 20:00:56.388164] openib_check_cq: completed recv wr_id 15e73740. [D 20:00:56.388193] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaaaccbe010 rkey 48002700 nic_wr_credit 1. [D 20:00:56.388220] openib_post_rr: da13:3346 bh 25 wr_id 15e73740. [D 20:00:56.388249] openib_check_cq: completed send wr_id 15e72f88. [D 20:00:56.388278] *** all msgpairs complete (all flows posted) [D 20:00:56.388303] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.388328] [SM Exiting]: (0x15e88fa0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.388355] sm control op 6 [D 20:00:56.388380] [SM Entering]: (0x15e8c340) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.388405] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.388431] (0x15e8c3d8) io_datafile_complete_operations (tag 1) [D 20:00:56.388456] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.388482] - complete_context_send_or_recv called [D 20:00:56.388505] got a send completion with context index 0 [D 20:00:56.388530] matched send in context 0; continuing. [D 20:00:56.388555] [SM Exiting]: (0x15e8c340) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.388581] sm control op 6 [D 20:00:56.388606] [SM Entering]: (0x15e8c340) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.388631] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.388656] (0x15e8c3d8) io_datafile_complete_operations (tag 0) [D 20:00:56.388681] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.388706] - complete_context_send_or_recv called [D 20:00:56.388731] got a recv completion with context index 0 [D 20:00:56.388754] - io_process_context_recv called [D 20:00:56.388779] - io_process_context_recv called [D 20:00:56.388804] PINT_decode [D 20:00:56.388829] lebf_decode_resp [D 20:00:56.388854] PINT_encode_release [D 20:00:56.388877] lebf_encode_rel [D 20:00:56.388903] PINT_decode_release [D 20:00:56.388927] lebf_decode_rel [D 20:00:56.388953] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.388978] io_post_flow: entry [D 20:00:56.389003] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.389028] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.391849] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.391883] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 134217728 [D 20:00:56.391912] PINT_new_request_states n=1 [D 20:00:56.391939] PINT_new_request_states n=1 [D 20:00:56.391964] flowproto posting 0x15e8d190 [D 20:00:56.391999] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:56.392025] ========================================================= [D 20:00:56.392049] PINT_process_request [D 20:00:56.392074] tiling 6 copies [D 20:00:56.392099] skipping ahead to target_offset [D 20:00:56.392123] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.392149] lvl 0 el 0 blk 0 by 0 [D 20:00:56.392174] to 0 ta 134217728 fi 167772160 [D 20:00:56.392199] mto 0 mta 0 mfi 33554432 [D 20:00:56.392224] basic type or contiguous data [D 20:00:56.392248] Request disp [D 20:00:56.392271] contig_offset = 0 contig_size = 201326592 lvl_flag = 1 [D 20:00:56.392296] process logical skip [D 20:00:56.392320] retval = 134217728 [D 20:00:56.392345] exiting logical skip because distribute indicates done [D 20:00:56.392369] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.392395] lvl 0 el 0 blk 0 by 134217728 [D 20:00:56.392420] to 134217728 ta 134217728 fi 167772160 [D 20:00:56.392445] mto 0 mta 0 mfi 33554432 [D 20:00:56.392470] basic type or contiguous data [D 20:00:56.392494] Request disp [D 20:00:56.392518] contig_offset = 134217728 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.392542] calling distribute [D 20:00:56.392567] PINT_distribute [D 20:00:56.392591] of 134217728 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.392618] begin iteration loff: 134217728 [D 20:00:56.392642] segment exceeds byte limit [D 20:00:56.392667] file being extended [D 20:00:56.392691] client lstof 0 diff 0 sgof 0 [D 20:00:56.392716] **********CALL***PROCESS********* [D 20:00:56.392743] segment of 0 sz 2097152 [D 20:00:56.392767] PINT_process_request [D 20:00:56.392791] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.392817] lvl 0 el 0 blk 0 by 0 [D 20:00:56.392843] to 0 ta 0 fi 2097152 [D 20:00:56.392867] basic type or contiguous data [D 20:00:56.392890] Request disp [D 20:00:56.392914] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.392939] process a segment [D 20:00:56.392963] of 0 sz 2097152 [D 20:00:56.392988] add a segment 0 [D 20:00:56.393012] retval = 2097152 [D 20:00:56.393035] exiting distribute returned less than expected [D 20:00:56.393060] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.393086] *****RETURN***FROM***PROCESS***** [D 20:00:56.393111] end iteration [D 20:00:56.393135] done with segments or bytes [D 20:00:56.393158] of 136314880 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.393184] next loff: 136314880 next poff: 136314880 [D 20:00:56.393209] return value 2097152 [D 20:00:56.393233] finished [D 20:00:56.393257] retval = 2097152 [D 20:00:56.393281] exiting distribute returned less than expected [D 20:00:56.393305] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 136314880 fo 167772160 eof 0 [D 20:00:56.393331] ========================================================= [D 20:00:56.393356] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:56.393382] element 0: offset: 0xba469000, size: 2097152 [D 20:00:56.393411] post_sr: da13:3346 bh 44 len 32 wr_id 15e72fa0. [D 20:00:56.393441] flowproto posted 0x15e8d190 [D 20:00:56.393465] Job flows in progress (post time): 5 [D 20:00:56.393493] posted flow for context 0x15e8cbe0 [D 20:00:56.393518] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.393543] preposting write ack for context 0x15e8cbe0. [D 20:00:56.393568] PINT_encode_calc_max_size [D 20:00:56.393594] BMI_post_recv: addr: 367403208, offset: 0x15e77040, size: 24, tag: 12 [D 20:00:56.393622] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 0. [D 20:00:56.393651] post_sr: da13:3346 bh 45 len 16 wr_id 15e72fb8. [D 20:00:56.393680] openib_check_cq: completed send wr_id 15e72fa0. [D 20:00:56.393706] openib_check_cq: completed recv wr_id 15e73758. [D 20:00:56.393734] openib_post_sr_rdmaw: ibv_post_send wr_id 15ec8088 to da13:3346 remote addr 2aaaacebf010 rkey c8002800 nic_wr_credit 1. [D 20:00:56.393762] openib_post_rr: da13:3346 bh 26 wr_id 15e73758. [D 20:00:56.393789] openib_check_cq: completed send wr_id 15e72fb8. [D 20:00:56.393816] *** all msgpairs complete (all flows posted) [D 20:00:56.393841] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.393866] [SM Exiting]: (0x15e8c340) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.393893] sm control op 6 [D 20:00:56.393918] [SM Entering]: (0x15e8f790) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.393943] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.393968] (0x15e8f828) io_datafile_complete_operations (tag 1) [D 20:00:56.393994] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.394019] - complete_context_send_or_recv called [D 20:00:56.394042] got a send completion with context index 0 [D 20:00:56.394068] matched send in context 0; continuing. [D 20:00:56.394092] [SM Exiting]: (0x15e8f790) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.394118] sm control op 6 [D 20:00:56.394143] [SM Entering]: (0x15e8f790) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.394169] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.394194] (0x15e8f828) io_datafile_complete_operations (tag 0) [D 20:00:56.394220] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.394245] - complete_context_send_or_recv called [D 20:00:56.394268] got a recv completion with context index 0 [D 20:00:56.394295] - io_process_context_recv called [D 20:00:56.394320] - io_process_context_recv called [D 20:00:56.394345] PINT_decode [D 20:00:56.394370] lebf_decode_resp [D 20:00:56.394393] PINT_encode_release [D 20:00:56.394418] lebf_encode_rel [D 20:00:56.394443] PINT_decode_release [D 20:00:56.394467] lebf_decode_rel [D 20:00:56.394494] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.394518] io_post_flow: entry [D 20:00:56.394543] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.394568] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.397359] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.397394] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 167772160 [D 20:00:56.397422] PINT_new_request_states n=1 [D 20:00:56.397449] PINT_new_request_states n=1 [D 20:00:56.397475] flowproto posting 0x15e90588 [D 20:00:56.397513] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:56.397540] ========================================================= [D 20:00:56.397564] PINT_process_request [D 20:00:56.397589] tiling 7 copies [D 20:00:56.397613] skipping ahead to target_offset [D 20:00:56.397640] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.397669] lvl 0 el 0 blk 0 by 0 [D 20:00:56.397694] to 0 ta 167772160 fi 201326592 [D 20:00:56.397719] mto 0 mta 0 mfi 33554432 [D 20:00:56.397744] basic type or contiguous data [D 20:00:56.397768] Request disp [D 20:00:56.397792] contig_offset = 0 contig_size = 234881024 lvl_flag = 1 [D 20:00:56.397817] process logical skip [D 20:00:56.397841] retval = 167772160 [D 20:00:56.397865] exiting logical skip because distribute indicates done [D 20:00:56.397890] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.397915] lvl 0 el 0 blk 0 by 167772160 [D 20:00:56.397940] to 167772160 ta 167772160 fi 201326592 [D 20:00:56.397965] mto 0 mta 0 mfi 33554432 [D 20:00:56.397990] basic type or contiguous data [D 20:00:56.398014] Request disp [D 20:00:56.398037] contig_offset = 167772160 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.398063] calling distribute [D 20:00:56.398087] PINT_distribute [D 20:00:56.398111] of 167772160 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.398138] begin iteration loff: 167772160 [D 20:00:56.398162] segment exceeds byte limit [D 20:00:56.398187] file being extended [D 20:00:56.398210] client lstof 0 diff 0 sgof 0 [D 20:00:56.398235] **********CALL***PROCESS********* [D 20:00:56.398259] segment of 0 sz 2097152 [D 20:00:56.398283] PINT_process_request [D 20:00:56.398307] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.398333] lvl 0 el 0 blk 0 by 0 [D 20:00:56.398358] to 0 ta 0 fi 2097152 [D 20:00:56.398383] basic type or contiguous data [D 20:00:56.398406] Request disp [D 20:00:56.398430] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.398456] process a segment [D 20:00:56.398479] of 0 sz 2097152 [D 20:00:56.398504] add a segment 0 [D 20:00:56.398528] retval = 2097152 [D 20:00:56.398551] exiting distribute returned less than expected [D 20:00:56.398575] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.398601] *****RETURN***FROM***PROCESS***** [D 20:00:56.398626] end iteration [D 20:00:56.398650] done with segments or bytes [D 20:00:56.398674] of 169869312 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.398699] next loff: 169869312 next poff: 169869312 [D 20:00:56.398724] return value 2097152 [D 20:00:56.398748] finished [D 20:00:56.398772] retval = 2097152 [D 20:00:56.398797] exiting distribute returned less than expected [D 20:00:56.398820] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 169869312 fo 201326592 eof 0 [D 20:00:56.398846] ========================================================= [D 20:00:56.398871] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:56.398897] element 0: offset: 0xb8467000, size: 2097152 [D 20:00:56.398929] post_sr: da13:3346 bh 46 len 32 wr_id 15e72fd0. [D 20:00:56.398960] flowproto posted 0x15e90588 [D 20:00:56.398984] Job flows in progress (post time): 6 [D 20:00:56.399012] posted flow for context 0x15e8ffd8 [D 20:00:56.399037] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.399063] preposting write ack for context 0x15e8ffd8. [D 20:00:56.399086] PINT_encode_calc_max_size [D 20:00:56.399113] BMI_post_recv: addr: 367403208, offset: 0x15ec4c08, size: 24, tag: 13 [D 20:00:56.399141] openib_check_cq: completed rdma wr_id 15ec8088, nic_wr_credit 0. [D 20:00:56.399169] post_sr: da13:3346 bh 47 len 16 wr_id 15e72fe8. [D 20:00:56.399196] openib_check_cq: completed send wr_id 15e72fd0. [D 20:00:56.399222] openib_check_cq: completed recv wr_id 15e73770. [D 20:00:56.399250] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaaad0c0010 rkey c8002900 nic_wr_credit 1. [D 20:00:56.399278] openib_post_rr: da13:3346 bh 27 wr_id 15e73770. [D 20:00:56.399305] openib_check_cq: completed send wr_id 15e72fe8. [D 20:00:56.399333] *** all msgpairs complete (all flows posted) [D 20:00:56.399357] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.399383] [SM Exiting]: (0x15e8f790) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.399409] sm control op 6 [D 20:00:56.399434] [SM Entering]: (0x15e92b88) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.399459] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.399484] (0x15e92c20) io_datafile_complete_operations (tag 1) [D 20:00:56.399510] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.399536] - complete_context_send_or_recv called [D 20:00:56.399559] got a send completion with context index 0 [D 20:00:56.399584] matched send in context 0; continuing. [D 20:00:56.399609] [SM Exiting]: (0x15e92b88) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.399635] sm control op 6 [D 20:00:56.399661] [SM Entering]: (0x15e92b88) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.399691] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.399720] (0x15e92c20) io_datafile_complete_operations (tag 0) [D 20:00:56.399748] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.399777] - complete_context_send_or_recv called [D 20:00:56.399802] got a recv completion with context index 0 [D 20:00:56.399827] - io_process_context_recv called [D 20:00:56.399850] - io_process_context_recv called [D 20:00:56.399875] PINT_decode [D 20:00:56.399899] lebf_decode_resp [D 20:00:56.399924] PINT_encode_release [D 20:00:56.399949] lebf_encode_rel [D 20:00:56.399973] PINT_decode_release [D 20:00:56.399997] lebf_decode_rel [D 20:00:56.400023] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.400049] io_post_flow: entry [D 20:00:56.400073] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.400097] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.402883] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.402916] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 201326592 [D 20:00:56.402945] PINT_new_request_states n=1 [D 20:00:56.402971] PINT_new_request_states n=1 [D 20:00:56.402997] flowproto posting 0x15e93980 [D 20:00:56.403031] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:56.403058] ========================================================= [D 20:00:56.403081] PINT_process_request [D 20:00:56.403106] tiling 8 copies [D 20:00:56.403131] skipping ahead to target_offset [D 20:00:56.403155] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.403181] lvl 0 el 0 blk 0 by 0 [D 20:00:56.403205] to 0 ta 201326592 fi 234881024 [D 20:00:56.403230] mto 0 mta 0 mfi 33554432 [D 20:00:56.403255] basic type or contiguous data [D 20:00:56.403280] Request disp [D 20:00:56.403306] contig_offset = 0 contig_size = 268435456 lvl_flag = 1 [D 20:00:56.403332] process logical skip [D 20:00:56.403356] retval = 201326592 [D 20:00:56.403381] exiting logical skip because distribute indicates done [D 20:00:56.403405] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.403431] lvl 0 el 0 blk 0 by 201326592 [D 20:00:56.403456] to 201326592 ta 201326592 fi 234881024 [D 20:00:56.403481] mto 0 mta 0 mfi 33554432 [D 20:00:56.403505] basic type or contiguous data [D 20:00:56.403529] Request disp [D 20:00:56.403553] contig_offset = 201326592 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.403578] calling distribute [D 20:00:56.403602] PINT_distribute [D 20:00:56.403626] of 201326592 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.403653] begin iteration loff: 201326592 [D 20:00:56.403682] segment exceeds byte limit [D 20:00:56.403710] file being extended [D 20:00:56.403738] client lstof 0 diff 0 sgof 0 [D 20:00:56.403766] **********CALL***PROCESS********* [D 20:00:56.403791] segment of 0 sz 2097152 [D 20:00:56.403816] PINT_process_request [D 20:00:56.403840] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.403866] lvl 0 el 0 blk 0 by 0 [D 20:00:56.403891] to 0 ta 0 fi 2097152 [D 20:00:56.403915] basic type or contiguous data [D 20:00:56.403939] Request disp [D 20:00:56.403963] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.403988] process a segment [D 20:00:56.404012] of 0 sz 2097152 [D 20:00:56.404036] add a segment 0 [D 20:00:56.404060] retval = 2097152 [D 20:00:56.404084] exiting distribute returned less than expected [D 20:00:56.404109] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.404134] *****RETURN***FROM***PROCESS***** [D 20:00:56.404158] end iteration [D 20:00:56.404182] done with segments or bytes [D 20:00:56.404206] of 203423744 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.404232] next loff: 203423744 next poff: 203423744 [D 20:00:56.404256] return value 2097152 [D 20:00:56.404281] finished [D 20:00:56.404305] retval = 2097152 [D 20:00:56.404329] exiting distribute returned less than expected [D 20:00:56.404353] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 203423744 fo 234881024 eof 0 [D 20:00:56.404378] ========================================================= [D 20:00:56.404403] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:56.404428] element 0: offset: 0xb6465000, size: 2097152 [D 20:00:56.404458] post_sr: da13:3346 bh 48 len 32 wr_id 15e73000. [D 20:00:56.404487] flowproto posted 0x15e93980 [D 20:00:56.404512] Job flows in progress (post time): 7 [D 20:00:56.404540] posted flow for context 0x15e933d0 [D 20:00:56.404564] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.404589] preposting write ack for context 0x15e933d0. [D 20:00:56.404614] PINT_encode_calc_max_size [D 20:00:56.404641] BMI_post_recv: addr: 367403208, offset: 0x15e755d8, size: 24, tag: 14 [D 20:00:56.404667] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 0. [D 20:00:56.404696] post_sr: da13:3346 bh 49 len 16 wr_id 15e73018. [D 20:00:56.404723] openib_check_cq: completed send wr_id 15e73000. [D 20:00:56.404748] openib_check_cq: completed recv wr_id 15e73788. [D 20:00:56.404777] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaaad2c1010 rkey c8002a00 nic_wr_credit 1. [D 20:00:56.404805] openib_post_rr: da13:3346 bh 28 wr_id 15e73788. [D 20:00:56.404831] openib_check_cq: completed send wr_id 15e73018. [D 20:00:56.404859] *** all msgpairs complete (all flows posted) [D 20:00:56.404884] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.404909] [SM Exiting]: (0x15e92b88) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.404935] sm control op 6 [D 20:00:56.404960] [SM Entering]: (0x15e95f68) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.404988] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.405014] (0x15e96000) io_datafile_complete_operations (tag 1) [D 20:00:56.405039] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.405064] - complete_context_send_or_recv called [D 20:00:56.405088] got a send completion with context index 0 [D 20:00:56.405113] matched send in context 0; continuing. [D 20:00:56.405138] [SM Exiting]: (0x15e95f68) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.405163] sm control op 6 [D 20:00:56.405188] [SM Entering]: (0x15e95f68) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.405214] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.405239] (0x15e96000) io_datafile_complete_operations (tag 0) [D 20:00:56.405263] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.405289] - complete_context_send_or_recv called [D 20:00:56.405313] got a recv completion with context index 0 [D 20:00:56.405337] - io_process_context_recv called [D 20:00:56.405362] - io_process_context_recv called [D 20:00:56.405385] PINT_decode [D 20:00:56.405410] lebf_decode_resp [D 20:00:56.405435] PINT_encode_release [D 20:00:56.405459] lebf_encode_rel [D 20:00:56.405484] PINT_decode_release [D 20:00:56.405508] lebf_decode_rel [D 20:00:56.405534] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.405560] io_post_flow: entry [D 20:00:56.405584] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.405610] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.408386] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.408419] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 234881024 [D 20:00:56.408449] PINT_new_request_states n=1 [D 20:00:56.408474] PINT_new_request_states n=1 [D 20:00:56.408500] flowproto posting 0x15e96d60 [D 20:00:56.408538] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:56.408565] ========================================================= [D 20:00:56.408590] PINT_process_request [D 20:00:56.408615] tiling 9 copies [D 20:00:56.408638] skipping ahead to target_offset [D 20:00:56.408662] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.408689] lvl 0 el 0 blk 0 by 0 [D 20:00:56.408714] to 0 ta 234881024 fi 268435456 [D 20:00:56.408739] mto 0 mta 0 mfi 33554432 [D 20:00:56.408763] basic type or contiguous data [D 20:00:56.408787] Request disp [D 20:00:56.408811] contig_offset = 0 contig_size = 301989888 lvl_flag = 1 [D 20:00:56.408837] process logical skip [D 20:00:56.408861] retval = 234881024 [D 20:00:56.408885] exiting logical skip because distribute indicates done [D 20:00:56.408909] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.408935] lvl 0 el 0 blk 0 by 234881024 [D 20:00:56.408960] to 234881024 ta 234881024 fi 268435456 [D 20:00:56.408985] mto 0 mta 0 mfi 33554432 [D 20:00:56.409009] basic type or contiguous data [D 20:00:56.409033] Request disp [D 20:00:56.409057] contig_offset = 234881024 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.409083] calling distribute [D 20:00:56.409107] PINT_distribute [D 20:00:56.409131] of 234881024 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.409157] begin iteration loff: 234881024 [D 20:00:56.409182] segment exceeds byte limit [D 20:00:56.409206] file being extended [D 20:00:56.409230] client lstof 0 diff 0 sgof 0 [D 20:00:56.409255] **********CALL***PROCESS********* [D 20:00:56.409278] segment of 0 sz 2097152 [D 20:00:56.409303] PINT_process_request [D 20:00:56.409327] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.409353] lvl 0 el 0 blk 0 by 0 [D 20:00:56.409378] to 0 ta 0 fi 2097152 [D 20:00:56.409401] basic type or contiguous data [D 20:00:56.409425] Request disp [D 20:00:56.409449] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.409475] process a segment [D 20:00:56.409502] of 0 sz 2097152 [D 20:00:56.409526] add a segment 0 [D 20:00:56.409550] retval = 2097152 [D 20:00:56.409575] exiting distribute returned less than expected [D 20:00:56.409599] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.409626] *****RETURN***FROM***PROCESS***** [D 20:00:56.409649] end iteration [D 20:00:56.409676] done with segments or bytes [D 20:00:56.409700] of 236978176 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.409726] next loff: 236978176 next poff: 236978176 [D 20:00:56.409750] return value 2097152 [D 20:00:56.409775] finished [D 20:00:56.409799] retval = 2097152 [D 20:00:56.409823] exiting distribute returned less than expected [D 20:00:56.409847] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 236978176 fo 268435456 eof 0 [D 20:00:56.409873] ========================================================= [D 20:00:56.409897] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:56.409923] element 0: offset: 0xb4463000, size: 2097152 [D 20:00:56.409953] post_sr: da13:3346 bh 50 len 32 wr_id 15e73030. [D 20:00:56.409982] flowproto posted 0x15e96d60 [D 20:00:56.410007] Job flows in progress (post time): 8 [D 20:00:56.410034] posted flow for context 0x15e967b0 [D 20:00:56.410059] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.410085] preposting write ack for context 0x15e967b0. [D 20:00:56.410109] PINT_encode_calc_max_size [D 20:00:56.410136] BMI_post_recv: addr: 367403208, offset: 0x15e76fd0, size: 24, tag: 15 [D 20:00:56.410164] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 0. [D 20:00:56.410193] post_sr: da13:3346 bh 51 len 16 wr_id 15e73048. [D 20:00:56.410220] openib_check_cq: completed send wr_id 15e73030. [D 20:00:56.410245] openib_check_cq: completed recv wr_id 15e737a0. [D 20:00:56.410274] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaaad4c2010 rkey c8002b00 nic_wr_credit 1. [D 20:00:56.410302] openib_post_rr: da13:3346 bh 29 wr_id 15e737a0. [D 20:00:56.410328] openib_check_cq: completed send wr_id 15e73048. [D 20:00:56.410356] *** all msgpairs complete (all flows posted) [D 20:00:56.410381] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.410407] [SM Exiting]: (0x15e95f68) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.410432] sm control op 6 [D 20:00:56.410457] [SM Entering]: (0x15e99308) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.410483] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.410508] (0x15e993a0) io_datafile_complete_operations (tag 1) [D 20:00:56.410533] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.410559] - complete_context_send_or_recv called [D 20:00:56.410583] got a send completion with context index 0 [D 20:00:56.410608] matched send in context 0; continuing. [D 20:00:56.410633] [SM Exiting]: (0x15e99308) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.410659] sm control op 6 [D 20:00:56.410683] [SM Entering]: (0x15e99308) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.410709] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.410734] (0x15e993a0) io_datafile_complete_operations (tag 0) [D 20:00:56.410759] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.410784] - complete_context_send_or_recv called [D 20:00:56.410809] got a recv completion with context index 0 [D 20:00:56.410833] - io_process_context_recv called [D 20:00:56.410858] - io_process_context_recv called [D 20:00:56.410881] PINT_decode [D 20:00:56.410906] lebf_decode_resp [D 20:00:56.410930] PINT_encode_release [D 20:00:56.410955] lebf_encode_rel [D 20:00:56.410980] PINT_decode_release [D 20:00:56.411005] lebf_decode_rel [D 20:00:56.411030] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.411055] io_post_flow: entry [D 20:00:56.411080] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.411108] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.413858] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.413890] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 268435456 [D 20:00:56.413918] PINT_new_request_states n=1 [D 20:00:56.413945] PINT_new_request_states n=1 [D 20:00:56.413969] flowproto posting 0x15e9a158 [D 20:00:56.414006] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:56.414031] ========================================================= [D 20:00:56.414057] PINT_process_request [D 20:00:56.414081] tiling 10 copies [D 20:00:56.414106] skipping ahead to target_offset [D 20:00:56.414130] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.414155] lvl 0 el 0 blk 0 by 0 [D 20:00:56.414181] to 0 ta 268435456 fi 301989888 [D 20:00:56.414206] mto 0 mta 0 mfi 33554432 [D 20:00:56.414231] basic type or contiguous data [D 20:00:56.414255] Request disp [D 20:00:56.414278] contig_offset = 0 contig_size = 335544320 lvl_flag = 1 [D 20:00:56.414303] process logical skip [D 20:00:56.414328] retval = 268435456 [D 20:00:56.414352] exiting logical skip because distribute indicates done [D 20:00:56.414376] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.414402] lvl 0 el 0 blk 0 by 268435456 [D 20:00:56.414426] to 268435456 ta 268435456 fi 301989888 [D 20:00:56.414451] mto 0 mta 0 mfi 33554432 [D 20:00:56.414476] basic type or contiguous data [D 20:00:56.414500] Request disp [D 20:00:56.414525] contig_offset = 268435456 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.414549] calling distribute [D 20:00:56.414573] PINT_distribute [D 20:00:56.414597] of 268435456 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.414624] begin iteration loff: 268435456 [D 20:00:56.414649] segment exceeds byte limit [D 20:00:56.414672] file being extended [D 20:00:56.414696] client lstof 0 diff 0 sgof 0 [D 20:00:56.414721] **********CALL***PROCESS********* [D 20:00:56.414745] segment of 0 sz 2097152 [D 20:00:56.414769] PINT_process_request [D 20:00:56.414793] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.414818] lvl 0 el 0 blk 0 by 0 [D 20:00:56.414843] to 0 ta 0 fi 2097152 [D 20:00:56.414868] basic type or contiguous data [D 20:00:56.414892] Request disp [D 20:00:56.414916] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.414941] process a segment [D 20:00:56.414965] of 0 sz 2097152 [D 20:00:56.414989] add a segment 0 [D 20:00:56.415013] retval = 2097152 [D 20:00:56.415037] exiting distribute returned less than expected [D 20:00:56.415061] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.415087] *****RETURN***FROM***PROCESS***** [D 20:00:56.415111] end iteration [D 20:00:56.415135] done with segments or bytes [D 20:00:56.415159] of 270532608 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.415184] next loff: 270532608 next poff: 270532608 [D 20:00:56.415209] return value 2097152 [D 20:00:56.415233] finished [D 20:00:56.415257] retval = 2097152 [D 20:00:56.415281] exiting distribute returned less than expected [D 20:00:56.415305] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 270532608 fo 301989888 eof 0 [D 20:00:56.415331] ========================================================= [D 20:00:56.415356] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:56.415381] element 0: offset: 0xb2461000, size: 2097152 [D 20:00:56.415410] post_sr: da13:3346 bh 52 len 32 wr_id 15e73060. [D 20:00:56.415440] flowproto posted 0x15e9a158 [D 20:00:56.415465] Job flows in progress (post time): 9 [D 20:00:56.415493] posted flow for context 0x15e99ba8 [D 20:00:56.415517] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.415543] preposting write ack for context 0x15e99ba8. [D 20:00:56.415567] PINT_encode_calc_max_size [D 20:00:56.415599] BMI_post_recv: addr: 367403208, offset: 0x15e991e0, size: 24, tag: 16 [D 20:00:56.415626] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 0. [D 20:00:56.415656] post_sr: da13:3346 bh 53 len 16 wr_id 15e73078. [D 20:00:56.415688] openib_check_cq: completed send wr_id 15e73060. [D 20:00:56.415717] openib_check_cq: completed recv wr_id 15e737b8. [D 20:00:56.415749] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaaad6c3010 rkey c8002c00 nic_wr_credit 1. [D 20:00:56.415781] openib_post_rr: da13:3346 bh 30 wr_id 15e737b8. [D 20:00:56.415806] openib_check_cq: completed send wr_id 15e73078. [D 20:00:56.415835] *** all msgpairs complete (all flows posted) [D 20:00:56.415860] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.415884] [SM Exiting]: (0x15e99308) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.415910] sm control op 6 [D 20:00:56.415935] [SM Entering]: (0x15e9c758) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.415961] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.415986] (0x15e9c7f0) io_datafile_complete_operations (tag 1) [D 20:00:56.416011] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.416036] - complete_context_send_or_recv called [D 20:00:56.416061] got a send completion with context index 0 [D 20:00:56.416086] matched send in context 0; continuing. [D 20:00:56.416110] [SM Exiting]: (0x15e9c758) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.416136] sm control op 6 [D 20:00:56.416160] [SM Entering]: (0x15e9c758) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.416185] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.416211] (0x15e9c7f0) io_datafile_complete_operations (tag 0) [D 20:00:56.416235] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.416260] - complete_context_send_or_recv called [D 20:00:56.416285] got a recv completion with context index 0 [D 20:00:56.416310] - io_process_context_recv called [D 20:00:56.416334] - io_process_context_recv called [D 20:00:56.416357] PINT_decode [D 20:00:56.416382] lebf_decode_resp [D 20:00:56.416406] PINT_encode_release [D 20:00:56.416431] lebf_encode_rel [D 20:00:56.416456] PINT_decode_release [D 20:00:56.416480] lebf_decode_rel [D 20:00:56.416505] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.416531] io_post_flow: entry [D 20:00:56.416555] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.416580] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.419324] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.419357] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 301989888 [D 20:00:56.419385] PINT_new_request_states n=1 [D 20:00:56.419411] PINT_new_request_states n=1 [D 20:00:56.419442] flowproto posting 0x15e9d550 [D 20:00:56.419474] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:56.419500] ========================================================= [D 20:00:56.419525] PINT_process_request [D 20:00:56.419549] tiling 11 copies [D 20:00:56.419573] skipping ahead to target_offset [D 20:00:56.419597] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.419624] lvl 0 el 0 blk 0 by 0 [D 20:00:56.419648] to 0 ta 301989888 fi 335544320 [D 20:00:56.419677] mto 0 mta 0 mfi 33554432 [D 20:00:56.419705] basic type or contiguous data [D 20:00:56.419733] Request disp [D 20:00:56.419760] contig_offset = 0 contig_size = 369098752 lvl_flag = 1 [D 20:00:56.419788] process logical skip [D 20:00:56.419812] retval = 301989888 [D 20:00:56.419836] exiting logical skip because distribute indicates done [D 20:00:56.419860] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.419886] lvl 0 el 0 blk 0 by 301989888 [D 20:00:56.419911] to 301989888 ta 301989888 fi 335544320 [D 20:00:56.419939] mto 0 mta 0 mfi 33554432 [D 20:00:56.419965] basic type or contiguous data [D 20:00:56.419989] Request disp [D 20:00:56.420013] contig_offset = 301989888 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.420038] calling distribute [D 20:00:56.420063] PINT_distribute [D 20:00:56.420086] of 301989888 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.420113] begin iteration loff: 301989888 [D 20:00:56.420138] segment exceeds byte limit [D 20:00:56.420162] file being extended [D 20:00:56.420186] client lstof 0 diff 0 sgof 0 [D 20:00:56.420210] **********CALL***PROCESS********* [D 20:00:56.420234] segment of 0 sz 2097152 [D 20:00:56.420259] PINT_process_request [D 20:00:56.420283] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.420309] lvl 0 el 0 blk 0 by 0 [D 20:00:56.420333] to 0 ta 0 fi 2097152 [D 20:00:56.420358] basic type or contiguous data [D 20:00:56.420382] Request disp [D 20:00:56.420406] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.420431] process a segment [D 20:00:56.420454] of 0 sz 2097152 [D 20:00:56.420479] add a segment 0 [D 20:00:56.420503] retval = 2097152 [D 20:00:56.420527] exiting distribute returned less than expected [D 20:00:56.420551] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.420578] *****RETURN***FROM***PROCESS***** [D 20:00:56.420602] end iteration [D 20:00:56.420626] done with segments or bytes [D 20:00:56.420650] of 304087040 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.420676] next loff: 304087040 next poff: 304087040 [D 20:00:56.420701] return value 2097152 [D 20:00:56.420724] finished [D 20:00:56.420748] retval = 2097152 [D 20:00:56.420772] exiting distribute returned less than expected [D 20:00:56.420796] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 304087040 fo 335544320 eof 0 [D 20:00:56.420823] ========================================================= [D 20:00:56.420847] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:56.420872] element 0: offset: 0xb045f000, size: 2097152 [D 20:00:56.420902] post_sr: da13:3346 bh 54 len 32 wr_id 15e73090. [D 20:00:56.420931] flowproto posted 0x15e9d550 [D 20:00:56.420957] Job flows in progress (post time): 10 [D 20:00:56.420984] posted flow for context 0x15e9cfa0 [D 20:00:56.421009] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.421034] preposting write ack for context 0x15e9cfa0. [D 20:00:56.421059] PINT_encode_calc_max_size [D 20:00:56.421086] BMI_post_recv: addr: 367403208, offset: 0x15e75cb8, size: 24, tag: 17 [D 20:00:56.421113] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 0. [D 20:00:56.421141] post_sr: da13:3346 bh 55 len 16 wr_id 15e730a8. [D 20:00:56.421169] openib_check_cq: completed send wr_id 15e73090. [D 20:00:56.421194] openib_check_cq: completed recv wr_id 15e737d0. [D 20:00:56.421222] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaaad8c4010 rkey c8002d00 nic_wr_credit 1. [D 20:00:56.421250] openib_post_rr: da13:3346 bh 31 wr_id 15e737d0. [D 20:00:56.421277] openib_check_cq: completed send wr_id 15e730a8. [D 20:00:56.421305] *** all msgpairs complete (all flows posted) [D 20:00:56.421330] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.421356] [SM Exiting]: (0x15e9c758) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.421381] sm control op 6 [D 20:00:56.421406] [SM Entering]: (0x15e9fb50) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.421431] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.421457] (0x15e9fbe8) io_datafile_complete_operations (tag 1) [D 20:00:56.421483] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.421507] - complete_context_send_or_recv called [D 20:00:56.421531] got a send completion with context index 0 [D 20:00:56.421556] matched send in context 0; continuing. [D 20:00:56.421581] [SM Exiting]: (0x15e9fb50) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.421609] sm control op 6 [D 20:00:56.421634] [SM Entering]: (0x15e9fb50) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.421662] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.421688] (0x15e9fbe8) io_datafile_complete_operations (tag 0) [D 20:00:56.421712] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.421737] - complete_context_send_or_recv called [D 20:00:56.421762] got a recv completion with context index 0 [D 20:00:56.421786] - io_process_context_recv called [D 20:00:56.421811] - io_process_context_recv called [D 20:00:56.421834] PINT_decode [D 20:00:56.421859] lebf_decode_resp [D 20:00:56.421884] PINT_encode_release [D 20:00:56.421909] lebf_encode_rel [D 20:00:56.421934] PINT_decode_release [D 20:00:56.421957] lebf_decode_rel [D 20:00:56.421983] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.422009] io_post_flow: entry [D 20:00:56.422034] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.422059] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.424809] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.424840] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 335544320 [D 20:00:56.424870] PINT_new_request_states n=1 [D 20:00:56.424896] PINT_new_request_states n=1 [D 20:00:56.424921] flowproto posting 0x15ea0948 [D 20:00:56.424954] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:56.424981] ========================================================= [D 20:00:56.425004] PINT_process_request [D 20:00:56.425029] tiling 12 copies [D 20:00:56.425053] skipping ahead to target_offset [D 20:00:56.425077] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.425104] lvl 0 el 0 blk 0 by 0 [D 20:00:56.425128] to 0 ta 335544320 fi 369098752 [D 20:00:56.425153] mto 0 mta 0 mfi 33554432 [D 20:00:56.425178] basic type or contiguous data [D 20:00:56.425202] Request disp [D 20:00:56.425226] contig_offset = 0 contig_size = 402653184 lvl_flag = 1 [D 20:00:56.425251] process logical skip [D 20:00:56.425274] retval = 335544320 [D 20:00:56.425298] exiting logical skip because distribute indicates done [D 20:00:56.425323] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.425349] lvl 0 el 0 blk 0 by 335544320 [D 20:00:56.425374] to 335544320 ta 335544320 fi 369098752 [D 20:00:56.425398] mto 0 mta 0 mfi 33554432 [D 20:00:56.425423] basic type or contiguous data [D 20:00:56.425447] Request disp [D 20:00:56.425471] contig_offset = 335544320 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.425496] calling distribute [D 20:00:56.425519] PINT_distribute [D 20:00:56.425543] of 335544320 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.425570] begin iteration loff: 335544320 [D 20:00:56.425596] segment exceeds byte limit [D 20:00:56.425620] file being extended [D 20:00:56.425643] client lstof 0 diff 0 sgof 0 [D 20:00:56.425670] **********CALL***PROCESS********* [D 20:00:56.425695] segment of 0 sz 2097152 [D 20:00:56.425719] PINT_process_request [D 20:00:56.425743] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.425769] lvl 0 el 0 blk 0 by 0 [D 20:00:56.425794] to 0 ta 0 fi 2097152 [D 20:00:56.425819] basic type or contiguous data [D 20:00:56.425843] Request disp [D 20:00:56.425867] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.425891] process a segment [D 20:00:56.425914] of 0 sz 2097152 [D 20:00:56.425939] add a segment 0 [D 20:00:56.425963] retval = 2097152 [D 20:00:56.425987] exiting distribute returned less than expected [D 20:00:56.426011] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.426036] *****RETURN***FROM***PROCESS***** [D 20:00:56.426061] end iteration [D 20:00:56.426085] done with segments or bytes [D 20:00:56.426109] of 337641472 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.426139] next loff: 337641472 next poff: 337641472 [D 20:00:56.426165] return value 2097152 [D 20:00:56.426189] finished [D 20:00:56.426213] retval = 2097152 [D 20:00:56.426237] exiting distribute returned less than expected [D 20:00:56.426261] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 337641472 fo 369098752 eof 0 [D 20:00:56.426287] ========================================================= [D 20:00:56.426312] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:56.426338] element 0: offset: 0xae45d000, size: 2097152 [D 20:00:56.426369] post_sr: da13:3346 bh 56 len 32 wr_id 15e730c0. [D 20:00:56.426399] flowproto posted 0x15ea0948 [D 20:00:56.426424] Job flows in progress (post time): 11 [D 20:00:56.426451] posted flow for context 0x15ea0398 [D 20:00:56.426477] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.426503] preposting write ack for context 0x15ea0398. [D 20:00:56.426527] PINT_encode_calc_max_size [D 20:00:56.426553] BMI_post_recv: addr: 367403208, offset: 0x15efd0c0, size: 24, tag: 18 [D 20:00:56.426581] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 0. [D 20:00:56.426609] post_sr: da13:3346 bh 57 len 16 wr_id 15e730d8. [D 20:00:56.426636] openib_check_cq: completed send wr_id 15e730c0. [D 20:00:56.426662] openib_check_cq: completed recv wr_id 15e737e8. [D 20:00:56.426691] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaaadac5010 rkey c8002e00 nic_wr_credit 1. [D 20:00:56.426718] openib_post_rr: da13:3346 bh 32 wr_id 15e737e8. [D 20:00:56.426745] openib_check_cq: completed send wr_id 15e730d8. [D 20:00:56.426774] *** all msgpairs complete (all flows posted) [D 20:00:56.426798] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.426823] [SM Exiting]: (0x15e9fb50) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.426850] sm control op 6 [D 20:00:56.426874] [SM Entering]: (0x15ea2f48) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.426899] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.426925] (0x15ea2fe0) io_datafile_complete_operations (tag 1) [D 20:00:56.426950] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.426976] - complete_context_send_or_recv called [D 20:00:56.427000] got a send completion with context index 0 [D 20:00:56.427024] matched send in context 0; continuing. [D 20:00:56.427048] [SM Exiting]: (0x15ea2f48) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.427074] sm control op 6 [D 20:00:56.427099] [SM Entering]: (0x15ea2f48) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.427124] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.427149] (0x15ea2fe0) io_datafile_complete_operations (tag 0) [D 20:00:56.427174] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.427199] - complete_context_send_or_recv called [D 20:00:56.427223] got a recv completion with context index 0 [D 20:00:56.427248] - io_process_context_recv called [D 20:00:56.427272] - io_process_context_recv called [D 20:00:56.427296] PINT_decode [D 20:00:56.427321] lebf_decode_resp [D 20:00:56.427345] PINT_encode_release [D 20:00:56.427370] lebf_encode_rel [D 20:00:56.427394] PINT_decode_release [D 20:00:56.427418] lebf_decode_rel [D 20:00:56.427445] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.427470] io_post_flow: entry [D 20:00:56.427495] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.427519] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.430282] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.430315] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 369098752 [D 20:00:56.430344] PINT_new_request_states n=1 [D 20:00:56.430370] PINT_new_request_states n=1 [D 20:00:56.430395] flowproto posting 0x15ea3d40 [D 20:00:56.430432] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:56.430459] ========================================================= [D 20:00:56.430483] PINT_process_request [D 20:00:56.430507] tiling 13 copies [D 20:00:56.430532] skipping ahead to target_offset [D 20:00:56.430556] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.430583] lvl 0 el 0 blk 0 by 0 [D 20:00:56.430606] to 0 ta 369098752 fi 402653184 [D 20:00:56.430631] mto 0 mta 0 mfi 33554432 [D 20:00:56.430656] basic type or contiguous data [D 20:00:56.430680] Request disp [D 20:00:56.430704] contig_offset = 0 contig_size = 436207616 lvl_flag = 1 [D 20:00:56.430729] process logical skip [D 20:00:56.430753] retval = 369098752 [D 20:00:56.430777] exiting logical skip because distribute indicates done [D 20:00:56.430801] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.430828] lvl 0 el 0 blk 0 by 369098752 [D 20:00:56.430852] to 369098752 ta 369098752 fi 402653184 [D 20:00:56.430877] mto 0 mta 0 mfi 33554432 [D 20:00:56.430902] basic type or contiguous data [D 20:00:56.430926] Request disp [D 20:00:56.430950] contig_offset = 369098752 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.430974] calling distribute [D 20:00:56.430998] PINT_distribute [D 20:00:56.431023] of 369098752 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.431050] begin iteration loff: 369098752 [D 20:00:56.431075] segment exceeds byte limit [D 20:00:56.431098] file being extended [D 20:00:56.431122] client lstof 0 diff 0 sgof 0 [D 20:00:56.431147] **********CALL***PROCESS********* [D 20:00:56.431171] segment of 0 sz 2097152 [D 20:00:56.431196] PINT_process_request [D 20:00:56.431220] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.431245] lvl 0 el 0 blk 0 by 0 [D 20:00:56.431270] to 0 ta 0 fi 2097152 [D 20:00:56.431295] basic type or contiguous data [D 20:00:56.431319] Request disp [D 20:00:56.431343] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.431367] process a segment [D 20:00:56.431391] of 0 sz 2097152 [D 20:00:56.431415] add a segment 0 [D 20:00:56.431440] retval = 2097152 [D 20:00:56.431464] exiting distribute returned less than expected [D 20:00:56.431488] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.431513] *****RETURN***FROM***PROCESS***** [D 20:00:56.431538] end iteration [D 20:00:56.431562] done with segments or bytes [D 20:00:56.431586] of 371195904 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.431612] next loff: 371195904 next poff: 371195904 [D 20:00:56.431636] return value 2097152 [D 20:00:56.431663] finished [D 20:00:56.431691] retval = 2097152 [D 20:00:56.431718] exiting distribute returned less than expected [D 20:00:56.431746] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 371195904 fo 402653184 eof 0 [D 20:00:56.431776] ========================================================= [D 20:00:56.431800] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:56.431826] element 0: offset: 0xac45b000, size: 2097152 [D 20:00:56.431856] post_sr: da13:3346 bh 58 len 32 wr_id 15e730f0. [D 20:00:56.431885] flowproto posted 0x15ea3d40 [D 20:00:56.431910] Job flows in progress (post time): 12 [D 20:00:56.431937] posted flow for context 0x15ea3790 [D 20:00:56.431962] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.431987] preposting write ack for context 0x15ea3790. [D 20:00:56.432012] PINT_encode_calc_max_size [D 20:00:56.432039] BMI_post_recv: addr: 367403208, offset: 0x15effb58, size: 24, tag: 19 [D 20:00:56.432066] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 0. [D 20:00:56.432095] post_sr: da13:3346 bh 59 len 16 wr_id 15e73108. [D 20:00:56.432122] openib_check_cq: completed send wr_id 15e730f0. [D 20:00:56.432147] openib_check_cq: completed recv wr_id 15e73800. [D 20:00:56.432176] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaaadcc6010 rkey c8002f00 nic_wr_credit 1. [D 20:00:56.432206] openib_post_rr: da13:3346 bh 33 wr_id 15e73800. [D 20:00:56.432233] openib_check_cq: completed send wr_id 15e73108. [D 20:00:56.432261] *** all msgpairs complete (all flows posted) [D 20:00:56.432287] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.432312] [SM Exiting]: (0x15ea2f48) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.432338] sm control op 6 [D 20:00:56.432362] [SM Entering]: (0x15ea6340) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.432388] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.432414] (0x15ea63d8) io_datafile_complete_operations (tag 1) [D 20:00:56.432438] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.432464] - complete_context_send_or_recv called [D 20:00:56.432488] got a send completion with context index 0 [D 20:00:56.432514] matched send in context 0; continuing. [D 20:00:56.432538] [SM Exiting]: (0x15ea6340) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.432564] sm control op 6 [D 20:00:56.432588] [SM Entering]: (0x15ea6340) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.432614] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.432639] (0x15ea63d8) io_datafile_complete_operations (tag 0) [D 20:00:56.432663] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.432689] - complete_context_send_or_recv called [D 20:00:56.432713] got a recv completion with context index 0 [D 20:00:56.432738] - io_process_context_recv called [D 20:00:56.432762] - io_process_context_recv called [D 20:00:56.432786] PINT_decode [D 20:00:56.432810] lebf_decode_resp [D 20:00:56.432835] PINT_encode_release [D 20:00:56.432859] lebf_encode_rel [D 20:00:56.432884] PINT_decode_release [D 20:00:56.432909] lebf_decode_rel [D 20:00:56.432934] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.432959] io_post_flow: entry [D 20:00:56.432984] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.433009] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.435731] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.435769] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 402653184 [D 20:00:56.435797] PINT_new_request_states n=1 [D 20:00:56.435822] PINT_new_request_states n=1 [D 20:00:56.435847] flowproto posting 0x15ea7138 [D 20:00:56.435884] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:00:56.435910] ========================================================= [D 20:00:56.435935] PINT_process_request [D 20:00:56.435958] tiling 14 copies [D 20:00:56.435983] skipping ahead to target_offset [D 20:00:56.436007] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.436034] lvl 0 el 0 blk 0 by 0 [D 20:00:56.436058] to 0 ta 402653184 fi 436207616 [D 20:00:56.436082] mto 0 mta 0 mfi 33554432 [D 20:00:56.436107] basic type or contiguous data [D 20:00:56.436131] Request disp [D 20:00:56.436155] contig_offset = 0 contig_size = 469762048 lvl_flag = 1 [D 20:00:56.436180] process logical skip [D 20:00:56.436203] retval = 402653184 [D 20:00:56.436228] exiting logical skip because distribute indicates done [D 20:00:56.436252] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.436278] lvl 0 el 0 blk 0 by 402653184 [D 20:00:56.436303] to 402653184 ta 402653184 fi 436207616 [D 20:00:56.436327] mto 0 mta 0 mfi 33554432 [D 20:00:56.436352] basic type or contiguous data [D 20:00:56.436376] Request disp [D 20:00:56.436400] contig_offset = 402653184 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.436426] calling distribute [D 20:00:56.436450] PINT_distribute [D 20:00:56.436472] of 402653184 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.436499] begin iteration loff: 402653184 [D 20:00:56.436525] segment exceeds byte limit [D 20:00:56.436552] file being extended [D 20:00:56.436576] client lstof 0 diff 0 sgof 0 [D 20:00:56.436601] **********CALL***PROCESS********* [D 20:00:56.436625] segment of 0 sz 2097152 [D 20:00:56.436649] PINT_process_request [D 20:00:56.436674] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.436699] lvl 0 el 0 blk 0 by 0 [D 20:00:56.436724] to 0 ta 0 fi 2097152 [D 20:00:56.436749] basic type or contiguous data [D 20:00:56.436773] Request disp [D 20:00:56.436797] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.436821] process a segment [D 20:00:56.436845] of 0 sz 2097152 [D 20:00:56.436869] add a segment 0 [D 20:00:56.436894] retval = 2097152 [D 20:00:56.436918] exiting distribute returned less than expected [D 20:00:56.436942] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.436967] *****RETURN***FROM***PROCESS***** [D 20:00:56.436992] end iteration [D 20:00:56.437016] done with segments or bytes [D 20:00:56.437039] of 404750336 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.437066] next loff: 404750336 next poff: 404750336 [D 20:00:56.437090] return value 2097152 [D 20:00:56.437114] finished [D 20:00:56.437138] retval = 2097152 [D 20:00:56.437163] exiting distribute returned less than expected [D 20:00:56.437187] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 404750336 fo 436207616 eof 0 [D 20:00:56.437212] ========================================================= [D 20:00:56.437237] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:00:56.437262] element 0: offset: 0xaa459000, size: 2097152 [D 20:00:56.437292] post_sr: da13:3346 bh 60 len 32 wr_id 15e73120. [D 20:00:56.437321] flowproto posted 0x15ea7138 [D 20:00:56.437347] Job flows in progress (post time): 13 [D 20:00:56.437374] posted flow for context 0x15ea6b88 [D 20:00:56.437398] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.437424] preposting write ack for context 0x15ea6b88. [D 20:00:56.437449] PINT_encode_calc_max_size [D 20:00:56.437475] BMI_post_recv: addr: 367403208, offset: 0x15f025f0, size: 24, tag: 20 [D 20:00:56.437502] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 0. [D 20:00:56.437531] post_sr: da13:3346 bh 61 len 16 wr_id 15e73138. [D 20:00:56.437558] openib_check_cq: completed send wr_id 15e73120. [D 20:00:56.437583] openib_check_cq: completed recv wr_id 15e73818. [D 20:00:56.437612] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaaadec7010 rkey 48003000 nic_wr_credit 1. [D 20:00:56.437642] openib_post_rr: da13:3346 bh 34 wr_id 15e73818. [D 20:00:56.437670] openib_check_cq: completed send wr_id 15e73138. [D 20:00:56.437700] *** all msgpairs complete (all flows posted) [D 20:00:56.437725] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.437749] [SM Exiting]: (0x15ea6340) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.437775] sm control op 6 [D 20:00:56.437800] [SM Entering]: (0x15ea9738) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.437826] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.437851] (0x15ea97d0) io_datafile_complete_operations (tag 1) [D 20:00:56.437876] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.437901] - complete_context_send_or_recv called [D 20:00:56.437926] got a send completion with context index 0 [D 20:00:56.437951] matched send in context 0; continuing. [D 20:00:56.437975] [SM Exiting]: (0x15ea9738) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.438001] sm control op 6 [D 20:00:56.438025] [SM Entering]: (0x15ea9738) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.438051] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.438076] (0x15ea97d0) io_datafile_complete_operations (tag 0) [D 20:00:56.438101] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.438129] - complete_context_send_or_recv called [D 20:00:56.438153] got a recv completion with context index 0 [D 20:00:56.438179] - io_process_context_recv called [D 20:00:56.438202] - io_process_context_recv called [D 20:00:56.438226] PINT_decode [D 20:00:56.438251] lebf_decode_resp [D 20:00:56.438276] PINT_encode_release [D 20:00:56.438300] lebf_encode_rel [D 20:00:56.438324] PINT_decode_release [D 20:00:56.438349] lebf_decode_rel [D 20:00:56.438375] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.438401] io_post_flow: entry [D 20:00:56.438425] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.438449] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.441152] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.441184] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 436207616 [D 20:00:56.441212] PINT_new_request_states n=1 [D 20:00:56.441238] PINT_new_request_states n=1 [D 20:00:56.441263] flowproto posting 0x15eaa530 [D 20:00:56.441300] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:00:56.441326] ========================================================= [D 20:00:56.441350] PINT_process_request [D 20:00:56.441374] tiling 15 copies [D 20:00:56.441398] skipping ahead to target_offset [D 20:00:56.441423] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.441449] lvl 0 el 0 blk 0 by 0 [D 20:00:56.441473] to 0 ta 436207616 fi 469762048 [D 20:00:56.441498] mto 0 mta 0 mfi 33554432 [D 20:00:56.441523] basic type or contiguous data [D 20:00:56.441547] Request disp [D 20:00:56.441571] contig_offset = 0 contig_size = 503316480 lvl_flag = 1 [D 20:00:56.441596] process logical skip [D 20:00:56.441620] retval = 436207616 [D 20:00:56.441644] exiting logical skip because distribute indicates done [D 20:00:56.441671] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.441697] lvl 0 el 0 blk 0 by 436207616 [D 20:00:56.441721] to 436207616 ta 436207616 fi 469762048 [D 20:00:56.441746] mto 0 mta 0 mfi 33554432 [D 20:00:56.441771] basic type or contiguous data [D 20:00:56.441796] Request disp [D 20:00:56.441820] contig_offset = 436207616 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.441845] calling distribute [D 20:00:56.441869] PINT_distribute [D 20:00:56.441893] of 436207616 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.441919] begin iteration loff: 436207616 [D 20:00:56.441945] segment exceeds byte limit [D 20:00:56.441968] file being extended [D 20:00:56.441992] client lstof 0 diff 0 sgof 0 [D 20:00:56.442017] **********CALL***PROCESS********* [D 20:00:56.442041] segment of 0 sz 2097152 [D 20:00:56.442065] PINT_process_request [D 20:00:56.442089] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.442115] lvl 0 el 0 blk 0 by 0 [D 20:00:56.442140] to 0 ta 0 fi 2097152 [D 20:00:56.442165] basic type or contiguous data [D 20:00:56.442189] Request disp [D 20:00:56.442213] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.442237] process a segment [D 20:00:56.442261] of 0 sz 2097152 [D 20:00:56.442285] add a segment 0 [D 20:00:56.442309] retval = 2097152 [D 20:00:56.442333] exiting distribute returned less than expected [D 20:00:56.442357] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.442383] *****RETURN***FROM***PROCESS***** [D 20:00:56.442407] end iteration [D 20:00:56.442431] done with segments or bytes [D 20:00:56.442455] of 438304768 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.442481] next loff: 438304768 next poff: 438304768 [D 20:00:56.442505] return value 2097152 [D 20:00:56.442530] finished [D 20:00:56.442554] retval = 2097152 [D 20:00:56.442578] exiting distribute returned less than expected [D 20:00:56.442602] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 438304768 fo 469762048 eof 0 [D 20:00:56.442628] ========================================================= [D 20:00:56.442656] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:00:56.442682] element 0: offset: 0xa8457000, size: 2097152 [D 20:00:56.442714] post_sr: da13:3346 bh 62 len 32 wr_id 15e73150. [D 20:00:56.442744] flowproto posted 0x15eaa530 [D 20:00:56.442769] Job flows in progress (post time): 14 [D 20:00:56.442795] posted flow for context 0x15ea9f80 [D 20:00:56.442820] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.442846] preposting write ack for context 0x15ea9f80. [D 20:00:56.442871] PINT_encode_calc_max_size [D 20:00:56.442897] BMI_post_recv: addr: 367403208, offset: 0x15f05088, size: 24, tag: 21 [D 20:00:56.442924] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 0. [D 20:00:56.442953] post_sr: da13:3346 bh 63 len 16 wr_id 15e73168. [D 20:00:56.442979] openib_check_cq: completed send wr_id 15e73150. [D 20:00:56.443005] openib_check_cq: completed recv wr_id 15e73830. [D 20:00:56.443035] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaaae0c8010 rkey 48003100 nic_wr_credit 1. [D 20:00:56.443062] openib_post_rr: da13:3346 bh 35 wr_id 15e73830. [D 20:00:56.443089] openib_check_cq: completed send wr_id 15e73168. [D 20:00:56.443118] *** all msgpairs complete (all flows posted) [D 20:00:56.443142] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.443167] [SM Exiting]: (0x15ea9738) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.443194] sm control op 6 [D 20:00:56.443219] [SM Entering]: (0x15eadd18) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.443244] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.443269] (0x15eaddb0) io_datafile_complete_operations (tag 1) [D 20:00:56.443294] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.443320] - complete_context_send_or_recv called [D 20:00:56.443344] got a send completion with context index 0 [D 20:00:56.443368] matched send in context 0; continuing. [D 20:00:56.443393] [SM Exiting]: (0x15eadd18) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.443419] sm control op 6 [D 20:00:56.443444] [SM Entering]: (0x15eb1140) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.443470] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.443494] (0x15eb11d8) io_datafile_complete_operations (tag 1) [D 20:00:56.443519] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.443545] - complete_context_send_or_recv called [D 20:00:56.443569] got a send completion with context index 0 [D 20:00:56.443593] matched send in context 0; continuing. [D 20:00:56.443617] [SM Exiting]: (0x15eb1140) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.443643] sm control op 6 [D 20:00:56.443672] [SM Entering]: (0x15eb45f8) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.443701] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.443730] (0x15eb4690) io_datafile_complete_operations (tag 1) [D 20:00:56.443758] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.443786] - complete_context_send_or_recv called [D 20:00:56.443810] got a send completion with context index 0 [D 20:00:56.443835] matched send in context 0; continuing. [D 20:00:56.443860] [SM Exiting]: (0x15eb45f8) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.443885] sm control op 6 [D 20:00:56.443910] [SM Entering]: (0x15eb7ad0) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.443936] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.443961] (0x15eb7b68) io_datafile_complete_operations (tag 1) [D 20:00:56.443985] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.444011] - complete_context_send_or_recv called [D 20:00:56.444035] got a send completion with context index 0 [D 20:00:56.444060] matched send in context 0; continuing. [D 20:00:56.444087] [SM Exiting]: (0x15eb7ad0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.444113] sm control op 6 [D 20:00:56.444138] [SM Entering]: (0x15ebafa8) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.444164] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.444189] (0x15ebb040) io_datafile_complete_operations (tag 1) [D 20:00:56.444213] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.444239] - complete_context_send_or_recv called [D 20:00:56.444263] got a send completion with context index 0 [D 20:00:56.444288] matched send in context 0; continuing. [D 20:00:56.444312] [SM Exiting]: (0x15ebafa8) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.444337] sm control op 6 [D 20:00:56.444361] [SM Entering]: (0x15eadd18) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:00:56.444387] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.444412] (0x15eaddb0) io_datafile_complete_operations (tag 0) [D 20:00:56.444437] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.444462] - complete_context_send_or_recv called [D 20:00:56.444486] got a recv completion with context index 0 [D 20:00:56.444511] - io_process_context_recv called [D 20:00:56.444535] - io_process_context_recv called [D 20:00:56.444560] PINT_decode [D 20:00:56.444583] lebf_decode_resp [D 20:00:56.444608] PINT_encode_release [D 20:00:56.444632] lebf_encode_rel [D 20:00:56.444658] PINT_decode_release [D 20:00:56.444682] lebf_decode_rel [D 20:00:56.444707] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.444733] io_post_flow: entry [D 20:00:56.444757] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:00:56.444783] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:00:56.449369] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:00:56.449411] bstream_size = 0, datafile nr=0, ct=1, file_req_off = 469762048 [D 20:00:56.449440] PINT_new_request_states n=1 [D 20:00:56.449465] PINT_new_request_states n=1 [D 20:00:56.449490] flowproto posting 0x15eaeb40 [D 20:00:56.449524] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:00:56.449551] ========================================================= [D 20:00:56.449576] PINT_process_request [D 20:00:56.449600] tiling 16 copies [D 20:00:56.449625] skipping ahead to target_offset [D 20:00:56.449648] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.449677] lvl 0 el 0 blk 0 by 0 [D 20:00:56.449702] to 0 ta 469762048 fi 503316480 [D 20:00:56.449727] mto 0 mta 0 mfi 33554432 [D 20:00:56.449751] basic type or contiguous data [D 20:00:56.449775] Request disp [D 20:00:56.449800] contig_offset = 0 contig_size = 536870912 lvl_flag = 1 [D 20:00:56.449825] process logical skip [D 20:00:56.449849] retval = 469762048 [D 20:00:56.449872] exiting logical skip because distribute indicates done [D 20:00:56.449896] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.449923] lvl 0 el 0 blk 0 by 469762048 [D 20:00:56.449948] to 469762048 ta 469762048 fi 503316480 [D 20:00:56.449973] mto 0 mta 0 mfi 33554432 [D 20:00:56.449996] basic type or contiguous data [D 20:00:56.450021] Request disp [D 20:00:56.450045] contig_offset = 469762048 contig_size = 67108864 lvl_flag = 1 [D 20:00:56.450070] calling distribute [D 20:00:56.450094] PINT_distribute [D 20:00:56.450118] of 469762048 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 0 exfl 1 [D 20:00:56.450144] begin iteration loff: 469762048 [D 20:00:56.450169] segment exceeds byte limit [D 20:00:56.450193] file being extended [D 20:00:56.450217] client lstof 0 diff 0 sgof 0 [D 20:00:56.450242] **********CALL***PROCESS********* [D 20:00:56.450265] segment of 0 sz 2097152 [D 20:00:56.450289] PINT_process_request [D 20:00:56.450314] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.450343] lvl 0 el 0 blk 0 by 0 [D 20:00:56.450368] to 0 ta 0 fi 2097152 [D 20:00:56.450392] basic type or contiguous data [D 20:00:56.450416] Request disp [D 20:00:56.450440] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:00:56.450465] process a segment [D 20:00:56.450489] of 0 sz 2097152 [D 20:00:56.450513] add a segment 0 [D 20:00:56.450537] retval = 2097152 [D 20:00:56.450561] exiting distribute returned less than expected [D 20:00:56.450585] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:00:56.450611] *****RETURN***FROM***PROCESS***** [D 20:00:56.450634] end iteration [D 20:00:56.450658] done with segments or bytes [D 20:00:56.450682] of 471859200 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.450708] next loff: 471859200 next poff: 471859200 [D 20:00:56.450733] return value 2097152 [D 20:00:56.450757] finished [D 20:00:56.450780] retval = 2097152 [D 20:00:56.450804] exiting distribute returned less than expected [D 20:00:56.450828] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 471859200 fo 503316480 eof 0 [D 20:00:56.450855] ========================================================= [D 20:00:56.450879] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:00:56.450903] element 0: offset: 0xa6455000, size: 2097152 [D 20:00:56.450933] post_sr: da13:3346 bh 64 len 32 wr_id 15e73180. [D 20:00:56.450963] flowproto posted 0x15eaeb40 [D 20:00:56.450989] Job flows in progress (post time): 15 [D 20:00:56.451016] posted flow for context 0x15eae590 [D 20:00:56.451041] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.451066] preposting write ack for context 0x15eae590. [D 20:00:56.451091] PINT_encode_calc_max_size [D 20:00:56.451117] BMI_post_recv: addr: 367403208, offset: 0x15f07b20, size: 24, tag: 22 [D 20:00:56.451145] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 0. [D 20:00:56.451173] post_sr: da13:3346 bh 65 len 16 wr_id 15e73198. [D 20:00:56.451200] openib_check_cq: completed send wr_id 15e73180. [D 20:00:56.451226] openib_check_cq: completed send wr_id 15e73198. [D 20:00:56.451254] *** all msgpairs complete (all flows posted) [D 20:00:56.451279] 1 flows pending, 1 write acks pending, 0 msgpair [D 20:00:56.451304] [SM Exiting]: (0x15eadd18) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.451331] sm control op 6 [D 20:00:56.451355] [SM Entering]: (0x15ebe480) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.451381] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.451406] (0x15ebe518) io_datafile_complete_operations (tag 1) [D 20:00:56.451431] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.451456] - complete_context_send_or_recv called [D 20:00:56.451480] got a send completion with context index 0 [D 20:00:56.451504] matched send in context 0; continuing. [D 20:00:56.451529] [SM Exiting]: (0x15ebe480) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.451556] sm control op 6 [D 20:00:56.451580] [SM Entering]: (0x15ec1880) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.451605] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.451630] (0x15ec1918) io_datafile_complete_operations (tag 1) [D 20:00:56.451655] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.451685] - complete_context_send_or_recv called [D 20:00:56.451713] got a send completion with context index 0 [D 20:00:56.451741] matched send in context 0; continuing. [D 20:00:56.451769] [SM Exiting]: (0x15ec1880) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.451795] sm control op 6 [D 20:00:56.451820] [SM Entering]: (0x15ec4d38) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.451845] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.451870] (0x15ec4dd0) io_datafile_complete_operations (tag 1) [D 20:00:56.451897] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.451923] - complete_context_send_or_recv called [D 20:00:56.451947] got a send completion with context index 0 [D 20:00:56.451972] matched send in context 0; continuing. [D 20:00:56.451995] [SM Exiting]: (0x15ec4d38) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.452021] sm control op 6 [D 20:00:56.452046] [SM Entering]: (0x15ec8210) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.452071] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.452096] (0x15ec82a8) io_datafile_complete_operations (tag 1) [D 20:00:56.452121] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.452146] - complete_context_send_or_recv called [D 20:00:56.452170] got a send completion with context index 0 [D 20:00:56.452194] matched send in context 0; continuing. [D 20:00:56.452219] [SM Exiting]: (0x15ec8210) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.452244] sm control op 6 [D 20:00:56.452268] [SM Entering]: (0x15ecb6e8) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.452294] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.452319] (0x15ecb780) io_datafile_complete_operations (tag 1) [D 20:00:56.452343] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.452368] - complete_context_send_or_recv called [D 20:00:56.452393] got a send completion with context index 0 [D 20:00:56.452417] matched send in context 0; continuing. [D 20:00:56.452441] [SM Exiting]: (0x15ecb6e8) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.452466] sm control op 6 [D 20:00:56.452491] [SM Entering]: (0x15ecebc0) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.452517] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.452542] (0x15ecec58) io_datafile_complete_operations (tag 1) [D 20:00:56.452567] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.452591] - complete_context_send_or_recv called [D 20:00:56.452616] got a send completion with context index 0 [D 20:00:56.452640] matched send in context 0; continuing. [D 20:00:56.452664] [SM Exiting]: (0x15ecebc0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.452690] sm control op 6 [D 20:00:56.452714] [SM Entering]: (0x15ed2098) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.452740] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.452765] (0x15ed2130) io_datafile_complete_operations (tag 1) [D 20:00:56.452790] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.452816] - complete_context_send_or_recv called [D 20:00:56.452839] got a send completion with context index 0 [D 20:00:56.452863] matched send in context 0; continuing. [D 20:00:56.452888] [SM Exiting]: (0x15ed2098) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.452914] sm control op 6 [D 20:00:56.452938] [SM Entering]: (0x15ed5570) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.452963] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.452988] (0x15ed5608) io_datafile_complete_operations (tag 1) [D 20:00:56.453013] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.453038] - complete_context_send_or_recv called [D 20:00:56.453062] got a send completion with context index 0 [D 20:00:56.453086] matched send in context 0; continuing. [D 20:00:56.453110] [SM Exiting]: (0x15ed5570) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.453136] sm control op 6 [D 20:00:56.453160] [SM Entering]: (0x15ed8a48) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.453186] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.453210] (0x15ed8ae0) io_datafile_complete_operations (tag 1) [D 20:00:56.453237] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.453263] - complete_context_send_or_recv called [D 20:00:56.453287] got a send completion with context index 0 [D 20:00:56.453311] matched send in context 0; continuing. [D 20:00:56.453335] [SM Exiting]: (0x15ed8a48) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.453361] sm control op 6 [D 20:00:56.453386] [SM Entering]: (0x15edbf20) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.453411] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.453436] (0x15edbfb8) io_datafile_complete_operations (tag 1) [D 20:00:56.453461] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.453486] - complete_context_send_or_recv called [D 20:00:56.453510] got a send completion with context index 0 [D 20:00:56.453535] matched send in context 0; continuing. [D 20:00:56.453558] [SM Exiting]: (0x15edbf20) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.453585] PINT_client_state_machine_test id 22 [D 20:00:56.453624] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:56.453650] ========================================================= [D 20:00:56.453677] PINT_process_request [D 20:00:56.453701] tiling 2 copies [D 20:00:56.453725] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.453751] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.453776] to 2097152 ta 0 fi 33554432 [D 20:00:56.453801] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.453824] basic type or contiguous data [D 20:00:56.453848] Request disp [D 20:00:56.453872] contig_offset = 2097152 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.453897] calling distribute [D 20:00:56.453921] PINT_distribute [D 20:00:56.453944] of 2097152 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 2097152 exfl 1 [D 20:00:56.453970] begin iteration loff: 2097152 [D 20:00:56.453995] segment exceeds byte limit [D 20:00:56.454019] file being extended [D 20:00:56.454043] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.454068] **********CALL***PROCESS********* [D 20:00:56.454092] segment of 2097152 sz 2097152 [D 20:00:56.454115] PINT_process_request [D 20:00:56.454139] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.454166] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.454190] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.454215] basic type or contiguous data [D 20:00:56.454238] Request disp [D 20:00:56.454262] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.454287] process a segment [D 20:00:56.454311] of 2097152 sz 2097152 [D 20:00:56.454335] add a segment 0 [D 20:00:56.454359] retval = 2097152 [D 20:00:56.454382] exiting distribute returned less than expected [D 20:00:56.454406] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.454433] *****RETURN***FROM***PROCESS***** [D 20:00:56.454457] end iteration [D 20:00:56.454481] done with segments or bytes [D 20:00:56.454504] of 4194304 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.454530] next loff: 4194304 next poff: 4194304 [D 20:00:56.454554] return value 2097152 [D 20:00:56.454579] finished [D 20:00:56.454603] retval = 2097152 [D 20:00:56.454627] exiting distribute returned less than expected [D 20:00:56.454650] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 4194304 fo 33554432 eof 0 [D 20:00:56.454676] ========================================================= [D 20:00:56.454700] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:56.454726] element 0: offset: 0xc2671000, size: 2097152 [D 20:00:56.454754] post_sr: da13:3346 bh 66 len 32 wr_id 15e731b0. [D 20:00:56.454783] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:56.454809] ========================================================= [D 20:00:56.454833] PINT_process_request [D 20:00:56.454860] tiling 3 copies [D 20:00:56.454884] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.454911] lvl 0 el 0 blk 0 by 35651584 [D 20:00:56.454936] to 35651584 ta 33554432 fi 67108864 [D 20:00:56.454960] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.454985] basic type or contiguous data [D 20:00:56.455009] Request disp [D 20:00:56.455033] contig_offset = 35651584 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.455058] calling distribute [D 20:00:56.455081] PINT_distribute [D 20:00:56.455105] of 35651584 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 35651584 exfl 1 [D 20:00:56.455131] begin iteration loff: 35651584 [D 20:00:56.455156] segment exceeds byte limit [D 20:00:56.455180] file being extended [D 20:00:56.455204] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.455228] **********CALL***PROCESS********* [D 20:00:56.455252] segment of 2097152 sz 2097152 [D 20:00:56.455277] PINT_process_request [D 20:00:56.455301] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.455327] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.455351] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.455376] basic type or contiguous data [D 20:00:56.455400] Request disp [D 20:00:56.455424] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.455449] process a segment [D 20:00:56.455471] of 2097152 sz 2097152 [D 20:00:56.455496] add a segment 0 [D 20:00:56.455520] retval = 2097152 [D 20:00:56.455544] exiting distribute returned less than expected [D 20:00:56.455568] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.455594] *****RETURN***FROM***PROCESS***** [D 20:00:56.455617] end iteration [D 20:00:56.455641] done with segments or bytes [D 20:00:56.455668] of 37748736 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.455698] next loff: 37748736 next poff: 37748736 [D 20:00:56.455726] return value 2097152 [D 20:00:56.455754] finished [D 20:00:56.455780] retval = 2097152 [D 20:00:56.455804] exiting distribute returned less than expected [D 20:00:56.455829] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 37748736 fo 67108864 eof 0 [D 20:00:56.455855] ========================================================= [D 20:00:56.455879] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:56.455904] element 0: offset: 0xc066f000, size: 2097152 [D 20:00:56.455933] post_sr: da13:3346 bh 67 len 32 wr_id 15e731c8. [D 20:00:56.455962] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:56.455987] ========================================================= [D 20:00:56.456012] PINT_process_request [D 20:00:56.456036] tiling 4 copies [D 20:00:56.456061] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.456087] lvl 0 el 0 blk 0 by 69206016 [D 20:00:56.456111] to 69206016 ta 67108864 fi 100663296 [D 20:00:56.456136] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.456161] basic type or contiguous data [D 20:00:56.456185] Request disp [D 20:00:56.456209] contig_offset = 69206016 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.456234] calling distribute [D 20:00:56.456258] PINT_distribute [D 20:00:56.456282] of 69206016 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 69206016 exfl 1 [D 20:00:56.456309] begin iteration loff: 69206016 [D 20:00:56.456334] segment exceeds byte limit [D 20:00:56.456357] file being extended [D 20:00:56.456381] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.456406] **********CALL***PROCESS********* [D 20:00:56.456430] segment of 2097152 sz 2097152 [D 20:00:56.456454] PINT_process_request [D 20:00:56.456478] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.456504] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.456529] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.456554] basic type or contiguous data [D 20:00:56.456578] Request disp [D 20:00:56.456602] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.456631] process a segment [D 20:00:56.456655] of 2097152 sz 2097152 [D 20:00:56.456680] add a segment 0 [D 20:00:56.456704] retval = 2097152 [D 20:00:56.456727] exiting distribute returned less than expected [D 20:00:56.456751] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.456777] *****RETURN***FROM***PROCESS***** [D 20:00:56.456802] end iteration [D 20:00:56.456826] done with segments or bytes [D 20:00:56.456850] of 71303168 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.456875] next loff: 71303168 next poff: 71303168 [D 20:00:56.456899] return value 2097152 [D 20:00:56.456924] finished [D 20:00:56.456948] retval = 2097152 [D 20:00:56.456972] exiting distribute returned less than expected [D 20:00:56.456995] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 71303168 fo 100663296 eof 0 [D 20:00:56.457021] ========================================================= [D 20:00:56.457045] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:56.457071] element 0: offset: 0xbe66d000, size: 2097152 [D 20:00:56.457099] post_sr: da13:3346 bh 68 len 32 wr_id 15e731e0. [D 20:00:56.457128] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:56.457154] ========================================================= [D 20:00:56.457179] PINT_process_request [D 20:00:56.457202] tiling 5 copies [D 20:00:56.457227] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.457253] lvl 0 el 0 blk 0 by 102760448 [D 20:00:56.457278] to 102760448 ta 100663296 fi 134217728 [D 20:00:56.457304] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.457328] basic type or contiguous data [D 20:00:56.457352] Request disp [D 20:00:56.457376] contig_offset = 102760448 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.457402] calling distribute [D 20:00:56.457426] PINT_distribute [D 20:00:56.457449] of 102760448 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 102760448 exfl 1 [D 20:00:56.457476] begin iteration loff: 102760448 [D 20:00:56.457501] segment exceeds byte limit [D 20:00:56.457525] file being extended [D 20:00:56.457549] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.457573] **********CALL***PROCESS********* [D 20:00:56.457597] segment of 2097152 sz 2097152 [D 20:00:56.457621] PINT_process_request [D 20:00:56.457648] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.457676] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.457701] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.457727] basic type or contiguous data [D 20:00:56.457751] Request disp [D 20:00:56.457775] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.457799] process a segment [D 20:00:56.457822] of 2097152 sz 2097152 [D 20:00:56.457847] add a segment 0 [D 20:00:56.457871] retval = 2097152 [D 20:00:56.457895] exiting distribute returned less than expected [D 20:00:56.457919] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.457944] *****RETURN***FROM***PROCESS***** [D 20:00:56.457969] end iteration [D 20:00:56.457992] done with segments or bytes [D 20:00:56.458016] of 104857600 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.458042] next loff: 104857600 next poff: 104857600 [D 20:00:56.458066] return value 2097152 [D 20:00:56.458091] finished [D 20:00:56.458114] retval = 2097152 [D 20:00:56.458138] exiting distribute returned less than expected [D 20:00:56.458163] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 104857600 fo 134217728 eof 0 [D 20:00:56.458189] ========================================================= [D 20:00:56.458213] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:56.458238] element 0: offset: 0xbc66b000, size: 2097152 [D 20:00:56.458267] post_sr: da13:3346 bh 69 len 32 wr_id 15e731f8. [D 20:00:56.458298] sm control op 6 [D 20:00:56.458323] [SM Entering]: (0x15edf3f8) client_io_sm:io_datafile_complete_operations (status: 1) [D 20:00:56.458352] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.458377] (0x15edf490) io_datafile_complete_operations (tag 1) [D 20:00:56.458402] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:00:56.458427] - complete_context_send_or_recv called [D 20:00:56.458452] got a send completion with context index 0 [D 20:00:56.458476] matched send in context 0; continuing. [D 20:00:56.458500] [SM Exiting]: (0x15edf3f8) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:00:56.458526] PINT_client_state_machine_test id 22 [D 20:00:56.458554] openib_check_cq: completed send wr_id 15e731b0. [D 20:00:56.458580] openib_check_cq: completed recv wr_id 15e73848. [D 20:00:56.458609] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaaae2c9010 rkey 48003200 nic_wr_credit 1. [D 20:00:56.458636] openib_post_rr: da13:3346 bh 36 wr_id 15e73848. [D 20:00:56.458662] openib_check_cq: completed send wr_id 15e731c8. [D 20:00:56.458688] openib_check_cq: completed recv wr_id 15e73860. [D 20:00:56.458716] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaaae4ca010 rkey 48003300 nic_wr_credit 2. [D 20:00:56.458744] openib_post_rr: da13:3346 bh 37 wr_id 15e73860. [D 20:00:56.458769] openib_check_cq: completed send wr_id 15e731e0. [D 20:00:56.458794] openib_check_cq: completed recv wr_id 15e73878. [D 20:00:56.458823] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaaae6cb010 rkey 48003400 nic_wr_credit 3. [D 20:00:56.458850] openib_post_rr: da13:3346 bh 38 wr_id 15e73878. [D 20:00:56.458876] openib_check_cq: completed send wr_id 15e731f8. [D 20:00:56.458902] openib_check_cq: completed recv wr_id 15e73890. [D 20:00:56.458930] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaaae8cc010 rkey 48003500 nic_wr_credit 4. [D 20:00:56.458957] openib_post_rr: da13:3346 bh 39 wr_id 15e73890. [D 20:00:56.458990] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:56.459015] ========================================================= [D 20:00:56.459040] PINT_process_request [D 20:00:56.459064] tiling 6 copies [D 20:00:56.459089] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.459115] lvl 0 el 0 blk 0 by 136314880 [D 20:00:56.459139] to 136314880 ta 134217728 fi 167772160 [D 20:00:56.459164] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.459189] basic type or contiguous data [D 20:00:56.459213] Request disp [D 20:00:56.459237] contig_offset = 136314880 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.459261] calling distribute [D 20:00:56.459285] PINT_distribute [D 20:00:56.459309] of 136314880 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 136314880 exfl 1 [D 20:00:56.459335] begin iteration loff: 136314880 [D 20:00:56.459361] segment exceeds byte limit [D 20:00:56.459384] file being extended [D 20:00:56.459408] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.459432] **********CALL***PROCESS********* [D 20:00:56.459456] segment of 2097152 sz 2097152 [D 20:00:56.459481] PINT_process_request [D 20:00:56.459505] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.459531] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.459556] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.459580] basic type or contiguous data [D 20:00:56.459605] Request disp [D 20:00:56.459628] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.459653] process a segment [D 20:00:56.459681] of 2097152 sz 2097152 [D 20:00:56.459709] add a segment 0 [D 20:00:56.459736] retval = 2097152 [D 20:00:56.459764] exiting distribute returned less than expected [D 20:00:56.459789] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.459815] *****RETURN***FROM***PROCESS***** [D 20:00:56.459840] end iteration [D 20:00:56.459864] done with segments or bytes [D 20:00:56.459887] of 138412032 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.459916] next loff: 138412032 next poff: 138412032 [D 20:00:56.459940] return value 2097152 [D 20:00:56.459965] finished [D 20:00:56.459989] retval = 2097152 [D 20:00:56.460013] exiting distribute returned less than expected [D 20:00:56.460037] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 138412032 fo 167772160 eof 0 [D 20:00:56.460063] ========================================================= [D 20:00:56.460087] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:56.460113] element 0: offset: 0xba669000, size: 2097152 [D 20:00:56.460142] post_sr: da13:3346 bh 70 len 32 wr_id 15e73210. [D 20:00:56.460171] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:56.460197] ========================================================= [D 20:00:56.460221] PINT_process_request [D 20:00:56.460246] tiling 7 copies [D 20:00:56.460269] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.460296] lvl 0 el 0 blk 0 by 169869312 [D 20:00:56.460321] to 169869312 ta 167772160 fi 201326592 [D 20:00:56.460346] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.460371] basic type or contiguous data [D 20:00:56.460394] Request disp [D 20:00:56.460419] contig_offset = 169869312 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.460445] calling distribute [D 20:00:56.460469] PINT_distribute [D 20:00:56.460493] of 169869312 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 169869312 exfl 1 [D 20:00:56.460519] begin iteration loff: 169869312 [D 20:00:56.460544] segment exceeds byte limit [D 20:00:56.460568] file being extended [D 20:00:56.460592] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.460617] **********CALL***PROCESS********* [D 20:00:56.460640] segment of 2097152 sz 2097152 [D 20:00:56.460664] PINT_process_request [D 20:00:56.460688] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.460715] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.460740] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.460764] basic type or contiguous data [D 20:00:56.460788] Request disp [D 20:00:56.460812] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.460838] process a segment [D 20:00:56.460862] of 2097152 sz 2097152 [D 20:00:56.460886] add a segment 0 [D 20:00:56.460909] retval = 2097152 [D 20:00:56.460934] exiting distribute returned less than expected [D 20:00:56.460958] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.460984] *****RETURN***FROM***PROCESS***** [D 20:00:56.461009] end iteration [D 20:00:56.461031] done with segments or bytes [D 20:00:56.461055] of 171966464 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.461081] next loff: 171966464 next poff: 171966464 [D 20:00:56.461106] return value 2097152 [D 20:00:56.461130] finished [D 20:00:56.461153] retval = 2097152 [D 20:00:56.461178] exiting distribute returned less than expected [D 20:00:56.461202] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 171966464 fo 201326592 eof 0 [D 20:00:56.461228] ========================================================= [D 20:00:56.461253] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:56.461277] element 0: offset: 0xb8667000, size: 2097152 [D 20:00:56.461306] post_sr: da13:3346 bh 71 len 32 wr_id 15e73228. [D 20:00:56.461336] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:56.461361] ========================================================= [D 20:00:56.461386] PINT_process_request [D 20:00:56.461410] tiling 8 copies [D 20:00:56.461434] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.461461] lvl 0 el 0 blk 0 by 203423744 [D 20:00:56.461485] to 203423744 ta 201326592 fi 234881024 [D 20:00:56.461510] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.461535] basic type or contiguous data [D 20:00:56.461559] Request disp [D 20:00:56.461583] contig_offset = 203423744 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.461610] calling distribute [D 20:00:56.461634] PINT_distribute [D 20:00:56.461661] of 203423744 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 203423744 exfl 1 [D 20:00:56.461688] begin iteration loff: 203423744 [D 20:00:56.461712] segment exceeds byte limit [D 20:00:56.461737] file being extended [D 20:00:56.461761] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.461786] **********CALL***PROCESS********* [D 20:00:56.461810] segment of 2097152 sz 2097152 [D 20:00:56.461833] PINT_process_request [D 20:00:56.461857] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.461884] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.461909] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.461934] basic type or contiguous data [D 20:00:56.461957] Request disp [D 20:00:56.461981] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.462006] process a segment [D 20:00:56.462030] of 2097152 sz 2097152 [D 20:00:56.462054] add a segment 0 [D 20:00:56.462078] retval = 2097152 [D 20:00:56.462101] exiting distribute returned less than expected [D 20:00:56.462125] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.462152] *****RETURN***FROM***PROCESS***** [D 20:00:56.462176] end iteration [D 20:00:56.462200] done with segments or bytes [D 20:00:56.462224] of 205520896 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.462250] next loff: 205520896 next poff: 205520896 [D 20:00:56.462275] return value 2097152 [D 20:00:56.462299] finished [D 20:00:56.462323] retval = 2097152 [D 20:00:56.462347] exiting distribute returned less than expected [D 20:00:56.462371] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 205520896 fo 234881024 eof 0 [D 20:00:56.462397] ========================================================= [D 20:00:56.462422] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:56.462447] element 0: offset: 0xb6665000, size: 2097152 [D 20:00:56.462475] post_sr: da13:3346 bh 72 len 32 wr_id 15e73240. [D 20:00:56.462504] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:56.462530] ========================================================= [D 20:00:56.462554] PINT_process_request [D 20:00:56.462578] tiling 9 copies [D 20:00:56.462603] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.462629] lvl 0 el 0 blk 0 by 236978176 [D 20:00:56.462655] to 236978176 ta 234881024 fi 268435456 [D 20:00:56.462678] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.462703] basic type or contiguous data [D 20:00:56.462727] Request disp [D 20:00:56.462752] contig_offset = 236978176 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.462777] calling distribute [D 20:00:56.462800] PINT_distribute [D 20:00:56.462824] of 236978176 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 236978176 exfl 1 [D 20:00:56.462851] begin iteration loff: 236978176 [D 20:00:56.462876] segment exceeds byte limit [D 20:00:56.462900] file being extended [D 20:00:56.462924] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.462948] **********CALL***PROCESS********* [D 20:00:56.462972] segment of 2097152 sz 2097152 [D 20:00:56.462997] PINT_process_request [D 20:00:56.463021] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.463047] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.463071] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.463096] basic type or contiguous data [D 20:00:56.463120] Request disp [D 20:00:56.463144] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.463169] process a segment [D 20:00:56.463192] of 2097152 sz 2097152 [D 20:00:56.463217] add a segment 0 [D 20:00:56.463241] retval = 2097152 [D 20:00:56.463265] exiting distribute returned less than expected [D 20:00:56.463289] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.463315] *****RETURN***FROM***PROCESS***** [D 20:00:56.463339] end iteration [D 20:00:56.463366] done with segments or bytes [D 20:00:56.463390] of 239075328 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.463416] next loff: 239075328 next poff: 239075328 [D 20:00:56.463440] return value 2097152 [D 20:00:56.463465] finished [D 20:00:56.463488] retval = 2097152 [D 20:00:56.463513] exiting distribute returned less than expected [D 20:00:56.463537] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 239075328 fo 268435456 eof 0 [D 20:00:56.463562] ========================================================= [D 20:00:56.463587] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:56.463612] element 0: offset: 0xb4663000, size: 2097152 [D 20:00:56.463641] post_sr: da13:3346 bh 73 len 32 wr_id 15e73258. [D 20:00:56.463673] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:56.463703] ========================================================= [D 20:00:56.463730] PINT_process_request [D 20:00:56.463758] tiling 10 copies [D 20:00:56.463785] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.463810] lvl 0 el 0 blk 0 by 270532608 [D 20:00:56.463835] to 270532608 ta 268435456 fi 301989888 [D 20:00:56.463860] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.463885] basic type or contiguous data [D 20:00:56.463909] Request disp [D 20:00:56.463932] contig_offset = 270532608 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.463957] calling distribute [D 20:00:56.463981] PINT_distribute [D 20:00:56.464005] of 270532608 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 270532608 exfl 1 [D 20:00:56.464032] begin iteration loff: 270532608 [D 20:00:56.464056] segment exceeds byte limit [D 20:00:56.464080] file being extended [D 20:00:56.464104] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.464129] **********CALL***PROCESS********* [D 20:00:56.464153] segment of 2097152 sz 2097152 [D 20:00:56.464177] PINT_process_request [D 20:00:56.464200] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.464226] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.464252] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.464276] basic type or contiguous data [D 20:00:56.464301] Request disp [D 20:00:56.464324] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.464349] process a segment [D 20:00:56.464373] of 2097152 sz 2097152 [D 20:00:56.464397] add a segment 0 [D 20:00:56.464421] retval = 2097152 [D 20:00:56.464445] exiting distribute returned less than expected [D 20:00:56.464468] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.464494] *****RETURN***FROM***PROCESS***** [D 20:00:56.464519] end iteration [D 20:00:56.464542] done with segments or bytes [D 20:00:56.464566] of 272629760 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.464591] next loff: 272629760 next poff: 272629760 [D 20:00:56.464616] return value 2097152 [D 20:00:56.464641] finished [D 20:00:56.464664] retval = 2097152 [D 20:00:56.464689] exiting distribute returned less than expected [D 20:00:56.464713] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 272629760 fo 301989888 eof 0 [D 20:00:56.464738] ========================================================= [D 20:00:56.464762] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:56.464788] element 0: offset: 0xb2661000, size: 2097152 [D 20:00:56.464817] post_sr: da13:3346 bh 74 len 32 wr_id 15e73270. [D 20:00:56.464847] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 3. [D 20:00:56.464877] post_sr: da13:3346 bh 75 len 16 wr_id 15e73288. [D 20:00:56.464902] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 2. [D 20:00:56.464931] post_sr: da13:3346 bh 76 len 16 wr_id 15e732a0. [D 20:00:56.464964] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:56.464990] ========================================================= [D 20:00:56.465015] PINT_process_request [D 20:00:56.465042] tiling 11 copies [D 20:00:56.465066] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.465092] lvl 0 el 0 blk 0 by 304087040 [D 20:00:56.465117] to 304087040 ta 301989888 fi 335544320 [D 20:00:56.465142] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.465167] basic type or contiguous data [D 20:00:56.465190] Request disp [D 20:00:56.465214] contig_offset = 304087040 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.465239] calling distribute [D 20:00:56.465263] PINT_distribute [D 20:00:56.465287] of 304087040 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 304087040 exfl 1 [D 20:00:56.465313] begin iteration loff: 304087040 [D 20:00:56.465338] segment exceeds byte limit [D 20:00:56.465362] file being extended [D 20:00:56.465386] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.465411] **********CALL***PROCESS********* [D 20:00:56.465435] segment of 2097152 sz 2097152 [D 20:00:56.465459] PINT_process_request [D 20:00:56.465483] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.465510] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.465535] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.465559] basic type or contiguous data [D 20:00:56.465583] Request disp [D 20:00:56.465607] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.465632] process a segment [D 20:00:56.465656] of 2097152 sz 2097152 [D 20:00:56.465682] add a segment 0 [D 20:00:56.465706] retval = 2097152 [D 20:00:56.465730] exiting distribute returned less than expected [D 20:00:56.465754] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.465780] *****RETURN***FROM***PROCESS***** [D 20:00:56.465805] end iteration [D 20:00:56.465828] done with segments or bytes [D 20:00:56.465852] of 306184192 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.465878] next loff: 306184192 next poff: 306184192 [D 20:00:56.465902] return value 2097152 [D 20:00:56.465927] finished [D 20:00:56.465950] retval = 2097152 [D 20:00:56.465974] exiting distribute returned less than expected [D 20:00:56.465999] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 306184192 fo 335544320 eof 0 [D 20:00:56.466025] ========================================================= [D 20:00:56.466049] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:56.466074] element 0: offset: 0xb065f000, size: 2097152 [D 20:00:56.466103] post_sr: da13:3346 bh 77 len 32 wr_id 15e732b8. [D 20:00:56.466132] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:56.466157] ========================================================= [D 20:00:56.466182] PINT_process_request [D 20:00:56.466206] tiling 12 copies [D 20:00:56.466231] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.466257] lvl 0 el 0 blk 0 by 337641472 [D 20:00:56.466281] to 337641472 ta 335544320 fi 369098752 [D 20:00:56.466306] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.466331] basic type or contiguous data [D 20:00:56.466355] Request disp [D 20:00:56.466379] contig_offset = 337641472 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.466404] calling distribute [D 20:00:56.466428] PINT_distribute [D 20:00:56.466451] of 337641472 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 337641472 exfl 1 [D 20:00:56.466478] begin iteration loff: 337641472 [D 20:00:56.466503] segment exceeds byte limit [D 20:00:56.466526] file being extended [D 20:00:56.466550] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.466575] **********CALL***PROCESS********* [D 20:00:56.466599] segment of 2097152 sz 2097152 [D 20:00:56.466624] PINT_process_request [D 20:00:56.466648] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.466674] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.466699] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.466723] basic type or contiguous data [D 20:00:56.466747] Request disp [D 20:00:56.466771] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.466798] process a segment [D 20:00:56.466822] of 2097152 sz 2097152 [D 20:00:56.466847] add a segment 0 [D 20:00:56.466871] retval = 2097152 [D 20:00:56.466895] exiting distribute returned less than expected [D 20:00:56.466918] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.466945] *****RETURN***FROM***PROCESS***** [D 20:00:56.466969] end iteration [D 20:00:56.466993] done with segments or bytes [D 20:00:56.467017] of 339738624 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.467042] next loff: 339738624 next poff: 339738624 [D 20:00:56.467067] return value 2097152 [D 20:00:56.467091] finished [D 20:00:56.467115] retval = 2097152 [D 20:00:56.467139] exiting distribute returned less than expected [D 20:00:56.467163] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 339738624 fo 369098752 eof 0 [D 20:00:56.467189] ========================================================= [D 20:00:56.467213] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:56.467239] element 0: offset: 0xae65d000, size: 2097152 [D 20:00:56.467267] post_sr: da13:3346 bh 78 len 32 wr_id 15e732d0. [D 20:00:56.467296] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:56.467322] ========================================================= [D 20:00:56.467347] PINT_process_request [D 20:00:56.467370] tiling 13 copies [D 20:00:56.467394] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.467421] lvl 0 el 0 blk 0 by 371195904 [D 20:00:56.467446] to 371195904 ta 369098752 fi 402653184 [D 20:00:56.467471] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.467495] basic type or contiguous data [D 20:00:56.467519] Request disp [D 20:00:56.467543] contig_offset = 371195904 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.467569] calling distribute [D 20:00:56.467593] PINT_distribute [D 20:00:56.467617] of 371195904 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 371195904 exfl 1 [D 20:00:56.467645] begin iteration loff: 371195904 [D 20:00:56.467673] segment exceeds byte limit [D 20:00:56.467700] file being extended [D 20:00:56.467728] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.467757] **********CALL***PROCESS********* [D 20:00:56.467782] segment of 2097152 sz 2097152 [D 20:00:56.467807] PINT_process_request [D 20:00:56.467831] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.467857] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.467882] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.467906] basic type or contiguous data [D 20:00:56.467930] Request disp [D 20:00:56.467954] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.467980] process a segment [D 20:00:56.468004] of 2097152 sz 2097152 [D 20:00:56.468028] add a segment 0 [D 20:00:56.468051] retval = 2097152 [D 20:00:56.468075] exiting distribute returned less than expected [D 20:00:56.468100] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.468126] *****RETURN***FROM***PROCESS***** [D 20:00:56.468150] end iteration [D 20:00:56.468173] done with segments or bytes [D 20:00:56.468197] of 373293056 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.468223] next loff: 373293056 next poff: 373293056 [D 20:00:56.468248] return value 2097152 [D 20:00:56.468273] finished [D 20:00:56.468297] retval = 2097152 [D 20:00:56.468320] exiting distribute returned less than expected [D 20:00:56.468344] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 373293056 fo 402653184 eof 0 [D 20:00:56.468371] ========================================================= [D 20:00:56.468395] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:56.468420] element 0: offset: 0xac65b000, size: 2097152 [D 20:00:56.468448] post_sr: da13:3346 bh 79 len 32 wr_id 15e732e8. [D 20:00:56.468478] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:00:56.468505] ========================================================= [D 20:00:56.468530] PINT_process_request [D 20:00:56.468555] tiling 14 copies [D 20:00:56.468579] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.468605] lvl 0 el 0 blk 0 by 404750336 [D 20:00:56.468629] to 404750336 ta 402653184 fi 436207616 [D 20:00:56.468654] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.468679] basic type or contiguous data [D 20:00:56.468703] Request disp [D 20:00:56.468727] contig_offset = 404750336 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.468752] calling distribute [D 20:00:56.468776] PINT_distribute [D 20:00:56.468799] of 404750336 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 404750336 exfl 1 [D 20:00:56.468826] begin iteration loff: 404750336 [D 20:00:56.468851] segment exceeds byte limit [D 20:00:56.468875] file being extended [D 20:00:56.468899] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.468924] **********CALL***PROCESS********* [D 20:00:56.468948] segment of 2097152 sz 2097152 [D 20:00:56.468972] PINT_process_request [D 20:00:56.468996] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.469022] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.469047] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.469072] basic type or contiguous data [D 20:00:56.469096] Request disp [D 20:00:56.469120] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.469145] process a segment [D 20:00:56.469169] of 2097152 sz 2097152 [D 20:00:56.469193] add a segment 0 [D 20:00:56.469217] retval = 2097152 [D 20:00:56.469241] exiting distribute returned less than expected [D 20:00:56.469264] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.469291] *****RETURN***FROM***PROCESS***** [D 20:00:56.469315] end iteration [D 20:00:56.469339] done with segments or bytes [D 20:00:56.469363] of 406847488 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.469388] next loff: 406847488 next poff: 406847488 [D 20:00:56.469413] return value 2097152 [D 20:00:56.469438] finished [D 20:00:56.469462] retval = 2097152 [D 20:00:56.469486] exiting distribute returned less than expected [D 20:00:56.469510] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 406847488 fo 436207616 eof 0 [D 20:00:56.469536] ========================================================= [D 20:00:56.469560] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:00:56.469586] element 0: offset: 0xaa659000, size: 2097152 [D 20:00:56.469614] post_sr: da13:3346 bh 80 len 32 wr_id 15e73300. [D 20:00:56.469643] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:00:56.469671] ========================================================= [D 20:00:56.469696] PINT_process_request [D 20:00:56.469720] tiling 15 copies [D 20:00:56.469744] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.469771] lvl 0 el 0 blk 0 by 438304768 [D 20:00:56.469796] to 438304768 ta 436207616 fi 469762048 [D 20:00:56.469820] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.469845] basic type or contiguous data [D 20:00:56.469869] Request disp [D 20:00:56.469893] contig_offset = 438304768 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.469918] calling distribute [D 20:00:56.469943] PINT_distribute [D 20:00:56.469966] of 438304768 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 438304768 exfl 1 [D 20:00:56.469992] begin iteration loff: 438304768 [D 20:00:56.470018] segment exceeds byte limit [D 20:00:56.470042] file being extended [D 20:00:56.470065] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.470089] **********CALL***PROCESS********* [D 20:00:56.470113] segment of 2097152 sz 2097152 [D 20:00:56.470138] PINT_process_request [D 20:00:56.470162] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.470188] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.470212] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.470237] basic type or contiguous data [D 20:00:56.470264] Request disp [D 20:00:56.470288] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.470313] process a segment [D 20:00:56.470336] of 2097152 sz 2097152 [D 20:00:56.470360] add a segment 0 [D 20:00:56.470384] retval = 2097152 [D 20:00:56.470408] exiting distribute returned less than expected [D 20:00:56.470433] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.470458] *****RETURN***FROM***PROCESS***** [D 20:00:56.470483] end iteration [D 20:00:56.470507] done with segments or bytes [D 20:00:56.470530] of 440401920 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.470556] next loff: 440401920 next poff: 440401920 [D 20:00:56.470581] return value 2097152 [D 20:00:56.470605] finished [D 20:00:56.470629] retval = 2097152 [D 20:00:56.470653] exiting distribute returned less than expected [D 20:00:56.470677] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 440401920 fo 469762048 eof 0 [D 20:00:56.470704] ========================================================= [D 20:00:56.470727] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:00:56.470753] element 0: offset: 0xa8657000, size: 2097152 [D 20:00:56.470782] post_sr: da13:3346 bh 81 len 32 wr_id 15e73318. [D 20:00:56.470811] PINT_client_state_machine_test id 22 [D 20:00:56.470840] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 1. [D 20:00:56.470869] post_sr: da13:3346 bh 82 len 16 wr_id 15e73330. [D 20:00:56.470895] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 0. [D 20:00:56.470924] post_sr: da13:3346 bh 83 len 16 wr_id 15e73348. [D 20:00:56.470951] openib_check_cq: completed send wr_id 15e73210. [D 20:00:56.470975] openib_check_cq: completed send wr_id 15e73228. [D 20:00:56.471001] openib_check_cq: completed send wr_id 15e73240. [D 20:00:56.471026] openib_check_cq: completed send wr_id 15e73258. [D 20:00:56.471052] openib_check_cq: completed send wr_id 15e73270. [D 20:00:56.471076] openib_check_cq: completed send wr_id 15e73288. [D 20:00:56.471102] openib_check_cq: completed send wr_id 15e732a0. [D 20:00:56.471127] openib_check_cq: completed send wr_id 15e732b8. [D 20:00:56.471153] openib_check_cq: completed send wr_id 15e732d0. [D 20:00:56.471178] openib_check_cq: completed send wr_id 15e732e8. [D 20:00:56.471203] openib_check_cq: completed recv wr_id 15e738a8. [D 20:00:56.471231] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaaaeacd010 rkey 48003600 nic_wr_credit 1. [D 20:00:56.471258] openib_post_rr: da13:3346 bh 40 wr_id 15e738a8. [D 20:00:56.471284] openib_check_cq: completed recv wr_id 15e738c0. [D 20:00:56.471312] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaaaecce010 rkey 48003700 nic_wr_credit 2. [D 20:00:56.471339] openib_post_rr: da13:3346 bh 41 wr_id 15e738c0. [D 20:00:56.471365] openib_check_cq: completed recv wr_id 15e738d8. [D 20:00:56.471393] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaaaeecf010 rkey c8003800 nic_wr_credit 3. [D 20:00:56.471420] openib_post_rr: da13:3346 bh 42 wr_id 15e738d8. [D 20:00:56.471446] openib_check_cq: completed recv wr_id 15e738f0. [D 20:00:56.471474] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaaaf0d0010 rkey c8003900 nic_wr_credit 4. [D 20:00:56.471501] openib_post_rr: da13:3346 bh 43 wr_id 15e738f0. [D 20:00:56.471527] openib_check_cq: completed recv wr_id 15e73908. [D 20:00:56.471555] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaaaf2d1010 rkey c8003a00 nic_wr_credit 5. [D 20:00:56.471582] openib_post_rr: da13:3346 bh 44 wr_id 15e73908. [D 20:00:56.471608] openib_check_cq: completed recv wr_id 15e73920. [D 20:00:56.471636] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaaaf4d2010 rkey c8003b00 nic_wr_credit 6. [D 20:00:56.471666] openib_post_rr: da13:3346 bh 45 wr_id 15e73920. [D 20:00:56.471696] openib_check_cq: completed recv wr_id 15e73938. [D 20:00:56.471731] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaaaf6d3010 rkey c8003c00 nic_wr_credit 7. [D 20:00:56.471763] openib_post_rr: da13:3346 bh 46 wr_id 15e73938. [D 20:00:56.471790] openib_check_cq: completed recv wr_id 15e73950. [D 20:00:56.471818] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaaaf8d4010 rkey c8003d00 nic_wr_credit 8. [D 20:00:56.471845] openib_post_rr: da13:3346 bh 47 wr_id 15e73950. [D 20:00:56.471871] openib_check_cq: completed send wr_id 15e73300. [D 20:00:56.471896] openib_check_cq: completed recv wr_id 15e73968. [D 20:00:56.471924] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaaafad5010 rkey c8003e00 nic_wr_credit 9. [D 20:00:56.471951] openib_post_rr: da13:3346 bh 48 wr_id 15e73968. [D 20:00:56.471977] openib_check_cq: completed send wr_id 15e73318. [D 20:00:56.472002] openib_check_cq: completed send wr_id 15e73330. [D 20:00:56.472027] openib_check_cq: completed send wr_id 15e73348. [D 20:00:56.472059] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:56.472084] ========================================================= [D 20:00:56.472108] PINT_process_request [D 20:00:56.472133] tiling 2 copies [D 20:00:56.472157] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.472183] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.472208] to 4194304 ta 0 fi 33554432 [D 20:00:56.472232] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.472257] basic type or contiguous data [D 20:00:56.472281] Request disp [D 20:00:56.472305] contig_offset = 4194304 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.472330] calling distribute [D 20:00:56.472353] PINT_distribute [D 20:00:56.472377] of 4194304 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 4194304 exfl 1 [D 20:00:56.472403] begin iteration loff: 4194304 [D 20:00:56.472429] segment exceeds byte limit [D 20:00:56.472453] file being extended [D 20:00:56.472476] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.472501] **********CALL***PROCESS********* [D 20:00:56.472525] segment of 4194304 sz 2097152 [D 20:00:56.472549] PINT_process_request [D 20:00:56.472574] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.472599] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.472624] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.472649] basic type or contiguous data [D 20:00:56.472673] Request disp [D 20:00:56.472697] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.472722] process a segment [D 20:00:56.472745] of 4194304 sz 2097152 [D 20:00:56.472769] add a segment 0 [D 20:00:56.472793] retval = 2097152 [D 20:00:56.472817] exiting distribute returned less than expected [D 20:00:56.472841] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.472867] *****RETURN***FROM***PROCESS***** [D 20:00:56.472891] end iteration [D 20:00:56.472915] done with segments or bytes [D 20:00:56.472939] of 6291456 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.472965] next loff: 6291456 next poff: 6291456 [D 20:00:56.472990] return value 2097152 [D 20:00:56.473013] finished [D 20:00:56.473037] retval = 2097152 [D 20:00:56.473061] exiting distribute returned less than expected [D 20:00:56.473085] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 6291456 fo 33554432 eof 0 [D 20:00:56.473111] ========================================================= [D 20:00:56.473135] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:56.473160] element 0: offset: 0xc2871000, size: 2097152 [D 20:00:56.473189] post_sr: da13:3346 bh 84 len 32 wr_id 15e73360. [D 20:00:56.473217] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:56.473243] ========================================================= [D 20:00:56.473268] PINT_process_request [D 20:00:56.473292] tiling 3 copies [D 20:00:56.473316] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.473346] lvl 0 el 0 blk 0 by 37748736 [D 20:00:56.473372] to 37748736 ta 33554432 fi 67108864 [D 20:00:56.473396] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.473421] basic type or contiguous data [D 20:00:56.473444] Request disp [D 20:00:56.473469] contig_offset = 37748736 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.473494] calling distribute [D 20:00:56.473518] PINT_distribute [D 20:00:56.473542] of 37748736 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 37748736 exfl 1 [D 20:00:56.473567] begin iteration loff: 37748736 [D 20:00:56.473592] segment exceeds byte limit [D 20:00:56.473617] file being extended [D 20:00:56.473640] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.473668] **********CALL***PROCESS********* [D 20:00:56.473691] segment of 4194304 sz 2097152 [D 20:00:56.473716] PINT_process_request [D 20:00:56.473740] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.473766] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.473791] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.473815] basic type or contiguous data [D 20:00:56.473839] Request disp [D 20:00:56.473863] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.473888] process a segment [D 20:00:56.473912] of 4194304 sz 2097152 [D 20:00:56.473936] add a segment 0 [D 20:00:56.473959] retval = 2097152 [D 20:00:56.473983] exiting distribute returned less than expected [D 20:00:56.474007] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.474033] *****RETURN***FROM***PROCESS***** [D 20:00:56.474058] end iteration [D 20:00:56.474081] done with segments or bytes [D 20:00:56.474104] of 39845888 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.474130] next loff: 39845888 next poff: 39845888 [D 20:00:56.474155] return value 2097152 [D 20:00:56.474180] finished [D 20:00:56.474204] retval = 2097152 [D 20:00:56.474227] exiting distribute returned less than expected [D 20:00:56.474251] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 39845888 fo 67108864 eof 0 [D 20:00:56.474277] ========================================================= [D 20:00:56.474302] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:56.474327] element 0: offset: 0xc086f000, size: 2097152 [D 20:00:56.474355] post_sr: da13:3346 bh 85 len 32 wr_id 15e73378. [D 20:00:56.474384] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:56.474410] ========================================================= [D 20:00:56.474433] PINT_process_request [D 20:00:56.474458] tiling 4 copies [D 20:00:56.474482] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.474508] lvl 0 el 0 blk 0 by 71303168 [D 20:00:56.474534] to 71303168 ta 67108864 fi 100663296 [D 20:00:56.474557] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.474582] basic type or contiguous data [D 20:00:56.474606] Request disp [D 20:00:56.474631] contig_offset = 71303168 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.474656] calling distribute [D 20:00:56.474679] PINT_distribute [D 20:00:56.474703] of 71303168 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 71303168 exfl 1 [D 20:00:56.474729] begin iteration loff: 71303168 [D 20:00:56.474755] segment exceeds byte limit [D 20:00:56.474779] file being extended [D 20:00:56.474802] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.474826] **********CALL***PROCESS********* [D 20:00:56.474850] segment of 4194304 sz 2097152 [D 20:00:56.474875] PINT_process_request [D 20:00:56.474899] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.474925] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.474949] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.474974] basic type or contiguous data [D 20:00:56.474998] Request disp [D 20:00:56.475022] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.475047] process a segment [D 20:00:56.475070] of 4194304 sz 2097152 [D 20:00:56.475094] add a segment 0 [D 20:00:56.475120] retval = 2097152 [D 20:00:56.475145] exiting distribute returned less than expected [D 20:00:56.475170] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.475195] *****RETURN***FROM***PROCESS***** [D 20:00:56.475219] end iteration [D 20:00:56.475243] done with segments or bytes [D 20:00:56.475267] of 73400320 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.475293] next loff: 73400320 next poff: 73400320 [D 20:00:56.475317] return value 2097152 [D 20:00:56.475342] finished [D 20:00:56.475366] retval = 2097152 [D 20:00:56.475390] exiting distribute returned less than expected [D 20:00:56.475414] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 73400320 fo 100663296 eof 0 [D 20:00:56.475440] ========================================================= [D 20:00:56.475464] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:56.475490] element 0: offset: 0xbe86d000, size: 2097152 [D 20:00:56.475519] post_sr: da13:3346 bh 86 len 32 wr_id 15e73390. [D 20:00:56.475547] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:56.475573] ========================================================= [D 20:00:56.475597] PINT_process_request [D 20:00:56.475621] tiling 5 copies [D 20:00:56.475646] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.475676] lvl 0 el 0 blk 0 by 104857600 [D 20:00:56.475704] to 104857600 ta 100663296 fi 134217728 [D 20:00:56.475732] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.475761] basic type or contiguous data [D 20:00:56.475787] Request disp [D 20:00:56.475811] contig_offset = 104857600 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.475836] calling distribute [D 20:00:56.475861] PINT_distribute [D 20:00:56.475885] of 104857600 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 104857600 exfl 1 [D 20:00:56.475912] begin iteration loff: 104857600 [D 20:00:56.475936] segment exceeds byte limit [D 20:00:56.475960] file being extended [D 20:00:56.475984] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.476009] **********CALL***PROCESS********* [D 20:00:56.476033] segment of 4194304 sz 2097152 [D 20:00:56.476058] PINT_process_request [D 20:00:56.476081] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.476107] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.476132] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.476157] basic type or contiguous data [D 20:00:56.476181] Request disp [D 20:00:56.476204] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.476229] process a segment [D 20:00:56.476253] of 4194304 sz 2097152 [D 20:00:56.476278] add a segment 0 [D 20:00:56.476302] retval = 2097152 [D 20:00:56.476325] exiting distribute returned less than expected [D 20:00:56.476349] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.476376] *****RETURN***FROM***PROCESS***** [D 20:00:56.476400] end iteration [D 20:00:56.476424] done with segments or bytes [D 20:00:56.476448] of 106954752 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.476473] next loff: 106954752 next poff: 106954752 [D 20:00:56.476498] return value 2097152 [D 20:00:56.476523] finished [D 20:00:56.476547] retval = 2097152 [D 20:00:56.476571] exiting distribute returned less than expected [D 20:00:56.476594] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 106954752 fo 134217728 eof 0 [D 20:00:56.476620] ========================================================= [D 20:00:56.476645] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:56.476671] element 0: offset: 0xbc86b000, size: 2097152 [D 20:00:56.476699] post_sr: da13:3346 bh 87 len 32 wr_id 15e733a8. [D 20:00:56.476730] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 8. [D 20:00:56.476758] post_sr: da13:3346 bh 88 len 16 wr_id 15e733c0. [D 20:00:56.476785] openib_check_cq: completed recv wr_id 15e73980. [D 20:00:56.476816] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaaafcd6010 rkey c8003f00 nic_wr_credit 9. [D 20:00:56.476843] openib_post_rr: da13:3346 bh 49 wr_id 15e73980. [D 20:00:56.476870] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 8. [D 20:00:56.476898] post_sr: da13:3346 bh 89 len 16 wr_id 15e733d8. [D 20:00:56.478632] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 7. [D 20:00:56.478662] post_sr: da13:3346 bh 90 len 16 wr_id 15e733f0. [D 20:00:56.480842] PINT_client_state_machine_test id 22 [D 20:00:56.481087] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 6. [D 20:00:56.481116] post_sr: da13:3346 bh 91 len 16 wr_id 15e73408. [D 20:00:56.483538] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 5. [D 20:00:56.483567] post_sr: da13:3346 bh 92 len 16 wr_id 15e73420. [D 20:00:56.485994] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 4. [D 20:00:56.486023] post_sr: da13:3346 bh 93 len 16 wr_id 15e73438. [D 20:00:56.488444] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 3. [D 20:00:56.488473] post_sr: da13:3346 bh 94 len 16 wr_id 15e73450. [D 20:00:56.490873] PINT_client_state_machine_test id 22 [D 20:00:56.490902] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 2. [D 20:00:56.490931] post_sr: da13:3346 bh 95 len 16 wr_id 15e73468. [D 20:00:56.493345] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 1. [D 20:00:56.493374] post_sr: da13:3346 bh 96 len 16 wr_id 15e73480. [D 20:00:56.493401] openib_check_cq: completed send wr_id 15e73360. [D 20:00:56.493427] openib_check_cq: completed send wr_id 15e73378. [D 20:00:56.493452] openib_check_cq: completed send wr_id 15e73390. [D 20:00:56.493476] openib_check_cq: completed send wr_id 15e733a8. [D 20:00:56.493502] openib_check_cq: completed send wr_id 15e733c0. [D 20:00:56.493527] openib_check_cq: completed recv wr_id 15e73998. [D 20:00:56.493556] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaaafed7010 rkey 48004000 nic_wr_credit 2. [D 20:00:56.493582] openib_post_rr: da13:3346 bh 50 wr_id 15e73998. [D 20:00:56.493609] openib_check_cq: completed recv wr_id 15e739b0. [D 20:00:56.493637] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab00d8010 rkey 48004100 nic_wr_credit 3. [D 20:00:56.493677] openib_post_rr: da13:3346 bh 51 wr_id 15e739b0. [D 20:00:56.493703] openib_check_cq: completed recv wr_id 15e739c8. [D 20:00:56.493731] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab02d9010 rkey 48004200 nic_wr_credit 4. [D 20:00:56.493759] openib_post_rr: da13:3346 bh 52 wr_id 15e739c8. [D 20:00:56.493784] openib_check_cq: completed recv wr_id 15e739e0. [D 20:00:56.493813] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab04da010 rkey 48004300 nic_wr_credit 5. [D 20:00:56.493840] openib_post_rr: da13:3346 bh 53 wr_id 15e739e0. [D 20:00:56.493870] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:56.493896] ========================================================= [D 20:00:56.493921] PINT_process_request [D 20:00:56.493945] tiling 6 copies [D 20:00:56.493968] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.493995] lvl 0 el 0 blk 0 by 138412032 [D 20:00:56.494020] to 138412032 ta 134217728 fi 167772160 [D 20:00:56.494044] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.494069] basic type or contiguous data [D 20:00:56.494092] Request disp [D 20:00:56.494116] contig_offset = 138412032 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.494141] calling distribute [D 20:00:56.494165] PINT_distribute [D 20:00:56.494189] of 138412032 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 138412032 exfl 1 [D 20:00:56.494215] begin iteration loff: 138412032 [D 20:00:56.494240] segment exceeds byte limit [D 20:00:56.494264] file being extended [D 20:00:56.494287] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.494312] **********CALL***PROCESS********* [D 20:00:56.494338] segment of 4194304 sz 2097152 [D 20:00:56.494363] PINT_process_request [D 20:00:56.494387] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.494413] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.494438] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.494462] basic type or contiguous data [D 20:00:56.494486] Request disp [D 20:00:56.494511] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.494536] process a segment [D 20:00:56.494559] of 4194304 sz 2097152 [D 20:00:56.494584] add a segment 0 [D 20:00:56.494607] retval = 2097152 [D 20:00:56.494631] exiting distribute returned less than expected [D 20:00:56.494655] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.494682] *****RETURN***FROM***PROCESS***** [D 20:00:56.494706] end iteration [D 20:00:56.494729] done with segments or bytes [D 20:00:56.494753] of 140509184 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.494779] next loff: 140509184 next poff: 140509184 [D 20:00:56.494804] return value 2097152 [D 20:00:56.494829] finished [D 20:00:56.494853] retval = 2097152 [D 20:00:56.494876] exiting distribute returned less than expected [D 20:00:56.494900] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 140509184 fo 167772160 eof 0 [D 20:00:56.494926] ========================================================= [D 20:00:56.494950] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:56.494976] element 0: offset: 0xba869000, size: 2097152 [D 20:00:56.495004] post_sr: da13:3346 bh 97 len 32 wr_id 15e73498. [D 20:00:56.495811] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 4. [D 20:00:56.495841] post_sr: da13:3346 bh 98 len 16 wr_id 15e734b0. [D 20:00:56.495867] openib_check_cq: completed send wr_id 15e733d8. [D 20:00:56.495893] openib_check_cq: completed send wr_id 15e733f0. [D 20:00:56.495918] openib_check_cq: completed send wr_id 15e73408. [D 20:00:56.495944] openib_check_cq: completed send wr_id 15e73420. [D 20:00:56.495968] openib_check_cq: completed send wr_id 15e73438. [D 20:00:56.495994] openib_check_cq: completed send wr_id 15e73450. [D 20:00:56.496020] openib_check_cq: completed send wr_id 15e73468. [D 20:00:56.496045] openib_check_cq: completed send wr_id 15e73480. [D 20:00:56.496076] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:56.496102] ========================================================= [D 20:00:56.496127] PINT_process_request [D 20:00:56.496150] tiling 7 copies [D 20:00:56.496175] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.496201] lvl 0 el 0 blk 0 by 171966464 [D 20:00:56.496226] to 171966464 ta 167772160 fi 201326592 [D 20:00:56.496252] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.496275] basic type or contiguous data [D 20:00:56.496300] Request disp [D 20:00:56.496324] contig_offset = 171966464 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.496349] calling distribute [D 20:00:56.496373] PINT_distribute [D 20:00:56.496397] of 171966464 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 171966464 exfl 1 [D 20:00:56.496423] begin iteration loff: 171966464 [D 20:00:56.496448] segment exceeds byte limit [D 20:00:56.496472] file being extended [D 20:00:56.496496] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.496520] **********CALL***PROCESS********* [D 20:00:56.496544] segment of 4194304 sz 2097152 [D 20:00:56.496569] PINT_process_request [D 20:00:56.496593] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.496619] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.496644] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.496668] basic type or contiguous data [D 20:00:56.496692] Request disp [D 20:00:56.496716] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.496741] process a segment [D 20:00:56.496765] of 4194304 sz 2097152 [D 20:00:56.496788] add a segment 0 [D 20:00:56.496813] retval = 2097152 [D 20:00:56.496841] exiting distribute returned less than expected [D 20:00:56.496866] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.496892] *****RETURN***FROM***PROCESS***** [D 20:00:56.496916] end iteration [D 20:00:56.496940] done with segments or bytes [D 20:00:56.496964] of 174063616 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.496990] next loff: 174063616 next poff: 174063616 [D 20:00:56.497014] return value 2097152 [D 20:00:56.497038] finished [D 20:00:56.497062] retval = 2097152 [D 20:00:56.497086] exiting distribute returned less than expected [D 20:00:56.497110] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 174063616 fo 201326592 eof 0 [D 20:00:56.497137] ========================================================= [D 20:00:56.497160] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:56.497185] element 0: offset: 0xb8867000, size: 2097152 [D 20:00:56.497215] post_sr: da13:3346 bh 99 len 32 wr_id 15e734c8. [D 20:00:56.497242] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:56.497268] ========================================================= [D 20:00:56.497293] PINT_process_request [D 20:00:56.497317] tiling 8 copies [D 20:00:56.497342] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.497367] lvl 0 el 0 blk 0 by 205520896 [D 20:00:56.497392] to 205520896 ta 201326592 fi 234881024 [D 20:00:56.497417] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.497442] basic type or contiguous data [D 20:00:56.497466] Request disp [D 20:00:56.497489] contig_offset = 205520896 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.497515] calling distribute [D 20:00:56.497539] PINT_distribute [D 20:00:56.497563] of 205520896 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 205520896 exfl 1 [D 20:00:56.497589] begin iteration loff: 205520896 [D 20:00:56.497613] segment exceeds byte limit [D 20:00:56.497640] file being extended [D 20:00:56.497666] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.497691] **********CALL***PROCESS********* [D 20:00:56.497714] segment of 4194304 sz 2097152 [D 20:00:56.497739] PINT_process_request [D 20:00:56.497763] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.497789] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.497814] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.497838] basic type or contiguous data [D 20:00:56.497862] Request disp [D 20:00:56.497886] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.497911] process a segment [D 20:00:56.497935] of 4194304 sz 2097152 [D 20:00:56.497959] add a segment 0 [D 20:00:56.497982] retval = 2097152 [D 20:00:56.498006] exiting distribute returned less than expected [D 20:00:56.498030] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.498057] *****RETURN***FROM***PROCESS***** [D 20:00:56.498081] end iteration [D 20:00:56.498105] done with segments or bytes [D 20:00:56.498128] of 207618048 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.498154] next loff: 207618048 next poff: 207618048 [D 20:00:56.498179] return value 2097152 [D 20:00:56.498203] finished [D 20:00:56.498227] retval = 2097152 [D 20:00:56.498250] exiting distribute returned less than expected [D 20:00:56.498274] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 207618048 fo 234881024 eof 0 [D 20:00:56.498301] ========================================================= [D 20:00:56.498325] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:56.498351] element 0: offset: 0xb6865000, size: 2097152 [D 20:00:56.498377] post_sr: da13:3346 bh 0 len 32 wr_id 15e72b80. [D 20:00:56.498406] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:56.498432] ========================================================= [D 20:00:56.498456] PINT_process_request [D 20:00:56.498479] tiling 9 copies [D 20:00:56.498504] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.498533] lvl 0 el 0 blk 0 by 239075328 [D 20:00:56.498558] to 239075328 ta 234881024 fi 268435456 [D 20:00:56.498582] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.498607] basic type or contiguous data [D 20:00:56.498631] Request disp [D 20:00:56.498655] contig_offset = 239075328 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.498681] calling distribute [D 20:00:56.498704] PINT_distribute [D 20:00:56.498728] of 239075328 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 239075328 exfl 1 [D 20:00:56.498755] begin iteration loff: 239075328 [D 20:00:56.498780] segment exceeds byte limit [D 20:00:56.498804] file being extended [D 20:00:56.498827] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.498852] **********CALL***PROCESS********* [D 20:00:56.498876] segment of 4194304 sz 2097152 [D 20:00:56.498901] PINT_process_request [D 20:00:56.498925] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.498950] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.498975] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.499000] basic type or contiguous data [D 20:00:56.499024] Request disp [D 20:00:56.499048] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.499073] process a segment [D 20:00:56.499096] of 4194304 sz 2097152 [D 20:00:56.499120] add a segment 0 [D 20:00:56.499144] retval = 2097152 [D 20:00:56.499169] exiting distribute returned less than expected [D 20:00:56.499193] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.499218] *****RETURN***FROM***PROCESS***** [D 20:00:56.499243] end iteration [D 20:00:56.499267] done with segments or bytes [D 20:00:56.499291] of 241172480 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.499317] next loff: 241172480 next poff: 241172480 [D 20:00:56.499341] return value 2097152 [D 20:00:56.499365] finished [D 20:00:56.499389] retval = 2097152 [D 20:00:56.499413] exiting distribute returned less than expected [D 20:00:56.499437] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 241172480 fo 268435456 eof 0 [D 20:00:56.499464] ========================================================= [D 20:00:56.499487] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:56.499513] element 0: offset: 0xb4863000, size: 2097152 [D 20:00:56.499540] post_sr: da13:3346 bh 1 len 32 wr_id 15e72b98. [D 20:00:56.499568] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:56.499594] ========================================================= [D 20:00:56.499618] PINT_process_request [D 20:00:56.499643] tiling 10 copies [D 20:00:56.499670] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.499700] lvl 0 el 0 blk 0 by 272629760 [D 20:00:56.499729] to 272629760 ta 268435456 fi 301989888 [D 20:00:56.499757] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.499785] basic type or contiguous data [D 20:00:56.499809] Request disp [D 20:00:56.499833] contig_offset = 272629760 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.499857] calling distribute [D 20:00:56.499881] PINT_distribute [D 20:00:56.499905] of 272629760 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 272629760 exfl 1 [D 20:00:56.499932] begin iteration loff: 272629760 [D 20:00:56.499957] segment exceeds byte limit [D 20:00:56.499980] file being extended [D 20:00:56.500004] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.500029] **********CALL***PROCESS********* [D 20:00:56.500053] segment of 4194304 sz 2097152 [D 20:00:56.500078] PINT_process_request [D 20:00:56.500102] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.500128] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.500153] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.500178] basic type or contiguous data [D 20:00:56.500202] Request disp [D 20:00:56.500226] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.500250] process a segment [D 20:00:56.500276] of 4194304 sz 2097152 [D 20:00:56.500301] add a segment 0 [D 20:00:56.500325] retval = 2097152 [D 20:00:56.500349] exiting distribute returned less than expected [D 20:00:56.500373] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.500399] *****RETURN***FROM***PROCESS***** [D 20:00:56.500424] end iteration [D 20:00:56.500448] done with segments or bytes [D 20:00:56.500472] of 274726912 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.500497] next loff: 274726912 next poff: 274726912 [D 20:00:56.500522] return value 2097152 [D 20:00:56.500546] finished [D 20:00:56.500570] retval = 2097152 [D 20:00:56.500594] exiting distribute returned less than expected [D 20:00:56.500618] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 274726912 fo 301989888 eof 0 [D 20:00:56.500644] ========================================================= [D 20:00:56.500669] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:56.500694] element 0: offset: 0xb2861000, size: 2097152 [D 20:00:56.500720] post_sr: da13:3346 bh 2 len 32 wr_id 15e72bb0. [D 20:00:56.500749] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:56.500775] ========================================================= [D 20:00:56.500799] PINT_process_request [D 20:00:56.500823] tiling 11 copies [D 20:00:56.500847] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.500873] lvl 0 el 0 blk 0 by 306184192 [D 20:00:56.500898] to 306184192 ta 301989888 fi 335544320 [D 20:00:56.500923] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.500947] basic type or contiguous data [D 20:00:56.500971] Request disp [D 20:00:56.500996] contig_offset = 306184192 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.501021] calling distribute [D 20:00:56.501045] PINT_distribute [D 20:00:56.501069] of 306184192 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 306184192 exfl 1 [D 20:00:56.501094] begin iteration loff: 306184192 [D 20:00:56.501119] segment exceeds byte limit [D 20:00:56.501144] file being extended [D 20:00:56.501167] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.501192] **********CALL***PROCESS********* [D 20:00:56.501215] segment of 4194304 sz 2097152 [D 20:00:56.501240] PINT_process_request [D 20:00:56.501264] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.501290] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.501315] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.501339] basic type or contiguous data [D 20:00:56.501363] Request disp [D 20:00:56.501387] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.501412] process a segment [D 20:00:56.501436] of 4194304 sz 2097152 [D 20:00:56.501461] add a segment 0 [D 20:00:56.501484] retval = 2097152 [D 20:00:56.501508] exiting distribute returned less than expected [D 20:00:56.501532] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.501558] *****RETURN***FROM***PROCESS***** [D 20:00:56.501583] end iteration [D 20:00:56.501606] done with segments or bytes [D 20:00:56.501629] of 308281344 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.501655] next loff: 308281344 next poff: 308281344 [D 20:00:56.501683] return value 2097152 [D 20:00:56.501707] finished [D 20:00:56.501730] retval = 2097152 [D 20:00:56.501754] exiting distribute returned less than expected [D 20:00:56.501778] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 308281344 fo 335544320 eof 0 [D 20:00:56.501805] ========================================================= [D 20:00:56.501829] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:56.501854] element 0: offset: 0xb085f000, size: 2097152 [D 20:00:56.501881] post_sr: da13:3346 bh 3 len 32 wr_id 15e72bc8. [D 20:00:56.501911] PINT_client_state_machine_test id 22 [D 20:00:56.501939] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 3. [D 20:00:56.501969] post_sr: da13:3346 bh 4 len 16 wr_id 15e72be0. [D 20:00:56.501996] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 2. [D 20:00:56.502022] post_sr: da13:3346 bh 5 len 16 wr_id 15e72bf8. [D 20:00:56.502054] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:56.502079] ========================================================= [D 20:00:56.502104] PINT_process_request [D 20:00:56.502128] tiling 12 copies [D 20:00:56.502153] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.502180] lvl 0 el 0 blk 0 by 339738624 [D 20:00:56.502204] to 339738624 ta 335544320 fi 369098752 [D 20:00:56.502229] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.502254] basic type or contiguous data [D 20:00:56.502278] Request disp [D 20:00:56.502302] contig_offset = 339738624 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.502327] calling distribute [D 20:00:56.502351] PINT_distribute [D 20:00:56.502375] of 339738624 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 339738624 exfl 1 [D 20:00:56.502401] begin iteration loff: 339738624 [D 20:00:56.502426] segment exceeds byte limit [D 20:00:56.502450] file being extended [D 20:00:56.502474] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.502499] **********CALL***PROCESS********* [D 20:00:56.502522] segment of 4194304 sz 2097152 [D 20:00:56.502547] PINT_process_request [D 20:00:56.502571] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.502597] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.502622] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.502647] basic type or contiguous data [D 20:00:56.502671] Request disp [D 20:00:56.502695] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.502719] process a segment [D 20:00:56.502743] of 4194304 sz 2097152 [D 20:00:56.502767] add a segment 0 [D 20:00:56.502791] retval = 2097152 [D 20:00:56.502815] exiting distribute returned less than expected [D 20:00:56.502840] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.502865] *****RETURN***FROM***PROCESS***** [D 20:00:56.502889] end iteration [D 20:00:56.502913] done with segments or bytes [D 20:00:56.502937] of 341835776 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.502963] next loff: 341835776 next poff: 341835776 [D 20:00:56.502987] return value 2097152 [D 20:00:56.503012] finished [D 20:00:56.503036] retval = 2097152 [D 20:00:56.503060] exiting distribute returned less than expected [D 20:00:56.503084] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 341835776 fo 369098752 eof 0 [D 20:00:56.503110] ========================================================= [D 20:00:56.503134] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:56.503160] element 0: offset: 0xae85d000, size: 2097152 [D 20:00:56.503187] post_sr: da13:3346 bh 6 len 32 wr_id 15e72c10. [D 20:00:56.503215] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:56.503241] ========================================================= [D 20:00:56.503265] PINT_process_request [D 20:00:56.503289] tiling 13 copies [D 20:00:56.503314] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.503339] lvl 0 el 0 blk 0 by 373293056 [D 20:00:56.503364] to 373293056 ta 369098752 fi 402653184 [D 20:00:56.503389] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.503414] basic type or contiguous data [D 20:00:56.503437] Request disp [D 20:00:56.503461] contig_offset = 373293056 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.503486] calling distribute [D 20:00:56.503510] PINT_distribute [D 20:00:56.503534] of 373293056 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 373293056 exfl 1 [D 20:00:56.503560] begin iteration loff: 373293056 [D 20:00:56.503585] segment exceeds byte limit [D 20:00:56.503609] file being extended [D 20:00:56.503633] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.503658] **********CALL***PROCESS********* [D 20:00:56.503689] segment of 4194304 sz 2097152 [D 20:00:56.503718] PINT_process_request [D 20:00:56.503746] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.503775] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.503800] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.503825] basic type or contiguous data [D 20:00:56.503849] Request disp [D 20:00:56.503873] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.503898] process a segment [D 20:00:56.503922] of 4194304 sz 2097152 [D 20:00:56.503946] add a segment 0 [D 20:00:56.503971] retval = 2097152 [D 20:00:56.503994] exiting distribute returned less than expected [D 20:00:56.504018] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.504044] *****RETURN***FROM***PROCESS***** [D 20:00:56.504069] end iteration [D 20:00:56.504093] done with segments or bytes [D 20:00:56.504117] of 375390208 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.504142] next loff: 375390208 next poff: 375390208 [D 20:00:56.504167] return value 2097152 [D 20:00:56.504192] finished [D 20:00:56.504216] retval = 2097152 [D 20:00:56.504240] exiting distribute returned less than expected [D 20:00:56.504263] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 375390208 fo 402653184 eof 0 [D 20:00:56.504290] ========================================================= [D 20:00:56.504314] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:56.504340] element 0: offset: 0xac85b000, size: 2097152 [D 20:00:56.504366] post_sr: da13:3346 bh 7 len 32 wr_id 15e72c28. [D 20:00:56.504395] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:00:56.504421] ========================================================= [D 20:00:56.504446] PINT_process_request [D 20:00:56.504469] tiling 14 copies [D 20:00:56.504493] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.504520] lvl 0 el 0 blk 0 by 406847488 [D 20:00:56.504545] to 406847488 ta 402653184 fi 436207616 [D 20:00:56.504570] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.504594] basic type or contiguous data [D 20:00:56.504618] Request disp [D 20:00:56.504642] contig_offset = 406847488 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.504667] calling distribute [D 20:00:56.504692] PINT_distribute [D 20:00:56.504714] of 406847488 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 406847488 exfl 1 [D 20:00:56.504741] begin iteration loff: 406847488 [D 20:00:56.504766] segment exceeds byte limit [D 20:00:56.504790] file being extended [D 20:00:56.504814] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.504838] **********CALL***PROCESS********* [D 20:00:56.504862] segment of 4194304 sz 2097152 [D 20:00:56.504887] PINT_process_request [D 20:00:56.504911] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.504937] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.504961] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.504986] basic type or contiguous data [D 20:00:56.505010] Request disp [D 20:00:56.505034] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.505059] process a segment [D 20:00:56.505083] of 4194304 sz 2097152 [D 20:00:56.505107] add a segment 0 [D 20:00:56.505131] retval = 2097152 [D 20:00:56.505155] exiting distribute returned less than expected [D 20:00:56.505179] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.505206] *****RETURN***FROM***PROCESS***** [D 20:00:56.505229] end iteration [D 20:00:56.505253] done with segments or bytes [D 20:00:56.505277] of 408944640 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.505303] next loff: 408944640 next poff: 408944640 [D 20:00:56.505328] return value 2097152 [D 20:00:56.505352] finished [D 20:00:56.505375] retval = 2097152 [D 20:00:56.505400] exiting distribute returned less than expected [D 20:00:56.505424] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 408944640 fo 436207616 eof 0 [D 20:00:56.505453] ========================================================= [D 20:00:56.505477] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:00:56.505503] element 0: offset: 0xaa859000, size: 2097152 [D 20:00:56.505531] post_sr: da13:3346 bh 8 len 32 wr_id 15e72c40. [D 20:00:56.505561] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 1. [D 20:00:56.505589] post_sr: da13:3346 bh 9 len 16 wr_id 15e72c58. [D 20:00:56.505634] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 0. [D 20:00:56.505664] post_sr: da13:3346 bh 10 len 16 wr_id 15e72c70. [D 20:00:56.505691] openib_check_cq: completed send wr_id 15e73498. [D 20:00:56.505717] openib_check_cq: completed send wr_id 15e734b0. [D 20:00:56.505743] openib_check_cq: completed send wr_id 15e734c8. [D 20:00:56.505767] openib_check_cq: completed send wr_id 15e72b80. [D 20:00:56.505793] openib_check_cq: completed send wr_id 15e72b98. [D 20:00:56.505818] openib_check_cq: completed send wr_id 15e72bb0. [D 20:00:56.505844] openib_check_cq: completed send wr_id 15e72bc8. [D 20:00:56.505868] openib_check_cq: completed send wr_id 15e72be0. [D 20:00:56.505894] openib_check_cq: completed send wr_id 15e72bf8. [D 20:00:56.505919] openib_check_cq: completed send wr_id 15e72c10. [D 20:00:56.505945] openib_check_cq: completed send wr_id 15e72c28. [D 20:00:56.505970] openib_check_cq: completed send wr_id 15e72c40. [D 20:00:56.505995] openib_check_cq: completed send wr_id 15e72c58. [D 20:00:56.506020] openib_check_cq: completed send wr_id 15e72c70. [D 20:00:56.506052] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:00:56.506077] ========================================================= [D 20:00:56.506101] PINT_process_request [D 20:00:56.506126] tiling 15 copies [D 20:00:56.506150] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.506176] lvl 0 el 0 blk 0 by 440401920 [D 20:00:56.506201] to 440401920 ta 436207616 fi 469762048 [D 20:00:56.506226] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.506250] basic type or contiguous data [D 20:00:56.506274] Request disp [D 20:00:56.506299] contig_offset = 440401920 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.506323] calling distribute [D 20:00:56.506347] PINT_distribute [D 20:00:56.506371] of 440401920 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 1 [D 20:00:56.506397] begin iteration loff: 440401920 [D 20:00:56.506422] segment exceeds byte limit [D 20:00:56.506446] file being extended [D 20:00:56.506469] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.506494] **********CALL***PROCESS********* [D 20:00:56.506518] segment of 4194304 sz 2097152 [D 20:00:56.506543] PINT_process_request [D 20:00:56.506567] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.506592] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.506617] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.506642] basic type or contiguous data [D 20:00:56.506666] Request disp [D 20:00:56.506690] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.506715] process a segment [D 20:00:56.506738] of 4194304 sz 2097152 [D 20:00:56.506763] add a segment 0 [D 20:00:56.506787] retval = 2097152 [D 20:00:56.506811] exiting distribute returned less than expected [D 20:00:56.506834] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.506861] *****RETURN***FROM***PROCESS***** [D 20:00:56.506885] end iteration [D 20:00:56.506909] done with segments or bytes [D 20:00:56.506932] of 442499072 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.506958] next loff: 442499072 next poff: 442499072 [D 20:00:56.506982] return value 2097152 [D 20:00:56.507007] finished [D 20:00:56.507031] retval = 2097152 [D 20:00:56.507055] exiting distribute returned less than expected [D 20:00:56.507079] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 442499072 fo 469762048 eof 0 [D 20:00:56.507105] ========================================================= [D 20:00:56.507132] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:00:56.507158] element 0: offset: 0xa8857000, size: 2097152 [D 20:00:56.507184] post_sr: da13:3346 bh 11 len 32 wr_id 15e72c88. [D 20:00:56.507213] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:56.507239] ========================================================= [D 20:00:56.507263] PINT_process_request [D 20:00:56.507287] tiling 2 copies [D 20:00:56.507311] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.507337] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.507363] to 6291456 ta 0 fi 33554432 [D 20:00:56.507388] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.507412] basic type or contiguous data [D 20:00:56.507436] Request disp [D 20:00:56.507460] contig_offset = 6291456 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.507485] calling distribute [D 20:00:56.507509] PINT_distribute [D 20:00:56.507533] of 6291456 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 6291456 exfl 1 [D 20:00:56.507559] begin iteration loff: 6291456 [D 20:00:56.507584] segment exceeds byte limit [D 20:00:56.507608] file being extended [D 20:00:56.507634] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.507659] **********CALL***PROCESS********* [D 20:00:56.507688] segment of 6291456 sz 2097152 [D 20:00:56.507716] PINT_process_request [D 20:00:56.507743] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.507773] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.507799] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.507823] basic type or contiguous data [D 20:00:56.507847] Request disp [D 20:00:56.507871] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.507896] process a segment [D 20:00:56.507920] of 6291456 sz 2097152 [D 20:00:56.507944] add a segment 0 [D 20:00:56.507967] retval = 2097152 [D 20:00:56.507991] exiting distribute returned less than expected [D 20:00:56.508016] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.508042] *****RETURN***FROM***PROCESS***** [D 20:00:56.508067] end iteration [D 20:00:56.508090] done with segments or bytes [D 20:00:56.508113] of 8388608 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.508139] next loff: 8388608 next poff: 8388608 [D 20:00:56.508164] return value 2097152 [D 20:00:56.508189] finished [D 20:00:56.508213] retval = 2097152 [D 20:00:56.508236] exiting distribute returned less than expected [D 20:00:56.508260] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 8388608 fo 33554432 eof 0 [D 20:00:56.508286] ========================================================= [D 20:00:56.508311] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:56.508336] element 0: offset: 0xc2a71000, size: 2097152 [D 20:00:56.508362] post_sr: da13:3346 bh 12 len 32 wr_id 15e72ca0. [D 20:00:56.508391] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:56.508417] ========================================================= [D 20:00:56.508441] PINT_process_request [D 20:00:56.508465] tiling 3 copies [D 20:00:56.508490] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.508516] lvl 0 el 0 blk 0 by 39845888 [D 20:00:56.508541] to 39845888 ta 33554432 fi 67108864 [D 20:00:56.508565] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.508590] basic type or contiguous data [D 20:00:56.508615] Request disp [D 20:00:56.508638] contig_offset = 39845888 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.508664] calling distribute [D 20:00:56.508687] PINT_distribute [D 20:00:56.508711] of 39845888 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 39845888 exfl 1 [D 20:00:56.508737] begin iteration loff: 39845888 [D 20:00:56.508763] segment exceeds byte limit [D 20:00:56.508787] file being extended [D 20:00:56.508810] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.508835] **********CALL***PROCESS********* [D 20:00:56.508862] segment of 6291456 sz 2097152 [D 20:00:56.508887] PINT_process_request [D 20:00:56.508911] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.508936] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.508961] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.508986] basic type or contiguous data [D 20:00:56.509011] Request disp [D 20:00:56.509035] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.509059] process a segment [D 20:00:56.509083] of 6291456 sz 2097152 [D 20:00:56.509107] add a segment 0 [D 20:00:56.509132] retval = 2097152 [D 20:00:56.509156] exiting distribute returned less than expected [D 20:00:56.509180] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.509206] *****RETURN***FROM***PROCESS***** [D 20:00:56.509230] end iteration [D 20:00:56.509254] done with segments or bytes [D 20:00:56.509278] of 41943040 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.509305] next loff: 41943040 next poff: 41943040 [D 20:00:56.509329] return value 2097152 [D 20:00:56.509353] finished [D 20:00:56.509377] retval = 2097152 [D 20:00:56.509402] exiting distribute returned less than expected [D 20:00:56.509426] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 41943040 fo 67108864 eof 0 [D 20:00:56.509451] ========================================================= [D 20:00:56.509476] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:56.509501] element 0: offset: 0xc0a6f000, size: 2097152 [D 20:00:56.509529] post_sr: da13:3346 bh 13 len 32 wr_id 15e72cb8. [D 20:00:56.509557] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:56.509583] ========================================================= [D 20:00:56.509608] PINT_process_request [D 20:00:56.509632] tiling 4 copies [D 20:00:56.509655] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.509685] lvl 0 el 0 blk 0 by 73400320 [D 20:00:56.509710] to 73400320 ta 67108864 fi 100663296 [D 20:00:56.509735] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.509759] basic type or contiguous data [D 20:00:56.509783] Request disp [D 20:00:56.509807] contig_offset = 73400320 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.509833] calling distribute [D 20:00:56.509857] PINT_distribute [D 20:00:56.509881] of 73400320 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 73400320 exfl 1 [D 20:00:56.509907] begin iteration loff: 73400320 [D 20:00:56.509932] segment exceeds byte limit [D 20:00:56.509956] file being extended [D 20:00:56.509980] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.510004] **********CALL***PROCESS********* [D 20:00:56.510028] segment of 6291456 sz 2097152 [D 20:00:56.510053] PINT_process_request [D 20:00:56.510077] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.510104] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.510128] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.510153] basic type or contiguous data [D 20:00:56.510177] Request disp [D 20:00:56.510201] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.510227] process a segment [D 20:00:56.510251] of 6291456 sz 2097152 [D 20:00:56.510274] add a segment 0 [D 20:00:56.510298] retval = 2097152 [D 20:00:56.510323] exiting distribute returned less than expected [D 20:00:56.510347] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.510374] *****RETURN***FROM***PROCESS***** [D 20:00:56.510397] end iteration [D 20:00:56.510421] done with segments or bytes [D 20:00:56.510445] of 75497472 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.510471] next loff: 75497472 next poff: 75497472 [D 20:00:56.510496] return value 2097152 [D 20:00:56.510520] finished [D 20:00:56.510543] retval = 2097152 [D 20:00:56.510568] exiting distribute returned less than expected [D 20:00:56.510592] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 75497472 fo 100663296 eof 0 [D 20:00:56.510621] ========================================================= [D 20:00:56.510645] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:56.510671] element 0: offset: 0xbea6d000, size: 2097152 [D 20:00:56.510698] post_sr: da13:3346 bh 14 len 32 wr_id 15e72cd0. [D 20:00:56.510726] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:56.510752] ========================================================= [D 20:00:56.510776] PINT_process_request [D 20:00:56.510801] tiling 5 copies [D 20:00:56.510825] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.510851] lvl 0 el 0 blk 0 by 106954752 [D 20:00:56.510876] to 106954752 ta 100663296 fi 134217728 [D 20:00:56.510902] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.510927] basic type or contiguous data [D 20:00:56.510951] Request disp [D 20:00:56.510975] contig_offset = 106954752 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.511000] calling distribute [D 20:00:56.511024] PINT_distribute [D 20:00:56.511048] of 106954752 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 106954752 exfl 1 [D 20:00:56.511075] begin iteration loff: 106954752 [D 20:00:56.511099] segment exceeds byte limit [D 20:00:56.511123] file being extended [D 20:00:56.511147] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.511172] **********CALL***PROCESS********* [D 20:00:56.511196] segment of 6291456 sz 2097152 [D 20:00:56.511220] PINT_process_request [D 20:00:56.511244] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.511270] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.511295] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.511320] basic type or contiguous data [D 20:00:56.511343] Request disp [D 20:00:56.511367] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.511393] process a segment [D 20:00:56.511417] of 6291456 sz 2097152 [D 20:00:56.511441] add a segment 0 [D 20:00:56.511465] retval = 2097152 [D 20:00:56.511488] exiting distribute returned less than expected [D 20:00:56.511513] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.511539] *****RETURN***FROM***PROCESS***** [D 20:00:56.511563] end iteration [D 20:00:56.511587] done with segments or bytes [D 20:00:56.511610] of 109051904 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.511636] next loff: 109051904 next poff: 109051904 [D 20:00:56.511664] return value 2097152 [D 20:00:56.511692] finished [D 20:00:56.511719] retval = 2097152 [D 20:00:56.511746] exiting distribute returned less than expected [D 20:00:56.511774] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 109051904 fo 134217728 eof 0 [D 20:00:56.511799] ========================================================= [D 20:00:56.511824] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:56.511850] element 0: offset: 0xbca6b000, size: 2097152 [D 20:00:56.511877] post_sr: da13:3346 bh 15 len 32 wr_id 15e72ce8. [D 20:00:56.511907] openib_check_cq: completed send wr_id 15e72c88. [D 20:00:56.511934] openib_check_cq: completed recv wr_id 15e739f8. [D 20:00:56.511961] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab06db010 rkey 48004400 nic_wr_credit 1. [D 20:00:56.511988] openib_post_rr: da13:3346 bh 54 wr_id 15e739f8. [D 20:00:56.512015] openib_check_cq: completed recv wr_id 15e73a10. [D 20:00:56.512042] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab08dc010 rkey 48004500 nic_wr_credit 2. [D 20:00:56.512069] openib_post_rr: da13:3346 bh 55 wr_id 15e73a10. [D 20:00:56.512096] openib_check_cq: completed recv wr_id 15e73a28. [D 20:00:56.512124] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab0add010 rkey 48004600 nic_wr_credit 3. [D 20:00:56.512150] openib_post_rr: da13:3346 bh 56 wr_id 15e73a28. [D 20:00:56.512176] openib_check_cq: completed recv wr_id 15e73a40. [D 20:00:56.512207] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab0cde010 rkey 48004700 nic_wr_credit 4. [D 20:00:56.512234] openib_post_rr: da13:3346 bh 57 wr_id 15e73a40. [D 20:00:56.512260] openib_check_cq: completed send wr_id 15e72ca0. [D 20:00:56.512286] openib_check_cq: completed send wr_id 15e72cb8. [D 20:00:56.512310] openib_check_cq: completed send wr_id 15e72cd0. [D 20:00:56.512335] openib_check_cq: completed recv wr_id 15e73a58. [D 20:00:56.512363] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab0edf010 rkey c8004800 nic_wr_credit 5. [D 20:00:56.512391] openib_post_rr: da13:3346 bh 58 wr_id 15e73a58. [D 20:00:56.512416] openib_check_cq: completed send wr_id 15e72ce8. [D 20:00:56.512447] PINT_client_state_machine_test id 22 [D 20:00:56.514454] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 4. [D 20:00:56.514482] post_sr: da13:3346 bh 16 len 16 wr_id 15e72d00. [D 20:00:56.515225] openib_check_cq: completed recv wr_id 15e73a70. [D 20:00:56.515254] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaab10e0010 rkey c8004900 nic_wr_credit 5. [D 20:00:56.515281] openib_post_rr: da13:3346 bh 59 wr_id 15e73a70. [D 20:00:56.516902] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 4. [D 20:00:56.516930] post_sr: da13:3346 bh 17 len 16 wr_id 15e72d18. [D 20:00:56.519212] openib_check_cq: completed recv wr_id 15e73a88. [D 20:00:56.519242] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaab12e1010 rkey c8004a00 nic_wr_credit 5. [D 20:00:56.519268] openib_post_rr: da13:3346 bh 60 wr_id 15e73a88. [D 20:00:56.519357] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 4. [D 20:00:56.519384] post_sr: da13:3346 bh 18 len 16 wr_id 15e72d30. [D 20:00:56.521810] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 3. [D 20:00:56.521839] post_sr: da13:3346 bh 19 len 16 wr_id 15e72d48. [D 20:00:56.522475] PINT_client_state_machine_test id 22 [D 20:00:56.523249] openib_check_cq: completed recv wr_id 15e73aa0. [D 20:00:56.523278] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaab1600010 rkey c8004b00 nic_wr_credit 4. [D 20:00:56.523305] openib_post_rr: da13:3346 bh 61 wr_id 15e73aa0. [D 20:00:56.524262] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 3. [D 20:00:56.524290] post_sr: da13:3346 bh 20 len 16 wr_id 15e72d60. [D 20:00:56.524316] openib_check_cq: completed send wr_id 15e72d00. [D 20:00:56.524347] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:56.524373] ========================================================= [D 20:00:56.524396] PINT_process_request [D 20:00:56.524421] tiling 6 copies [D 20:00:56.524445] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.524471] lvl 0 el 0 blk 0 by 140509184 [D 20:00:56.524496] to 140509184 ta 134217728 fi 167772160 [D 20:00:56.524520] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.524545] basic type or contiguous data [D 20:00:56.524569] Request disp [D 20:00:56.524593] contig_offset = 140509184 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.524618] calling distribute [D 20:00:56.524641] PINT_distribute [D 20:00:56.524665] of 140509184 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 140509184 exfl 1 [D 20:00:56.524692] begin iteration loff: 140509184 [D 20:00:56.524717] segment exceeds byte limit [D 20:00:56.524741] file being extended [D 20:00:56.524765] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.524789] **********CALL***PROCESS********* [D 20:00:56.524813] segment of 6291456 sz 2097152 [D 20:00:56.524838] PINT_process_request [D 20:00:56.524862] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.524888] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.524913] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.524938] basic type or contiguous data [D 20:00:56.524962] Request disp [D 20:00:56.524986] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.525013] process a segment [D 20:00:56.525038] of 6291456 sz 2097152 [D 20:00:56.525062] add a segment 0 [D 20:00:56.525087] retval = 2097152 [D 20:00:56.525111] exiting distribute returned less than expected [D 20:00:56.525135] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.525161] *****RETURN***FROM***PROCESS***** [D 20:00:56.525185] end iteration [D 20:00:56.525209] done with segments or bytes [D 20:00:56.525233] of 142606336 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.525259] next loff: 142606336 next poff: 142606336 [D 20:00:56.525283] return value 2097152 [D 20:00:56.525307] finished [D 20:00:56.525331] retval = 2097152 [D 20:00:56.525355] exiting distribute returned less than expected [D 20:00:56.525379] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 142606336 fo 167772160 eof 0 [D 20:00:56.525405] ========================================================= [D 20:00:56.525429] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:56.525454] element 0: offset: 0xbaa69000, size: 2097152 [D 20:00:56.525482] post_sr: da13:3346 bh 21 len 32 wr_id 15e72d78. [D 20:00:56.526631] openib_check_cq: completed recv wr_id 15e73ab8. [D 20:00:56.526660] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaab1801010 rkey c8004c00 nic_wr_credit 4. [D 20:00:56.526687] openib_post_rr: da13:3346 bh 62 wr_id 15e73ab8. [D 20:00:56.526720] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 3. [D 20:00:56.526746] post_sr: da13:3346 bh 22 len 16 wr_id 15e72d90. [D 20:00:56.526773] openib_check_cq: completed send wr_id 15e72d18. [D 20:00:56.526803] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:56.526828] ========================================================= [D 20:00:56.526853] PINT_process_request [D 20:00:56.526877] tiling 7 copies [D 20:00:56.526902] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.526927] lvl 0 el 0 blk 0 by 174063616 [D 20:00:56.526952] to 174063616 ta 167772160 fi 201326592 [D 20:00:56.526977] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.527002] basic type or contiguous data [D 20:00:56.527026] Request disp [D 20:00:56.527050] contig_offset = 174063616 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.527074] calling distribute [D 20:00:56.527098] PINT_distribute [D 20:00:56.527122] of 174063616 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 174063616 exfl 1 [D 20:00:56.527149] begin iteration loff: 174063616 [D 20:00:56.527173] segment exceeds byte limit [D 20:00:56.527197] file being extended [D 20:00:56.527221] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.527245] **********CALL***PROCESS********* [D 20:00:56.527269] segment of 6291456 sz 2097152 [D 20:00:56.527293] PINT_process_request [D 20:00:56.527317] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.527343] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.527368] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.527392] basic type or contiguous data [D 20:00:56.527416] Request disp [D 20:00:56.527440] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.527464] process a segment [D 20:00:56.527488] of 6291456 sz 2097152 [D 20:00:56.527513] add a segment 0 [D 20:00:56.527537] retval = 2097152 [D 20:00:56.527561] exiting distribute returned less than expected [D 20:00:56.527585] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.527610] *****RETURN***FROM***PROCESS***** [D 20:00:56.527637] end iteration [D 20:00:56.527663] done with segments or bytes [D 20:00:56.527690] of 176160768 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.527720] next loff: 176160768 next poff: 176160768 [D 20:00:56.527749] return value 2097152 [D 20:00:56.527776] finished [D 20:00:56.527800] retval = 2097152 [D 20:00:56.527825] exiting distribute returned less than expected [D 20:00:56.527849] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 176160768 fo 201326592 eof 0 [D 20:00:56.527877] ========================================================= [D 20:00:56.527902] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:56.527928] element 0: offset: 0xb8a67000, size: 2097152 [D 20:00:56.527955] post_sr: da13:3346 bh 23 len 32 wr_id 15e72da8. [D 20:00:56.529171] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 2. [D 20:00:56.529199] post_sr: da13:3346 bh 24 len 16 wr_id 15e72dc0. [D 20:00:56.529225] openib_check_cq: completed send wr_id 15e72d30. [D 20:00:56.529251] openib_check_cq: completed send wr_id 15e72d48. [D 20:00:56.529281] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:56.529306] ========================================================= [D 20:00:56.529331] PINT_process_request [D 20:00:56.529355] tiling 8 copies [D 20:00:56.529378] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.529405] lvl 0 el 0 blk 0 by 207618048 [D 20:00:56.529430] to 207618048 ta 201326592 fi 234881024 [D 20:00:56.529455] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.529479] basic type or contiguous data [D 20:00:56.529503] Request disp [D 20:00:56.529527] contig_offset = 207618048 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.529552] calling distribute [D 20:00:56.529576] PINT_distribute [D 20:00:56.529600] of 207618048 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 207618048 exfl 1 [D 20:00:56.529625] begin iteration loff: 207618048 [D 20:00:56.529650] segment exceeds byte limit [D 20:00:56.529676] file being extended [D 20:00:56.529700] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.529725] **********CALL***PROCESS********* [D 20:00:56.529748] segment of 6291456 sz 2097152 [D 20:00:56.529773] PINT_process_request [D 20:00:56.529797] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.529823] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.529848] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.529872] basic type or contiguous data [D 20:00:56.529896] Request disp [D 20:00:56.529920] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.529945] process a segment [D 20:00:56.529969] of 6291456 sz 2097152 [D 20:00:56.529993] add a segment 0 [D 20:00:56.530016] retval = 2097152 [D 20:00:56.530040] exiting distribute returned less than expected [D 20:00:56.530064] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.530091] *****RETURN***FROM***PROCESS***** [D 20:00:56.530115] end iteration [D 20:00:56.530138] done with segments or bytes [D 20:00:56.530162] of 209715200 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.530188] next loff: 209715200 next poff: 209715200 [D 20:00:56.530213] return value 2097152 [D 20:00:56.530238] finished [D 20:00:56.530261] retval = 2097152 [D 20:00:56.530285] exiting distribute returned less than expected [D 20:00:56.530309] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 209715200 fo 234881024 eof 0 [D 20:00:56.530335] ========================================================= [D 20:00:56.530360] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:56.530385] element 0: offset: 0xb6a65000, size: 2097152 [D 20:00:56.530411] post_sr: da13:3346 bh 25 len 32 wr_id 15e72dd8. [D 20:00:56.530440] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:56.530466] ========================================================= [D 20:00:56.530490] PINT_process_request [D 20:00:56.530514] tiling 9 copies [D 20:00:56.530538] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.530565] lvl 0 el 0 blk 0 by 241172480 [D 20:00:56.530590] to 241172480 ta 234881024 fi 268435456 [D 20:00:56.530614] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.530639] basic type or contiguous data [D 20:00:56.530663] Request disp [D 20:00:56.530687] contig_offset = 241172480 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.530714] calling distribute [D 20:00:56.530739] PINT_distribute [D 20:00:56.530763] of 241172480 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 241172480 exfl 1 [D 20:00:56.530790] begin iteration loff: 241172480 [D 20:00:56.530815] segment exceeds byte limit [D 20:00:56.530838] file being extended [D 20:00:56.530862] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.530886] **********CALL***PROCESS********* [D 20:00:56.530910] segment of 6291456 sz 2097152 [D 20:00:56.530935] PINT_process_request [D 20:00:56.530959] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.530985] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.531010] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.531035] basic type or contiguous data [D 20:00:56.531059] Request disp [D 20:00:56.531083] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.531107] process a segment [D 20:00:56.531131] of 6291456 sz 2097152 [D 20:00:56.531156] add a segment 0 [D 20:00:56.531180] retval = 2097152 [D 20:00:56.531205] exiting distribute returned less than expected [D 20:00:56.531229] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.531254] *****RETURN***FROM***PROCESS***** [D 20:00:56.531279] end iteration [D 20:00:56.531303] done with segments or bytes [D 20:00:56.531327] of 243269632 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.531352] next loff: 243269632 next poff: 243269632 [D 20:00:56.531377] return value 2097152 [D 20:00:56.531401] finished [D 20:00:56.531425] retval = 2097152 [D 20:00:56.531450] exiting distribute returned less than expected [D 20:00:56.531474] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 243269632 fo 268435456 eof 0 [D 20:00:56.531499] ========================================================= [D 20:00:56.531524] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:56.531549] element 0: offset: 0xb4a63000, size: 2097152 [D 20:00:56.531576] post_sr: da13:3346 bh 26 len 32 wr_id 15e72df0. [D 20:00:56.531606] openib_check_cq: completed recv wr_id 15e73ad0. [D 20:00:56.531635] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab1a02010 rkey c8004d00 nic_wr_credit 3. [D 20:00:56.531664] openib_post_rr: da13:3346 bh 63 wr_id 15e73ad0. [D 20:00:56.531695] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 2. [D 20:00:56.531724] post_sr: da13:3346 bh 27 len 16 wr_id 15e72e08. [D 20:00:56.531755] openib_check_cq: completed send wr_id 15e72d60. [D 20:00:56.531783] openib_check_cq: completed send wr_id 15e72d78. [D 20:00:56.531813] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:56.531839] ========================================================= [D 20:00:56.531863] PINT_process_request [D 20:00:56.531888] tiling 10 copies [D 20:00:56.531911] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.531938] lvl 0 el 0 blk 0 by 274726912 [D 20:00:56.531963] to 274726912 ta 268435456 fi 301989888 [D 20:00:56.531988] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.532012] basic type or contiguous data [D 20:00:56.532036] Request disp [D 20:00:56.532060] contig_offset = 274726912 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.532085] calling distribute [D 20:00:56.532109] PINT_distribute [D 20:00:56.532133] of 274726912 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 274726912 exfl 1 [D 20:00:56.532158] begin iteration loff: 274726912 [D 20:00:56.532184] segment exceeds byte limit [D 20:00:56.532208] file being extended [D 20:00:56.532232] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.532257] **********CALL***PROCESS********* [D 20:00:56.532280] segment of 6291456 sz 2097152 [D 20:00:56.532304] PINT_process_request [D 20:00:56.532328] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.532355] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.532380] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.532406] basic type or contiguous data [D 20:00:56.532431] Request disp [D 20:00:56.532455] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.532480] process a segment [D 20:00:56.532504] of 6291456 sz 2097152 [D 20:00:56.532527] add a segment 0 [D 20:00:56.532551] retval = 2097152 [D 20:00:56.532575] exiting distribute returned less than expected [D 20:00:56.532599] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.532625] *****RETURN***FROM***PROCESS***** [D 20:00:56.532649] end iteration [D 20:00:56.532672] done with segments or bytes [D 20:00:56.532696] of 276824064 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.532722] next loff: 276824064 next poff: 276824064 [D 20:00:56.532747] return value 2097152 [D 20:00:56.532771] finished [D 20:00:56.532794] retval = 2097152 [D 20:00:56.532818] exiting distribute returned less than expected [D 20:00:56.532842] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 276824064 fo 301989888 eof 0 [D 20:00:56.532868] ========================================================= [D 20:00:56.532893] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:56.532917] element 0: offset: 0xb2a61000, size: 2097152 [D 20:00:56.532945] post_sr: da13:3346 bh 28 len 32 wr_id 15e72e20. [D 20:00:56.532975] PINT_client_state_machine_test id 22 [D 20:00:56.534084] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 1. [D 20:00:56.534111] post_sr: da13:3346 bh 29 len 16 wr_id 15e72e38. [D 20:00:56.534138] openib_check_cq: completed send wr_id 15e72d90. [D 20:00:56.534164] openib_check_cq: completed send wr_id 15e72da8. [D 20:00:56.534189] openib_check_cq: completed send wr_id 15e72dc0. [D 20:00:56.534214] openib_check_cq: completed send wr_id 15e72dd8. [D 20:00:56.534239] openib_check_cq: completed send wr_id 15e72df0. [D 20:00:56.534270] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:56.534296] ========================================================= [D 20:00:56.534319] PINT_process_request [D 20:00:56.534343] tiling 11 copies [D 20:00:56.534368] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.534394] lvl 0 el 0 blk 0 by 308281344 [D 20:00:56.534419] to 308281344 ta 301989888 fi 335544320 [D 20:00:56.534443] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.534469] basic type or contiguous data [D 20:00:56.534493] Request disp [D 20:00:56.534517] contig_offset = 308281344 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.534543] calling distribute [D 20:00:56.534566] PINT_distribute [D 20:00:56.534589] of 308281344 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 308281344 exfl 1 [D 20:00:56.534616] begin iteration loff: 308281344 [D 20:00:56.534641] segment exceeds byte limit [D 20:00:56.534665] file being extended [D 20:00:56.534688] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.534713] **********CALL***PROCESS********* [D 20:00:56.534737] segment of 6291456 sz 2097152 [D 20:00:56.534762] PINT_process_request [D 20:00:56.534786] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.534811] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.534836] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.534861] basic type or contiguous data [D 20:00:56.534885] Request disp [D 20:00:56.534909] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.534934] process a segment [D 20:00:56.534957] of 6291456 sz 2097152 [D 20:00:56.534982] add a segment 0 [D 20:00:56.535006] retval = 2097152 [D 20:00:56.535030] exiting distribute returned less than expected [D 20:00:56.535054] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.535079] *****RETURN***FROM***PROCESS***** [D 20:00:56.535104] end iteration [D 20:00:56.535127] done with segments or bytes [D 20:00:56.535151] of 310378496 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.535177] next loff: 310378496 next poff: 310378496 [D 20:00:56.535205] return value 2097152 [D 20:00:56.535231] finished [D 20:00:56.535255] retval = 2097152 [D 20:00:56.535279] exiting distribute returned less than expected [D 20:00:56.535303] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 310378496 fo 335544320 eof 0 [D 20:00:56.535329] ========================================================= [D 20:00:56.535353] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:56.535378] element 0: offset: 0xb0a5f000, size: 2097152 [D 20:00:56.535406] post_sr: da13:3346 bh 30 len 32 wr_id 15e72e50. [D 20:00:56.535434] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:56.535460] ========================================================= [D 20:00:56.535484] PINT_process_request [D 20:00:56.535509] tiling 12 copies [D 20:00:56.535532] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.535558] lvl 0 el 0 blk 0 by 341835776 [D 20:00:56.535583] to 341835776 ta 335544320 fi 369098752 [D 20:00:56.535608] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.535633] basic type or contiguous data [D 20:00:56.535656] Request disp [D 20:00:56.535686] contig_offset = 341835776 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.535715] calling distribute [D 20:00:56.535742] PINT_distribute [D 20:00:56.535769] of 341835776 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 341835776 exfl 1 [D 20:00:56.535796] begin iteration loff: 341835776 [D 20:00:56.535821] segment exceeds byte limit [D 20:00:56.535845] file being extended [D 20:00:56.535869] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.535894] **********CALL***PROCESS********* [D 20:00:56.535918] segment of 6291456 sz 2097152 [D 20:00:56.535942] PINT_process_request [D 20:00:56.535965] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.535992] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.536017] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.536042] basic type or contiguous data [D 20:00:56.536066] Request disp [D 20:00:56.536089] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.536114] process a segment [D 20:00:56.536137] of 6291456 sz 2097152 [D 20:00:56.536162] add a segment 0 [D 20:00:56.536186] retval = 2097152 [D 20:00:56.536210] exiting distribute returned less than expected [D 20:00:56.536233] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.536260] *****RETURN***FROM***PROCESS***** [D 20:00:56.536284] end iteration [D 20:00:56.536308] done with segments or bytes [D 20:00:56.536332] of 343932928 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.536357] next loff: 343932928 next poff: 343932928 [D 20:00:56.536382] return value 2097152 [D 20:00:56.536407] finished [D 20:00:56.536431] retval = 2097152 [D 20:00:56.536455] exiting distribute returned less than expected [D 20:00:56.536478] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 343932928 fo 369098752 eof 0 [D 20:00:56.536505] ========================================================= [D 20:00:56.536529] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:56.536555] element 0: offset: 0xaea5d000, size: 2097152 [D 20:00:56.536581] post_sr: da13:3346 bh 31 len 32 wr_id 15e72e68. [D 20:00:56.536613] openib_check_cq: completed recv wr_id 15e73ae8. [D 20:00:56.536642] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab1c03010 rkey c8004e00 nic_wr_credit 2. [D 20:00:56.536668] openib_post_rr: da13:3346 bh 64 wr_id 15e73ae8. [D 20:00:56.536694] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 1. [D 20:00:56.536721] post_sr: da13:3346 bh 32 len 16 wr_id 15e72e80. [D 20:00:56.536747] openib_check_cq: completed send wr_id 15e72e08. [D 20:00:56.536772] openib_check_cq: completed send wr_id 15e72e20. [D 20:00:56.536798] openib_check_cq: completed send wr_id 15e72e38. [D 20:00:56.536823] openib_check_cq: completed send wr_id 15e72e50. [D 20:00:56.536849] openib_check_cq: completed send wr_id 15e72e68. [D 20:00:56.536881] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:56.536907] ========================================================= [D 20:00:56.536931] PINT_process_request [D 20:00:56.536955] tiling 13 copies [D 20:00:56.536979] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.537005] lvl 0 el 0 blk 0 by 375390208 [D 20:00:56.537031] to 375390208 ta 369098752 fi 402653184 [D 20:00:56.537055] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.537080] basic type or contiguous data [D 20:00:56.537104] Request disp [D 20:00:56.537128] contig_offset = 375390208 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.537153] calling distribute [D 20:00:56.537177] PINT_distribute [D 20:00:56.537200] of 375390208 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 375390208 exfl 1 [D 20:00:56.537227] begin iteration loff: 375390208 [D 20:00:56.537252] segment exceeds byte limit [D 20:00:56.537276] file being extended [D 20:00:56.537300] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.537323] **********CALL***PROCESS********* [D 20:00:56.537347] segment of 6291456 sz 2097152 [D 20:00:56.537372] PINT_process_request [D 20:00:56.537401] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.537427] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.537452] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.537477] basic type or contiguous data [D 20:00:56.537501] Request disp [D 20:00:56.537525] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.537549] process a segment [D 20:00:56.537573] of 6291456 sz 2097152 [D 20:00:56.537597] add a segment 0 [D 20:00:56.537623] retval = 2097152 [D 20:00:56.537647] exiting distribute returned less than expected [D 20:00:56.537673] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.537700] *****RETURN***FROM***PROCESS***** [D 20:00:56.537724] end iteration [D 20:00:56.537748] done with segments or bytes [D 20:00:56.537772] of 377487360 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.537797] next loff: 377487360 next poff: 377487360 [D 20:00:56.537822] return value 2097152 [D 20:00:56.537847] finished [D 20:00:56.537871] retval = 2097152 [D 20:00:56.537895] exiting distribute returned less than expected [D 20:00:56.537919] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 377487360 fo 402653184 eof 0 [D 20:00:56.537944] ========================================================= [D 20:00:56.537969] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:56.537994] element 0: offset: 0xaca5b000, size: 2097152 [D 20:00:56.538022] post_sr: da13:3346 bh 33 len 32 wr_id 15e72e98. [D 20:00:56.538049] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:00:56.538075] ========================================================= [D 20:00:56.538100] PINT_process_request [D 20:00:56.538124] tiling 14 copies [D 20:00:56.538147] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.538174] lvl 0 el 0 blk 0 by 408944640 [D 20:00:56.538199] to 408944640 ta 402653184 fi 436207616 [D 20:00:56.538224] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.538249] basic type or contiguous data [D 20:00:56.538272] Request disp [D 20:00:56.538296] contig_offset = 408944640 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.538321] calling distribute [D 20:00:56.538345] PINT_distribute [D 20:00:56.538369] of 408944640 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 408944640 exfl 1 [D 20:00:56.538395] begin iteration loff: 408944640 [D 20:00:56.538420] segment exceeds byte limit [D 20:00:56.538444] file being extended [D 20:00:56.538468] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.538493] **********CALL***PROCESS********* [D 20:00:56.538516] segment of 6291456 sz 2097152 [D 20:00:56.538540] PINT_process_request [D 20:00:56.538564] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.538593] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.538619] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.538642] basic type or contiguous data [D 20:00:56.538667] Request disp [D 20:00:56.538690] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.538716] process a segment [D 20:00:56.538740] of 6291456 sz 2097152 [D 20:00:56.538763] add a segment 0 [D 20:00:56.538787] retval = 2097152 [D 20:00:56.538811] exiting distribute returned less than expected [D 20:00:56.538835] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.538862] *****RETURN***FROM***PROCESS***** [D 20:00:56.538886] end iteration [D 20:00:56.538909] done with segments or bytes [D 20:00:56.538933] of 411041792 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.538959] next loff: 411041792 next poff: 411041792 [D 20:00:56.538984] return value 2097152 [D 20:00:56.539009] finished [D 20:00:56.539031] retval = 2097152 [D 20:00:56.539056] exiting distribute returned less than expected [D 20:00:56.539080] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 411041792 fo 436207616 eof 0 [D 20:00:56.539106] ========================================================= [D 20:00:56.539131] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:00:56.539155] element 0: offset: 0xaaa59000, size: 2097152 [D 20:00:56.539183] post_sr: da13:3346 bh 34 len 32 wr_id 15e72eb0. [D 20:00:56.539213] openib_check_cq: completed recv wr_id 15e73b00. [D 20:00:56.539241] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab1e04010 rkey c8004f00 nic_wr_credit 2. [D 20:00:56.539269] openib_post_rr: da13:3346 bh 65 wr_id 15e73b00. [D 20:00:56.539295] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 1. [D 20:00:56.539321] post_sr: da13:3346 bh 35 len 16 wr_id 15e72ec8. [D 20:00:56.539348] openib_check_cq: completed send wr_id 15e72e80. [D 20:00:56.539374] openib_check_cq: completed send wr_id 15e72e98. [D 20:00:56.539399] openib_check_cq: completed send wr_id 15e72eb0. [D 20:00:56.539429] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:00:56.539455] ========================================================= [D 20:00:56.539479] PINT_process_request [D 20:00:56.539504] tiling 15 copies [D 20:00:56.539527] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.539553] lvl 0 el 0 blk 0 by 442499072 [D 20:00:56.539578] to 442499072 ta 436207616 fi 469762048 [D 20:00:56.539603] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:56.539628] basic type or contiguous data [D 20:00:56.539652] Request disp [D 20:00:56.539680] contig_offset = 442499072 contig_size = 60817408 lvl_flag = 1 [D 20:00:56.539709] calling distribute [D 20:00:56.539736] PINT_distribute [D 20:00:56.539764] of 442499072 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 1 [D 20:00:56.539792] begin iteration loff: 442499072 [D 20:00:56.539816] segment exceeds byte limit [D 20:00:56.539840] file being extended [D 20:00:56.539864] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:56.539889] **********CALL***PROCESS********* [D 20:00:56.539913] segment of 6291456 sz 2097152 [D 20:00:56.539937] PINT_process_request [D 20:00:56.539961] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.539987] lvl 0 el 0 blk 0 by 6291456 [D 20:00:56.540012] to 6291456 ta 6291456 fi 8388608 [D 20:00:56.540037] basic type or contiguous data [D 20:00:56.540060] Request disp [D 20:00:56.540083] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:56.540109] process a segment [D 20:00:56.540132] of 6291456 sz 2097152 [D 20:00:56.540157] add a segment 0 [D 20:00:56.540181] retval = 2097152 [D 20:00:56.540205] exiting distribute returned less than expected [D 20:00:56.540228] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:56.540254] *****RETURN***FROM***PROCESS***** [D 20:00:56.540281] end iteration [D 20:00:56.540306] done with segments or bytes [D 20:00:56.540329] of 444596224 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.540355] next loff: 444596224 next poff: 444596224 [D 20:00:56.540380] return value 2097152 [D 20:00:56.540404] finished [D 20:00:56.540428] retval = 2097152 [D 20:00:56.540452] exiting distribute returned less than expected [D 20:00:56.540475] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 444596224 fo 469762048 eof 0 [D 20:00:56.540502] ========================================================= [D 20:00:56.540526] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:00:56.540551] element 0: offset: 0xa8a57000, size: 2097152 [D 20:00:56.540578] post_sr: da13:3346 bh 36 len 32 wr_id 15e72ee0. [D 20:00:56.541733] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 0. [D 20:00:56.541761] post_sr: da13:3346 bh 37 len 16 wr_id 15e72ef8. [D 20:00:56.541788] openib_check_cq: completed send wr_id 15e72ec8. [D 20:00:56.541813] openib_check_cq: completed send wr_id 15e72ee0. [D 20:00:56.541838] openib_check_cq: completed send wr_id 15e72ef8. [D 20:00:56.541868] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:56.541893] ========================================================= [D 20:00:56.541918] PINT_process_request [D 20:00:56.541942] tiling 2 copies [D 20:00:56.541966] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.541993] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.542017] to 8388608 ta 0 fi 33554432 [D 20:00:56.542041] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.542066] basic type or contiguous data [D 20:00:56.542090] Request disp [D 20:00:56.542114] contig_offset = 8388608 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.542140] calling distribute [D 20:00:56.542163] PINT_distribute [D 20:00:56.542187] of 8388608 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 8388608 exfl 1 [D 20:00:56.542213] begin iteration loff: 8388608 [D 20:00:56.542238] segment exceeds byte limit [D 20:00:56.542262] file being extended [D 20:00:56.542285] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.542310] **********CALL***PROCESS********* [D 20:00:56.542334] segment of 8388608 sz 2097152 [D 20:00:56.542359] PINT_process_request [D 20:00:56.542383] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.542408] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.542433] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.542458] basic type or contiguous data [D 20:00:56.542482] Request disp [D 20:00:56.542506] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.542531] process a segment [D 20:00:56.542554] of 8388608 sz 2097152 [D 20:00:56.542579] add a segment 0 [D 20:00:56.542603] retval = 2097152 [D 20:00:56.542627] exiting distribute returned less than expected [D 20:00:56.542651] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.542677] *****RETURN***FROM***PROCESS***** [D 20:00:56.542701] end iteration [D 20:00:56.542725] done with segments or bytes [D 20:00:56.542749] of 10485760 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.542775] next loff: 10485760 next poff: 10485760 [D 20:00:56.542798] return value 2097152 [D 20:00:56.542823] finished [D 20:00:56.542847] retval = 2097152 [D 20:00:56.542871] exiting distribute returned less than expected [D 20:00:56.542895] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 10485760 fo 33554432 eof 0 [D 20:00:56.542921] ========================================================= [D 20:00:56.542945] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:56.542970] element 0: offset: 0xc2c71000, size: 2097152 [D 20:00:56.542997] post_sr: da13:3346 bh 38 len 32 wr_id 15e72f10. [D 20:00:56.543025] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:56.543051] ========================================================= [D 20:00:56.543079] PINT_process_request [D 20:00:56.543104] tiling 3 copies [D 20:00:56.543128] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.543154] lvl 0 el 0 blk 0 by 41943040 [D 20:00:56.543179] to 41943040 ta 33554432 fi 67108864 [D 20:00:56.543204] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.543228] basic type or contiguous data [D 20:00:56.543251] Request disp [D 20:00:56.543275] contig_offset = 41943040 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.543301] calling distribute [D 20:00:56.543325] PINT_distribute [D 20:00:56.543348] of 41943040 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 41943040 exfl 1 [D 20:00:56.543375] begin iteration loff: 41943040 [D 20:00:56.543399] segment exceeds byte limit [D 20:00:56.543423] file being extended [D 20:00:56.543447] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.543472] **********CALL***PROCESS********* [D 20:00:56.543495] segment of 8388608 sz 2097152 [D 20:00:56.543519] PINT_process_request [D 20:00:56.543543] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.543570] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.543595] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.543620] basic type or contiguous data [D 20:00:56.543643] Request disp [D 20:00:56.543670] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.543700] process a segment [D 20:00:56.543726] of 8388608 sz 2097152 [D 20:00:56.543754] add a segment 0 [D 20:00:56.543782] retval = 2097152 [D 20:00:56.543806] exiting distribute returned less than expected [D 20:00:56.543829] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.543855] *****RETURN***FROM***PROCESS***** [D 20:00:56.543880] end iteration [D 20:00:56.543904] done with segments or bytes [D 20:00:56.543928] of 44040192 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.543953] next loff: 44040192 next poff: 44040192 [D 20:00:56.543977] return value 2097152 [D 20:00:56.544002] finished [D 20:00:56.544026] retval = 2097152 [D 20:00:56.544050] exiting distribute returned less than expected [D 20:00:56.544074] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 44040192 fo 67108864 eof 0 [D 20:00:56.544100] ========================================================= [D 20:00:56.544124] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:56.544150] element 0: offset: 0xc0c6f000, size: 2097152 [D 20:00:56.544177] post_sr: da13:3346 bh 39 len 32 wr_id 15e72f28. [D 20:00:56.544206] PINT_client_state_machine_test id 22 [D 20:00:56.544235] openib_check_cq: completed send wr_id 15e72f10. [D 20:00:56.544260] openib_check_cq: completed send wr_id 15e72f28. [D 20:00:56.546706] openib_check_cq: completed recv wr_id 15e73b18. [D 20:00:56.546734] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab2005010 rkey 48005000 nic_wr_credit 1. [D 20:00:56.546762] openib_post_rr: da13:3346 bh 66 wr_id 15e73b18. [D 20:00:56.549221] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 0. [D 20:00:56.549248] post_sr: da13:3346 bh 40 len 16 wr_id 15e72f40. [D 20:00:56.549285] openib_check_cq: completed send wr_id 15e72f40. [D 20:00:56.549316] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:56.549342] ========================================================= [D 20:00:56.549366] PINT_process_request [D 20:00:56.549390] tiling 4 copies [D 20:00:56.549414] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.549441] lvl 0 el 0 blk 0 by 75497472 [D 20:00:56.549466] to 75497472 ta 67108864 fi 100663296 [D 20:00:56.549490] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.549515] basic type or contiguous data [D 20:00:56.549539] Request disp [D 20:00:56.549563] contig_offset = 75497472 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.549588] calling distribute [D 20:00:56.549612] PINT_distribute [D 20:00:56.549635] of 75497472 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 75497472 exfl 1 [D 20:00:56.549667] begin iteration loff: 75497472 [D 20:00:56.549693] segment exceeds byte limit [D 20:00:56.549716] file being extended [D 20:00:56.549740] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.549765] **********CALL***PROCESS********* [D 20:00:56.549789] segment of 8388608 sz 2097152 [D 20:00:56.549814] PINT_process_request [D 20:00:56.549838] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.549864] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.549889] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.549914] basic type or contiguous data [D 20:00:56.549938] Request disp [D 20:00:56.549962] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.549986] process a segment [D 20:00:56.550010] of 8388608 sz 2097152 [D 20:00:56.550034] add a segment 0 [D 20:00:56.550058] retval = 2097152 [D 20:00:56.550083] exiting distribute returned less than expected [D 20:00:56.550107] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.550133] *****RETURN***FROM***PROCESS***** [D 20:00:56.550157] end iteration [D 20:00:56.550181] done with segments or bytes [D 20:00:56.550205] of 77594624 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.550230] next loff: 77594624 next poff: 77594624 [D 20:00:56.550255] return value 2097152 [D 20:00:56.550279] finished [D 20:00:56.550303] retval = 2097152 [D 20:00:56.550328] exiting distribute returned less than expected [D 20:00:56.550352] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 77594624 fo 100663296 eof 0 [D 20:00:56.550378] ========================================================= [D 20:00:56.550402] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:56.550428] element 0: offset: 0xbec6d000, size: 2097152 [D 20:00:56.550455] post_sr: da13:3346 bh 41 len 32 wr_id 15e72f58. [D 20:00:56.550485] openib_check_cq: completed recv wr_id 15e73b30. [D 20:00:56.550513] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaab2206010 rkey 48005100 nic_wr_credit 1. [D 20:00:56.550540] openib_post_rr: da13:3346 bh 67 wr_id 15e73b30. [D 20:00:56.550566] openib_check_cq: completed send wr_id 15e72f58. [D 20:00:56.553000] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 0. [D 20:00:56.553028] post_sr: da13:3346 bh 42 len 16 wr_id 15e72f70. [D 20:00:56.553064] openib_check_cq: completed send wr_id 15e72f70. [D 20:00:56.553094] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:56.553120] ========================================================= [D 20:00:56.553145] PINT_process_request [D 20:00:56.553169] tiling 5 copies [D 20:00:56.553194] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.553219] lvl 0 el 0 blk 0 by 109051904 [D 20:00:56.553244] to 109051904 ta 100663296 fi 134217728 [D 20:00:56.553269] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.553294] basic type or contiguous data [D 20:00:56.553318] Request disp [D 20:00:56.553341] contig_offset = 109051904 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.553367] calling distribute [D 20:00:56.553391] PINT_distribute [D 20:00:56.553415] of 109051904 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 109051904 exfl 1 [D 20:00:56.553441] begin iteration loff: 109051904 [D 20:00:56.553465] segment exceeds byte limit [D 20:00:56.553489] file being extended [D 20:00:56.553513] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.553538] **********CALL***PROCESS********* [D 20:00:56.553562] segment of 8388608 sz 2097152 [D 20:00:56.553586] PINT_process_request [D 20:00:56.553610] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.553637] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.553664] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.553689] basic type or contiguous data [D 20:00:56.553713] Request disp [D 20:00:56.553737] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.553765] process a segment [D 20:00:56.553789] of 8388608 sz 2097152 [D 20:00:56.553812] add a segment 0 [D 20:00:56.553837] retval = 2097152 [D 20:00:56.553861] exiting distribute returned less than expected [D 20:00:56.553885] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.553911] *****RETURN***FROM***PROCESS***** [D 20:00:56.553936] end iteration [D 20:00:56.553959] done with segments or bytes [D 20:00:56.553982] of 111149056 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.554008] next loff: 111149056 next poff: 111149056 [D 20:00:56.554033] return value 2097152 [D 20:00:56.554058] finished [D 20:00:56.554080] retval = 2097152 [D 20:00:56.554105] exiting distribute returned less than expected [D 20:00:56.554129] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 111149056 fo 134217728 eof 0 [D 20:00:56.554155] ========================================================= [D 20:00:56.554180] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:56.554205] element 0: offset: 0xbcc6b000, size: 2097152 [D 20:00:56.554232] post_sr: da13:3346 bh 43 len 32 wr_id 15e72f88. [D 20:00:56.554262] PINT_client_state_machine_test id 22 [D 20:00:56.554290] openib_check_cq: completed recv wr_id 15e73b48. [D 20:00:56.554319] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab2407010 rkey 48005200 nic_wr_credit 1. [D 20:00:56.554346] openib_post_rr: da13:3346 bh 68 wr_id 15e73b48. [D 20:00:56.554372] openib_check_cq: completed send wr_id 15e72f88. [D 20:00:56.556809] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 0. [D 20:00:56.556837] post_sr: da13:3346 bh 44 len 16 wr_id 15e72fa0. [D 20:00:56.556873] openib_check_cq: completed send wr_id 15e72fa0. [D 20:00:56.556903] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:56.556929] ========================================================= [D 20:00:56.556954] PINT_process_request [D 20:00:56.556978] tiling 6 copies [D 20:00:56.557003] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.557028] lvl 0 el 0 blk 0 by 142606336 [D 20:00:56.557053] to 142606336 ta 134217728 fi 167772160 [D 20:00:56.557078] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.557103] basic type or contiguous data [D 20:00:56.557126] Request disp [D 20:00:56.557151] contig_offset = 142606336 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.557176] calling distribute [D 20:00:56.557200] PINT_distribute [D 20:00:56.557224] of 142606336 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 142606336 exfl 1 [D 20:00:56.557250] begin iteration loff: 142606336 [D 20:00:56.557275] segment exceeds byte limit [D 20:00:56.557299] file being extended [D 20:00:56.557323] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.557348] **********CALL***PROCESS********* [D 20:00:56.557372] segment of 8388608 sz 2097152 [D 20:00:56.557395] PINT_process_request [D 20:00:56.557419] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.557446] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.557471] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.557496] basic type or contiguous data [D 20:00:56.557519] Request disp [D 20:00:56.557543] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.557568] process a segment [D 20:00:56.557592] of 8388608 sz 2097152 [D 20:00:56.557618] add a segment 0 [D 20:00:56.557642] retval = 2097152 [D 20:00:56.557669] exiting distribute returned less than expected [D 20:00:56.557694] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.557720] *****RETURN***FROM***PROCESS***** [D 20:00:56.557744] end iteration [D 20:00:56.557768] done with segments or bytes [D 20:00:56.557792] of 144703488 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.557818] next loff: 144703488 next poff: 144703488 [D 20:00:56.557842] return value 2097152 [D 20:00:56.557867] finished [D 20:00:56.557890] retval = 2097152 [D 20:00:56.557917] exiting distribute returned less than expected [D 20:00:56.557941] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 144703488 fo 167772160 eof 0 [D 20:00:56.557968] ========================================================= [D 20:00:56.557991] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:56.558017] element 0: offset: 0xbac69000, size: 2097152 [D 20:00:56.558044] post_sr: da13:3346 bh 45 len 32 wr_id 15e72fb8. [D 20:00:56.558080] openib_check_cq: completed send wr_id 15e72fb8. [D 20:00:56.558455] openib_check_cq: completed recv wr_id 15e73b60. [D 20:00:56.558483] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab2608010 rkey 48005300 nic_wr_credit 1. [D 20:00:56.558510] openib_post_rr: da13:3346 bh 69 wr_id 15e73b60. [D 20:00:56.560971] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 0. [D 20:00:56.560998] post_sr: da13:3346 bh 46 len 16 wr_id 15e72fd0. [D 20:00:56.561036] openib_check_cq: completed send wr_id 15e72fd0. [D 20:00:56.561065] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:56.561091] ========================================================= [D 20:00:56.561116] PINT_process_request [D 20:00:56.561140] tiling 7 copies [D 20:00:56.561163] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.561190] lvl 0 el 0 blk 0 by 176160768 [D 20:00:56.561215] to 176160768 ta 167772160 fi 201326592 [D 20:00:56.561240] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.561264] basic type or contiguous data [D 20:00:56.561288] Request disp [D 20:00:56.561313] contig_offset = 176160768 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.561338] calling distribute [D 20:00:56.561362] PINT_distribute [D 20:00:56.561386] of 176160768 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 176160768 exfl 1 [D 20:00:56.561412] begin iteration loff: 176160768 [D 20:00:56.561438] segment exceeds byte limit [D 20:00:56.561462] file being extended [D 20:00:56.561486] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.561511] **********CALL***PROCESS********* [D 20:00:56.561534] segment of 8388608 sz 2097152 [D 20:00:56.561558] PINT_process_request [D 20:00:56.561583] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.561609] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.561634] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.561661] basic type or contiguous data [D 20:00:56.561685] Request disp [D 20:00:56.561709] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.561734] process a segment [D 20:00:56.561758] of 8388608 sz 2097152 [D 20:00:56.561782] add a segment 0 [D 20:00:56.561806] retval = 2097152 [D 20:00:56.561831] exiting distribute returned less than expected [D 20:00:56.561855] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.561882] *****RETURN***FROM***PROCESS***** [D 20:00:56.561905] end iteration [D 20:00:56.561929] done with segments or bytes [D 20:00:56.561953] of 178257920 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.561979] next loff: 178257920 next poff: 178257920 [D 20:00:56.562004] return value 2097152 [D 20:00:56.562028] finished [D 20:00:56.562052] retval = 2097152 [D 20:00:56.562076] exiting distribute returned less than expected [D 20:00:56.562100] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 178257920 fo 201326592 eof 0 [D 20:00:56.562126] ========================================================= [D 20:00:56.562150] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:56.562176] element 0: offset: 0xb8c67000, size: 2097152 [D 20:00:56.562203] post_sr: da13:3346 bh 47 len 32 wr_id 15e72fe8. [D 20:00:56.562233] openib_check_cq: completed recv wr_id 15e73b78. [D 20:00:56.562262] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab2809010 rkey 48005400 nic_wr_credit 1. [D 20:00:56.562289] openib_post_rr: da13:3346 bh 70 wr_id 15e73b78. [D 20:00:56.562318] openib_check_cq: completed send wr_id 15e72fe8. [D 20:00:56.564291] PINT_client_state_machine_test id 22 [D 20:00:56.564749] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 0. [D 20:00:56.564777] post_sr: da13:3346 bh 48 len 16 wr_id 15e73000. [D 20:00:56.564814] openib_check_cq: completed send wr_id 15e73000. [D 20:00:56.564845] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:56.564869] ========================================================= [D 20:00:56.564894] PINT_process_request [D 20:00:56.564918] tiling 8 copies [D 20:00:56.564942] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.564969] lvl 0 el 0 blk 0 by 209715200 [D 20:00:56.564993] to 209715200 ta 201326592 fi 234881024 [D 20:00:56.565018] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.565043] basic type or contiguous data [D 20:00:56.565067] Request disp [D 20:00:56.565091] contig_offset = 209715200 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.565115] calling distribute [D 20:00:56.565139] PINT_distribute [D 20:00:56.565163] of 209715200 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 209715200 exfl 1 [D 20:00:56.565190] begin iteration loff: 209715200 [D 20:00:56.565215] segment exceeds byte limit [D 20:00:56.565238] file being extended [D 20:00:56.565262] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.565287] **********CALL***PROCESS********* [D 20:00:56.565311] segment of 8388608 sz 2097152 [D 20:00:56.565336] PINT_process_request [D 20:00:56.565360] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.565386] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.565411] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.565436] basic type or contiguous data [D 20:00:56.565460] Request disp [D 20:00:56.565484] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.565509] process a segment [D 20:00:56.565533] of 8388608 sz 2097152 [D 20:00:56.565557] add a segment 0 [D 20:00:56.565581] retval = 2097152 [D 20:00:56.565606] exiting distribute returned less than expected [D 20:00:56.565629] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.565655] *****RETURN***FROM***PROCESS***** [D 20:00:56.565682] end iteration [D 20:00:56.565707] done with segments or bytes [D 20:00:56.565731] of 211812352 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.565756] next loff: 211812352 next poff: 211812352 [D 20:00:56.565780] return value 2097152 [D 20:00:56.565805] finished [D 20:00:56.565829] retval = 2097152 [D 20:00:56.565853] exiting distribute returned less than expected [D 20:00:56.565876] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 211812352 fo 234881024 eof 0 [D 20:00:56.565903] ========================================================= [D 20:00:56.565927] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:56.565953] element 0: offset: 0xb6c65000, size: 2097152 [D 20:00:56.565979] post_sr: da13:3346 bh 49 len 32 wr_id 15e73018. [D 20:00:56.566010] openib_check_cq: completed recv wr_id 15e73b90. [D 20:00:56.566039] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab2a0a010 rkey 48005500 nic_wr_credit 1. [D 20:00:56.566065] openib_post_rr: da13:3346 bh 71 wr_id 15e73b90. [D 20:00:56.566092] openib_check_cq: completed send wr_id 15e73018. [D 20:00:56.568529] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 0. [D 20:00:56.568556] post_sr: da13:3346 bh 50 len 16 wr_id 15e73030. [D 20:00:56.568593] openib_check_cq: completed send wr_id 15e73030. [D 20:00:56.568624] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:56.568650] ========================================================= [D 20:00:56.568674] PINT_process_request [D 20:00:56.568698] tiling 9 copies [D 20:00:56.568722] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.568749] lvl 0 el 0 blk 0 by 243269632 [D 20:00:56.568776] to 243269632 ta 234881024 fi 268435456 [D 20:00:56.568801] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.568826] basic type or contiguous data [D 20:00:56.568850] Request disp [D 20:00:56.568874] contig_offset = 243269632 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.568900] calling distribute [D 20:00:56.568923] PINT_distribute [D 20:00:56.568947] of 243269632 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 243269632 exfl 1 [D 20:00:56.568973] begin iteration loff: 243269632 [D 20:00:56.568999] segment exceeds byte limit [D 20:00:56.569023] file being extended [D 20:00:56.569046] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.569071] **********CALL***PROCESS********* [D 20:00:56.569095] segment of 8388608 sz 2097152 [D 20:00:56.569120] PINT_process_request [D 20:00:56.569144] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.569169] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.569194] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.569219] basic type or contiguous data [D 20:00:56.569243] Request disp [D 20:00:56.569267] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.569293] process a segment [D 20:00:56.569316] of 8388608 sz 2097152 [D 20:00:56.569340] add a segment 0 [D 20:00:56.569364] retval = 2097152 [D 20:00:56.569389] exiting distribute returned less than expected [D 20:00:56.569413] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.569438] *****RETURN***FROM***PROCESS***** [D 20:00:56.569462] end iteration [D 20:00:56.569486] done with segments or bytes [D 20:00:56.569510] of 245366784 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.569536] next loff: 245366784 next poff: 245366784 [D 20:00:56.569561] return value 2097152 [D 20:00:56.569584] finished [D 20:00:56.569608] retval = 2097152 [D 20:00:56.569632] exiting distribute returned less than expected [D 20:00:56.569656] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 245366784 fo 268435456 eof 0 [D 20:00:56.569684] ========================================================= [D 20:00:56.569709] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:56.569734] element 0: offset: 0xb4c63000, size: 2097152 [D 20:00:56.569762] post_sr: da13:3346 bh 51 len 32 wr_id 15e73048. [D 20:00:56.569791] openib_check_cq: completed recv wr_id 15e73ba8. [D 20:00:56.569820] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab2c0b010 rkey 48005600 nic_wr_credit 1. [D 20:00:56.569847] openib_post_rr: da13:3346 bh 72 wr_id 15e73ba8. [D 20:00:56.569873] openib_check_cq: completed send wr_id 15e73048. [D 20:00:56.572309] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 0. [D 20:00:56.572337] post_sr: da13:3346 bh 52 len 16 wr_id 15e73060. [D 20:00:56.572373] openib_check_cq: completed send wr_id 15e73060. [D 20:00:56.572403] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:56.572429] ========================================================= [D 20:00:56.572454] PINT_process_request [D 20:00:56.572478] tiling 10 copies [D 20:00:56.572502] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.572528] lvl 0 el 0 blk 0 by 276824064 [D 20:00:56.572553] to 276824064 ta 268435456 fi 301989888 [D 20:00:56.572578] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.572603] basic type or contiguous data [D 20:00:56.572627] Request disp [D 20:00:56.572650] contig_offset = 276824064 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.572675] calling distribute [D 20:00:56.572700] PINT_distribute [D 20:00:56.572724] of 276824064 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 276824064 exfl 1 [D 20:00:56.572750] begin iteration loff: 276824064 [D 20:00:56.572775] segment exceeds byte limit [D 20:00:56.572799] file being extended [D 20:00:56.572823] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.572848] **********CALL***PROCESS********* [D 20:00:56.572872] segment of 8388608 sz 2097152 [D 20:00:56.572898] PINT_process_request [D 20:00:56.572923] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.572949] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.572974] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.572999] basic type or contiguous data [D 20:00:56.573022] Request disp [D 20:00:56.573046] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.573071] process a segment [D 20:00:56.573095] of 8388608 sz 2097152 [D 20:00:56.573119] add a segment 0 [D 20:00:56.573143] retval = 2097152 [D 20:00:56.573167] exiting distribute returned less than expected [D 20:00:56.573191] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.573217] *****RETURN***FROM***PROCESS***** [D 20:00:56.573242] end iteration [D 20:00:56.573265] done with segments or bytes [D 20:00:56.573288] of 278921216 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.573314] next loff: 278921216 next poff: 278921216 [D 20:00:56.573339] return value 2097152 [D 20:00:56.573364] finished [D 20:00:56.573387] retval = 2097152 [D 20:00:56.573411] exiting distribute returned less than expected [D 20:00:56.573435] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 278921216 fo 301989888 eof 0 [D 20:00:56.573461] ========================================================= [D 20:00:56.573485] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:56.573511] element 0: offset: 0xb2c61000, size: 2097152 [D 20:00:56.573537] post_sr: da13:3346 bh 53 len 32 wr_id 15e73078. [D 20:00:56.573567] openib_check_cq: completed recv wr_id 15e73bc0. [D 20:00:56.573596] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab2e0c010 rkey 48005700 nic_wr_credit 1. [D 20:00:56.573622] openib_post_rr: da13:3346 bh 73 wr_id 15e73bc0. [D 20:00:56.573649] openib_check_cq: completed send wr_id 15e73078. [D 20:00:56.574323] PINT_client_state_machine_test id 22 [D 20:00:56.576082] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 0. [D 20:00:56.576109] post_sr: da13:3346 bh 54 len 16 wr_id 15e73090. [D 20:00:56.576147] openib_check_cq: completed send wr_id 15e73090. [D 20:00:56.576177] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:56.576203] ========================================================= [D 20:00:56.576227] PINT_process_request [D 20:00:56.576251] tiling 11 copies [D 20:00:56.576275] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.576301] lvl 0 el 0 blk 0 by 310378496 [D 20:00:56.576326] to 310378496 ta 301989888 fi 335544320 [D 20:00:56.576351] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.576375] basic type or contiguous data [D 20:00:56.576399] Request disp [D 20:00:56.576423] contig_offset = 310378496 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.576448] calling distribute [D 20:00:56.576472] PINT_distribute [D 20:00:56.576496] of 310378496 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 310378496 exfl 1 [D 20:00:56.576522] begin iteration loff: 310378496 [D 20:00:56.576547] segment exceeds byte limit [D 20:00:56.576571] file being extended [D 20:00:56.576595] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.576620] **********CALL***PROCESS********* [D 20:00:56.576643] segment of 8388608 sz 2097152 [D 20:00:56.576667] PINT_process_request [D 20:00:56.576691] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.576718] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.576742] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.576766] basic type or contiguous data [D 20:00:56.576790] Request disp [D 20:00:56.576814] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.576839] process a segment [D 20:00:56.576863] of 8388608 sz 2097152 [D 20:00:56.576888] add a segment 0 [D 20:00:56.576911] retval = 2097152 [D 20:00:56.576935] exiting distribute returned less than expected [D 20:00:56.576959] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.576988] *****RETURN***FROM***PROCESS***** [D 20:00:56.577013] end iteration [D 20:00:56.577036] done with segments or bytes [D 20:00:56.577060] of 312475648 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.577086] next loff: 312475648 next poff: 312475648 [D 20:00:56.577111] return value 2097152 [D 20:00:56.577135] finished [D 20:00:56.577158] retval = 2097152 [D 20:00:56.577182] exiting distribute returned less than expected [D 20:00:56.577206] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 312475648 fo 335544320 eof 0 [D 20:00:56.577232] ========================================================= [D 20:00:56.577257] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:56.577281] element 0: offset: 0xb0c5f000, size: 2097152 [D 20:00:56.577309] post_sr: da13:3346 bh 55 len 32 wr_id 15e730a8. [D 20:00:56.577344] openib_check_cq: completed send wr_id 15e730a8. [D 20:00:56.579794] openib_check_cq: completed recv wr_id 15e73bd8. [D 20:00:56.579823] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaab300e010 rkey c8005800 nic_wr_credit 1. [D 20:00:56.579851] openib_post_rr: da13:3346 bh 74 wr_id 15e73bd8. [D 20:00:56.582310] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 0. [D 20:00:56.582338] post_sr: da13:3346 bh 56 len 16 wr_id 15e730c0. [D 20:00:56.582374] openib_check_cq: completed send wr_id 15e730c0. [D 20:00:56.582405] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:56.582430] ========================================================= [D 20:00:56.582454] PINT_process_request [D 20:00:56.582478] tiling 12 copies [D 20:00:56.582503] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.582529] lvl 0 el 0 blk 0 by 343932928 [D 20:00:56.582553] to 343932928 ta 335544320 fi 369098752 [D 20:00:56.582578] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.582603] basic type or contiguous data [D 20:00:56.582627] Request disp [D 20:00:56.582651] contig_offset = 343932928 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.582675] calling distribute [D 20:00:56.582699] PINT_distribute [D 20:00:56.582723] of 343932928 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 343932928 exfl 1 [D 20:00:56.582750] begin iteration loff: 343932928 [D 20:00:56.582775] segment exceeds byte limit [D 20:00:56.582798] file being extended [D 20:00:56.582822] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.582847] **********CALL***PROCESS********* [D 20:00:56.582871] segment of 8388608 sz 2097152 [D 20:00:56.582896] PINT_process_request [D 20:00:56.582920] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.582945] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.582970] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.582995] basic type or contiguous data [D 20:00:56.583019] Request disp [D 20:00:56.583043] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.583067] process a segment [D 20:00:56.583091] of 8388608 sz 2097152 [D 20:00:56.583116] add a segment 0 [D 20:00:56.583140] retval = 2097152 [D 20:00:56.583164] exiting distribute returned less than expected [D 20:00:56.583188] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.583214] *****RETURN***FROM***PROCESS***** [D 20:00:56.583238] end iteration [D 20:00:56.583262] done with segments or bytes [D 20:00:56.583286] of 346030080 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.583312] next loff: 346030080 next poff: 346030080 [D 20:00:56.583336] return value 2097152 [D 20:00:56.583360] finished [D 20:00:56.583384] retval = 2097152 [D 20:00:56.583408] exiting distribute returned less than expected [D 20:00:56.583432] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 346030080 fo 369098752 eof 0 [D 20:00:56.583457] ========================================================= [D 20:00:56.583482] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:56.583510] element 0: offset: 0xaec5d000, size: 2097152 [D 20:00:56.583538] post_sr: da13:3346 bh 57 len 32 wr_id 15e730d8. [D 20:00:56.583573] openib_check_cq: completed send wr_id 15e730d8. [D 20:00:56.584344] openib_check_cq: completed recv wr_id 15e73bf0. [D 20:00:56.584373] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaab320f010 rkey c8005900 nic_wr_credit 1. [D 20:00:56.584401] openib_post_rr: da13:3346 bh 75 wr_id 15e73bf0. [D 20:00:56.584431] PINT_client_state_machine_test id 22 [D 20:00:56.586860] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 0. [D 20:00:56.586888] post_sr: da13:3346 bh 58 len 16 wr_id 15e730f0. [D 20:00:56.586925] openib_check_cq: completed send wr_id 15e730f0. [D 20:00:56.586956] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:56.586981] ========================================================= [D 20:00:56.587005] PINT_process_request [D 20:00:56.587029] tiling 13 copies [D 20:00:56.587054] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.587080] lvl 0 el 0 blk 0 by 377487360 [D 20:00:56.587104] to 377487360 ta 369098752 fi 402653184 [D 20:00:56.587129] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.587154] basic type or contiguous data [D 20:00:56.587178] Request disp [D 20:00:56.587202] contig_offset = 377487360 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.587226] calling distribute [D 20:00:56.587251] PINT_distribute [D 20:00:56.587274] of 377487360 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 377487360 exfl 1 [D 20:00:56.587301] begin iteration loff: 377487360 [D 20:00:56.587326] segment exceeds byte limit [D 20:00:56.587349] file being extended [D 20:00:56.587373] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.587398] **********CALL***PROCESS********* [D 20:00:56.587422] segment of 8388608 sz 2097152 [D 20:00:56.587447] PINT_process_request [D 20:00:56.587471] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.587496] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.587521] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.587546] basic type or contiguous data [D 20:00:56.587570] Request disp [D 20:00:56.587594] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.587620] process a segment [D 20:00:56.587644] of 8388608 sz 2097152 [D 20:00:56.587672] add a segment 0 [D 20:00:56.587698] retval = 2097152 [D 20:00:56.587727] exiting distribute returned less than expected [D 20:00:56.587754] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.587783] *****RETURN***FROM***PROCESS***** [D 20:00:56.587807] end iteration [D 20:00:56.587831] done with segments or bytes [D 20:00:56.587855] of 379584512 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.587881] next loff: 379584512 next poff: 379584512 [D 20:00:56.587905] return value 2097152 [D 20:00:56.587929] finished [D 20:00:56.587953] retval = 2097152 [D 20:00:56.587977] exiting distribute returned less than expected [D 20:00:56.588001] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 379584512 fo 402653184 eof 0 [D 20:00:56.588027] ========================================================= [D 20:00:56.588051] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:56.588076] element 0: offset: 0xacc5b000, size: 2097152 [D 20:00:56.588104] post_sr: da13:3346 bh 59 len 32 wr_id 15e73108. [D 20:00:56.588133] openib_check_cq: completed recv wr_id 15e73c08. [D 20:00:56.588162] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaab3410010 rkey c8005a00 nic_wr_credit 1. [D 20:00:56.588190] openib_post_rr: da13:3346 bh 76 wr_id 15e73c08. [D 20:00:56.588216] openib_check_cq: completed send wr_id 15e73108. [D 20:00:56.590652] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 0. [D 20:00:56.590680] post_sr: da13:3346 bh 60 len 16 wr_id 15e73120. [D 20:00:56.590721] openib_check_cq: completed send wr_id 15e73120. [D 20:00:56.590751] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:00:56.590777] ========================================================= [D 20:00:56.590802] PINT_process_request [D 20:00:56.590826] tiling 14 copies [D 20:00:56.590850] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.590876] lvl 0 el 0 blk 0 by 411041792 [D 20:00:56.590901] to 411041792 ta 402653184 fi 436207616 [D 20:00:56.590926] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.590950] basic type or contiguous data [D 20:00:56.590975] Request disp [D 20:00:56.590999] contig_offset = 411041792 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.591024] calling distribute [D 20:00:56.591048] PINT_distribute [D 20:00:56.591072] of 411041792 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 411041792 exfl 1 [D 20:00:56.591097] begin iteration loff: 411041792 [D 20:00:56.591122] segment exceeds byte limit [D 20:00:56.591147] file being extended [D 20:00:56.591171] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.591196] **********CALL***PROCESS********* [D 20:00:56.591219] segment of 8388608 sz 2097152 [D 20:00:56.591243] PINT_process_request [D 20:00:56.591267] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.591294] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.591319] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.591342] basic type or contiguous data [D 20:00:56.591367] Request disp [D 20:00:56.591391] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.591416] process a segment [D 20:00:56.591440] of 8388608 sz 2097152 [D 20:00:56.591464] add a segment 0 [D 20:00:56.591487] retval = 2097152 [D 20:00:56.591511] exiting distribute returned less than expected [D 20:00:56.591536] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.591562] *****RETURN***FROM***PROCESS***** [D 20:00:56.591586] end iteration [D 20:00:56.591609] done with segments or bytes [D 20:00:56.591633] of 413138944 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.591661] next loff: 413138944 next poff: 413138944 [D 20:00:56.591696] return value 2097152 [D 20:00:56.591732] finished [D 20:00:56.591762] retval = 2097152 [D 20:00:56.591790] exiting distribute returned less than expected [D 20:00:56.591817] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 413138944 fo 436207616 eof 0 [D 20:00:56.591848] ========================================================= [D 20:00:56.591875] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:00:56.591900] element 0: offset: 0xaac59000, size: 2097152 [D 20:00:56.591927] post_sr: da13:3346 bh 61 len 32 wr_id 15e73138. [D 20:00:56.591958] openib_check_cq: completed recv wr_id 15e73c20. [D 20:00:56.591986] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab3611010 rkey c8005b00 nic_wr_credit 1. [D 20:00:56.592013] openib_post_rr: da13:3346 bh 77 wr_id 15e73c20. [D 20:00:56.592040] openib_check_cq: completed send wr_id 15e73138. [D 20:00:56.594462] PINT_client_state_machine_test id 22 [D 20:00:56.594491] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 0. [D 20:00:56.594518] post_sr: da13:3346 bh 62 len 16 wr_id 15e73150. [D 20:00:56.594554] openib_check_cq: completed send wr_id 15e73150. [D 20:00:56.594585] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:00:56.594611] ========================================================= [D 20:00:56.594635] PINT_process_request [D 20:00:56.594659] tiling 15 copies [D 20:00:56.594683] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.594710] lvl 0 el 0 blk 0 by 444596224 [D 20:00:56.594734] to 444596224 ta 436207616 fi 469762048 [D 20:00:56.594758] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:56.594783] basic type or contiguous data [D 20:00:56.594807] Request disp [D 20:00:56.594831] contig_offset = 444596224 contig_size = 58720256 lvl_flag = 1 [D 20:00:56.594861] calling distribute [D 20:00:56.594885] PINT_distribute [D 20:00:56.594908] of 444596224 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 444596224 exfl 1 [D 20:00:56.594935] begin iteration loff: 444596224 [D 20:00:56.594960] segment exceeds byte limit [D 20:00:56.594984] file being extended [D 20:00:56.595007] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:56.595032] **********CALL***PROCESS********* [D 20:00:56.595056] segment of 8388608 sz 2097152 [D 20:00:56.595080] PINT_process_request [D 20:00:56.595104] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.595130] lvl 0 el 0 blk 0 by 8388608 [D 20:00:56.595155] to 8388608 ta 8388608 fi 10485760 [D 20:00:56.595180] basic type or contiguous data [D 20:00:56.595203] Request disp [D 20:00:56.595227] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:56.595252] process a segment [D 20:00:56.595275] of 8388608 sz 2097152 [D 20:00:56.595299] add a segment 0 [D 20:00:56.595323] retval = 2097152 [D 20:00:56.595347] exiting distribute returned less than expected [D 20:00:56.595372] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:56.595397] *****RETURN***FROM***PROCESS***** [D 20:00:56.595421] end iteration [D 20:00:56.595445] done with segments or bytes [D 20:00:56.595469] of 446693376 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.595495] next loff: 446693376 next poff: 446693376 [D 20:00:56.595519] return value 2097152 [D 20:00:56.595543] finished [D 20:00:56.595567] retval = 2097152 [D 20:00:56.595591] exiting distribute returned less than expected [D 20:00:56.595615] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 446693376 fo 469762048 eof 0 [D 20:00:56.595641] ========================================================= [D 20:00:56.595667] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:00:56.595697] element 0: offset: 0xa8c57000, size: 2097152 [D 20:00:56.595727] post_sr: da13:3346 bh 63 len 32 wr_id 15e73168. [D 20:00:56.595761] openib_check_cq: completed recv wr_id 15e73c38. [D 20:00:56.595791] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab3812010 rkey c8005c00 nic_wr_credit 1. [D 20:00:56.595818] openib_post_rr: da13:3346 bh 78 wr_id 15e73c38. [D 20:00:56.595845] openib_check_cq: completed send wr_id 15e73168. [D 20:00:56.598281] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 0. [D 20:00:56.598309] post_sr: da13:3346 bh 64 len 16 wr_id 15e73180. [D 20:00:56.598345] openib_check_cq: completed send wr_id 15e73180. [D 20:00:56.598376] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:56.598402] ========================================================= [D 20:00:56.598426] PINT_process_request [D 20:00:56.598450] tiling 2 copies [D 20:00:56.598474] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.598500] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.598525] to 10485760 ta 0 fi 33554432 [D 20:00:56.598549] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.598574] basic type or contiguous data [D 20:00:56.598598] Request disp [D 20:00:56.598621] contig_offset = 10485760 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.598646] calling distribute [D 20:00:56.598670] PINT_distribute [D 20:00:56.598693] of 10485760 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 10485760 exfl 1 [D 20:00:56.598720] begin iteration loff: 10485760 [D 20:00:56.598745] segment exceeds byte limit [D 20:00:56.598769] file being extended [D 20:00:56.598793] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.598817] **********CALL***PROCESS********* [D 20:00:56.598841] segment of 10485760 sz 2097152 [D 20:00:56.598866] PINT_process_request [D 20:00:56.598890] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.598917] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.598940] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.598968] basic type or contiguous data [D 20:00:56.598992] Request disp [D 20:00:56.599017] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.599041] process a segment [D 20:00:56.599065] of 10485760 sz 2097152 [D 20:00:56.599089] add a segment 0 [D 20:00:56.599113] retval = 2097152 [D 20:00:56.599137] exiting distribute returned less than expected [D 20:00:56.599162] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.599187] *****RETURN***FROM***PROCESS***** [D 20:00:56.599212] end iteration [D 20:00:56.599236] done with segments or bytes [D 20:00:56.599259] of 12582912 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.599286] next loff: 12582912 next poff: 12582912 [D 20:00:56.599309] return value 2097152 [D 20:00:56.599334] finished [D 20:00:56.599358] retval = 2097152 [D 20:00:56.599382] exiting distribute returned less than expected [D 20:00:56.599406] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 12582912 fo 33554432 eof 0 [D 20:00:56.599431] ========================================================= [D 20:00:56.599456] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:56.599481] element 0: offset: 0xc2e71000, size: 2097152 [D 20:00:56.599508] post_sr: da13:3346 bh 65 len 32 wr_id 15e73198. [D 20:00:56.599538] openib_check_cq: completed recv wr_id 15e73c50. [D 20:00:56.599567] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab3a13010 rkey c8005d00 nic_wr_credit 1. [D 20:00:56.599594] openib_post_rr: da13:3346 bh 79 wr_id 15e73c50. [D 20:00:56.599620] openib_check_cq: completed send wr_id 15e73198. [D 20:00:56.602053] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 0. [D 20:00:56.602081] post_sr: da13:3346 bh 66 len 16 wr_id 15e731b0. [D 20:00:56.602118] openib_check_cq: completed send wr_id 15e731b0. [D 20:00:56.602148] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:56.602174] ========================================================= [D 20:00:56.602198] PINT_process_request [D 20:00:56.602223] tiling 3 copies [D 20:00:56.602247] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.602272] lvl 0 el 0 blk 0 by 44040192 [D 20:00:56.602298] to 44040192 ta 33554432 fi 67108864 [D 20:00:56.602322] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.602347] basic type or contiguous data [D 20:00:56.602371] Request disp [D 20:00:56.602396] contig_offset = 44040192 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.602420] calling distribute [D 20:00:56.602444] PINT_distribute [D 20:00:56.602468] of 44040192 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 44040192 exfl 1 [D 20:00:56.602495] begin iteration loff: 44040192 [D 20:00:56.602519] segment exceeds byte limit [D 20:00:56.602543] file being extended [D 20:00:56.602567] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.602593] **********CALL***PROCESS********* [D 20:00:56.602617] segment of 10485760 sz 2097152 [D 20:00:56.602641] PINT_process_request [D 20:00:56.602665] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.602691] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.602716] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.602741] basic type or contiguous data [D 20:00:56.602765] Request disp [D 20:00:56.602788] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.602813] process a segment [D 20:00:56.602837] of 10485760 sz 2097152 [D 20:00:56.602862] add a segment 0 [D 20:00:56.602886] retval = 2097152 [D 20:00:56.602910] exiting distribute returned less than expected [D 20:00:56.602934] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.602960] *****RETURN***FROM***PROCESS***** [D 20:00:56.602984] end iteration [D 20:00:56.603008] done with segments or bytes [D 20:00:56.603032] of 46137344 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.603057] next loff: 46137344 next poff: 46137344 [D 20:00:56.603085] return value 2097152 [D 20:00:56.603110] finished [D 20:00:56.603134] retval = 2097152 [D 20:00:56.603157] exiting distribute returned less than expected [D 20:00:56.603181] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 46137344 fo 67108864 eof 0 [D 20:00:56.603208] ========================================================= [D 20:00:56.603233] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:56.603258] element 0: offset: 0xc0e6f000, size: 2097152 [D 20:00:56.603284] post_sr: da13:3346 bh 67 len 32 wr_id 15e731c8. [D 20:00:56.603315] openib_check_cq: completed recv wr_id 15e73c68. [D 20:00:56.603343] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab3c14010 rkey c8005e00 nic_wr_credit 1. [D 20:00:56.603370] openib_post_rr: da13:3346 bh 80 wr_id 15e73c68. [D 20:00:56.603397] openib_check_cq: completed send wr_id 15e731c8. [D 20:00:56.604491] PINT_client_state_machine_test id 22 [D 20:00:56.605832] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 0. [D 20:00:56.605858] post_sr: da13:3346 bh 68 len 16 wr_id 15e731e0. [D 20:00:56.605895] openib_check_cq: completed send wr_id 15e731e0. [D 20:00:56.605926] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:56.605952] ========================================================= [D 20:00:56.605976] PINT_process_request [D 20:00:56.605999] tiling 4 copies [D 20:00:56.606024] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.606050] lvl 0 el 0 blk 0 by 77594624 [D 20:00:56.606075] to 77594624 ta 67108864 fi 100663296 [D 20:00:56.606100] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.606124] basic type or contiguous data [D 20:00:56.606148] Request disp [D 20:00:56.606172] contig_offset = 77594624 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.606197] calling distribute [D 20:00:56.606221] PINT_distribute [D 20:00:56.606245] of 77594624 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 77594624 exfl 1 [D 20:00:56.606270] begin iteration loff: 77594624 [D 20:00:56.606295] segment exceeds byte limit [D 20:00:56.606319] file being extended [D 20:00:56.606343] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.606368] **********CALL***PROCESS********* [D 20:00:56.606391] segment of 10485760 sz 2097152 [D 20:00:56.606416] PINT_process_request [D 20:00:56.606440] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.606466] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.606491] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.606515] basic type or contiguous data [D 20:00:56.606539] Request disp [D 20:00:56.606563] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.606588] process a segment [D 20:00:56.606612] of 10485760 sz 2097152 [D 20:00:56.606637] add a segment 0 [D 20:00:56.606660] retval = 2097152 [D 20:00:56.606684] exiting distribute returned less than expected [D 20:00:56.606708] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.606734] *****RETURN***FROM***PROCESS***** [D 20:00:56.606759] end iteration [D 20:00:56.606782] done with segments or bytes [D 20:00:56.606805] of 79691776 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.606831] next loff: 79691776 next poff: 79691776 [D 20:00:56.606856] return value 2097152 [D 20:00:56.606881] finished [D 20:00:56.606904] retval = 2097152 [D 20:00:56.606927] exiting distribute returned less than expected [D 20:00:56.606951] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 79691776 fo 100663296 eof 0 [D 20:00:56.606978] ========================================================= [D 20:00:56.607002] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:56.607028] element 0: offset: 0xbee6d000, size: 2097152 [D 20:00:56.607054] post_sr: da13:3346 bh 69 len 32 wr_id 15e731f8. [D 20:00:56.607089] openib_check_cq: completed send wr_id 15e731f8. [D 20:00:56.610898] openib_check_cq: completed recv wr_id 15e73c80. [D 20:00:56.610927] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaab3e15010 rkey c8005f00 nic_wr_credit 1. [D 20:00:56.610955] openib_post_rr: da13:3346 bh 81 wr_id 15e73c80. [D 20:00:56.613413] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 0. [D 20:00:56.613441] post_sr: da13:3346 bh 70 len 16 wr_id 15e73210. [D 20:00:56.613477] openib_check_cq: completed send wr_id 15e73210. [D 20:00:56.613508] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:56.613533] ========================================================= [D 20:00:56.613557] PINT_process_request [D 20:00:56.613582] tiling 5 copies [D 20:00:56.613606] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.613632] lvl 0 el 0 blk 0 by 111149056 [D 20:00:56.613657] to 111149056 ta 100663296 fi 134217728 [D 20:00:56.613684] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.613709] basic type or contiguous data [D 20:00:56.613733] Request disp [D 20:00:56.613757] contig_offset = 111149056 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.613781] calling distribute [D 20:00:56.613806] PINT_distribute [D 20:00:56.613829] of 111149056 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 111149056 exfl 1 [D 20:00:56.613856] begin iteration loff: 111149056 [D 20:00:56.613881] segment exceeds byte limit [D 20:00:56.613904] file being extended [D 20:00:56.613928] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.613953] **********CALL***PROCESS********* [D 20:00:56.613977] segment of 10485760 sz 2097152 [D 20:00:56.614001] PINT_process_request [D 20:00:56.614025] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.614051] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.614076] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.614101] basic type or contiguous data [D 20:00:56.614125] Request disp [D 20:00:56.614149] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.614173] process a segment [D 20:00:56.614197] of 10485760 sz 2097152 [D 20:00:56.614221] add a segment 0 [D 20:00:56.614245] retval = 2097152 [D 20:00:56.614269] exiting distribute returned less than expected [D 20:00:56.614293] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.614319] *****RETURN***FROM***PROCESS***** [D 20:00:56.614344] end iteration [D 20:00:56.614367] done with segments or bytes [D 20:00:56.614391] of 113246208 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.614417] next loff: 113246208 next poff: 113246208 [D 20:00:56.614441] return value 2097152 [D 20:00:56.614466] finished [D 20:00:56.614489] retval = 2097152 [D 20:00:56.614514] exiting distribute returned less than expected [D 20:00:56.614538] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 113246208 fo 134217728 eof 0 [D 20:00:56.614563] ========================================================= [D 20:00:56.614588] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:56.614613] element 0: offset: 0xbce6b000, size: 2097152 [D 20:00:56.614640] post_sr: da13:3346 bh 71 len 32 wr_id 15e73228. [D 20:00:56.614669] PINT_client_state_machine_test id 22 [D 20:00:56.614698] openib_check_cq: completed recv wr_id 15e73c98. [D 20:00:56.614727] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab4016010 rkey 48006000 nic_wr_credit 1. [D 20:00:56.614753] openib_post_rr: da13:3346 bh 82 wr_id 15e73c98. [D 20:00:56.614780] openib_check_cq: completed send wr_id 15e73228. [D 20:00:56.617215] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 0. [D 20:00:56.617243] post_sr: da13:3346 bh 72 len 16 wr_id 15e73240. [D 20:00:56.617280] openib_check_cq: completed send wr_id 15e73240. [D 20:00:56.617310] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:56.617336] ========================================================= [D 20:00:56.617361] PINT_process_request [D 20:00:56.617387] tiling 6 copies [D 20:00:56.617412] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.617438] lvl 0 el 0 blk 0 by 144703488 [D 20:00:56.617463] to 144703488 ta 134217728 fi 167772160 [D 20:00:56.617487] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.617512] basic type or contiguous data [D 20:00:56.617536] Request disp [D 20:00:56.617560] contig_offset = 144703488 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.617586] calling distribute [D 20:00:56.617609] PINT_distribute [D 20:00:56.617635] of 144703488 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 144703488 exfl 1 [D 20:00:56.617664] begin iteration loff: 144703488 [D 20:00:56.617689] segment exceeds byte limit [D 20:00:56.617713] file being extended [D 20:00:56.617737] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.617762] **********CALL***PROCESS********* [D 20:00:56.617786] segment of 10485760 sz 2097152 [D 20:00:56.617810] PINT_process_request [D 20:00:56.617834] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.617860] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.617885] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.617910] basic type or contiguous data [D 20:00:56.617934] Request disp [D 20:00:56.617958] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.617983] process a segment [D 20:00:56.618007] of 10485760 sz 2097152 [D 20:00:56.618031] add a segment 0 [D 20:00:56.618055] retval = 2097152 [D 20:00:56.618079] exiting distribute returned less than expected [D 20:00:56.618103] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.618129] *****RETURN***FROM***PROCESS***** [D 20:00:56.618154] end iteration [D 20:00:56.618178] done with segments or bytes [D 20:00:56.618202] of 146800640 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.618227] next loff: 146800640 next poff: 146800640 [D 20:00:56.618252] return value 2097152 [D 20:00:56.618276] finished [D 20:00:56.618300] retval = 2097152 [D 20:00:56.618325] exiting distribute returned less than expected [D 20:00:56.618349] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 146800640 fo 167772160 eof 0 [D 20:00:56.618375] ========================================================= [D 20:00:56.618399] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:56.618425] element 0: offset: 0xbae69000, size: 2097152 [D 20:00:56.618452] post_sr: da13:3346 bh 73 len 32 wr_id 15e73258. [D 20:00:56.618482] openib_check_cq: completed recv wr_id 15e73cb0. [D 20:00:56.618510] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab4217010 rkey 48006100 nic_wr_credit 1. [D 20:00:56.618537] openib_post_rr: da13:3346 bh 83 wr_id 15e73cb0. [D 20:00:56.618563] openib_check_cq: completed send wr_id 15e73258. [D 20:00:56.621000] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 0. [D 20:00:56.621027] post_sr: da13:3346 bh 74 len 16 wr_id 15e73270. [D 20:00:56.621064] openib_check_cq: completed send wr_id 15e73270. [D 20:00:56.621094] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:56.621120] ========================================================= [D 20:00:56.621144] PINT_process_request [D 20:00:56.621169] tiling 7 copies [D 20:00:56.621192] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.621219] lvl 0 el 0 blk 0 by 178257920 [D 20:00:56.621244] to 178257920 ta 167772160 fi 201326592 [D 20:00:56.621269] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.621294] basic type or contiguous data [D 20:00:56.621317] Request disp [D 20:00:56.621341] contig_offset = 178257920 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.621367] calling distribute [D 20:00:56.621391] PINT_distribute [D 20:00:56.621415] of 178257920 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 178257920 exfl 1 [D 20:00:56.621441] begin iteration loff: 178257920 [D 20:00:56.621466] segment exceeds byte limit [D 20:00:56.621493] file being extended [D 20:00:56.621517] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.621541] **********CALL***PROCESS********* [D 20:00:56.621565] segment of 10485760 sz 2097152 [D 20:00:56.621589] PINT_process_request [D 20:00:56.621613] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.621640] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.621667] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.621692] basic type or contiguous data [D 20:00:56.621716] Request disp [D 20:00:56.621741] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.621766] process a segment [D 20:00:56.621789] of 10485760 sz 2097152 [D 20:00:56.621813] add a segment 0 [D 20:00:56.621837] retval = 2097152 [D 20:00:56.621861] exiting distribute returned less than expected [D 20:00:56.621886] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.621912] *****RETURN***FROM***PROCESS***** [D 20:00:56.621935] end iteration [D 20:00:56.621959] done with segments or bytes [D 20:00:56.621983] of 180355072 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.622009] next loff: 180355072 next poff: 180355072 [D 20:00:56.622034] return value 2097152 [D 20:00:56.622058] finished [D 20:00:56.622081] retval = 2097152 [D 20:00:56.622105] exiting distribute returned less than expected [D 20:00:56.622130] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 180355072 fo 201326592 eof 0 [D 20:00:56.622156] ========================================================= [D 20:00:56.622180] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:56.622205] element 0: offset: 0xb8e67000, size: 2097152 [D 20:00:56.622232] post_sr: da13:3346 bh 75 len 32 wr_id 15e73288. [D 20:00:56.622268] openib_check_cq: completed send wr_id 15e73288. [D 20:00:56.623169] openib_check_cq: completed recv wr_id 15e73cc8. [D 20:00:56.623198] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab4418010 rkey 48006200 nic_wr_credit 1. [D 20:00:56.623225] openib_post_rr: da13:3346 bh 84 wr_id 15e73cc8. [D 20:00:56.624699] PINT_client_state_machine_test id 22 [D 20:00:56.625684] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 0. [D 20:00:56.625712] post_sr: da13:3346 bh 76 len 16 wr_id 15e732a0. [D 20:00:56.625748] openib_check_cq: completed send wr_id 15e732a0. [D 20:00:56.625779] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:56.625804] ========================================================= [D 20:00:56.625829] PINT_process_request [D 20:00:56.625853] tiling 8 copies [D 20:00:56.625878] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.625904] lvl 0 el 0 blk 0 by 211812352 [D 20:00:56.625928] to 211812352 ta 201326592 fi 234881024 [D 20:00:56.625953] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.625978] basic type or contiguous data [D 20:00:56.626002] Request disp [D 20:00:56.626026] contig_offset = 211812352 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.626050] calling distribute [D 20:00:56.626075] PINT_distribute [D 20:00:56.626099] of 211812352 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 211812352 exfl 1 [D 20:00:56.626125] begin iteration loff: 211812352 [D 20:00:56.626151] segment exceeds byte limit [D 20:00:56.626174] file being extended [D 20:00:56.626198] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.626222] **********CALL***PROCESS********* [D 20:00:56.626246] segment of 10485760 sz 2097152 [D 20:00:56.626271] PINT_process_request [D 20:00:56.626295] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.626321] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.626346] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.626371] basic type or contiguous data [D 20:00:56.626395] Request disp [D 20:00:56.626419] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.626443] process a segment [D 20:00:56.626470] of 10485760 sz 2097152 [D 20:00:56.626495] add a segment 0 [D 20:00:56.626519] retval = 2097152 [D 20:00:56.626543] exiting distribute returned less than expected [D 20:00:56.626566] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.626593] *****RETURN***FROM***PROCESS***** [D 20:00:56.626617] end iteration [D 20:00:56.626641] done with segments or bytes [D 20:00:56.626665] of 213909504 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.626690] next loff: 213909504 next poff: 213909504 [D 20:00:56.626715] return value 2097152 [D 20:00:56.626740] finished [D 20:00:56.626764] retval = 2097152 [D 20:00:56.626788] exiting distribute returned less than expected [D 20:00:56.626811] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 213909504 fo 234881024 eof 0 [D 20:00:56.626838] ========================================================= [D 20:00:56.626862] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:56.626888] element 0: offset: 0xb6e65000, size: 2097152 [D 20:00:56.626914] post_sr: da13:3346 bh 77 len 32 wr_id 15e732b8. [D 20:00:56.626945] openib_check_cq: completed recv wr_id 15e73ce0. [D 20:00:56.626974] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab4619010 rkey 48006300 nic_wr_credit 1. [D 20:00:56.627000] openib_post_rr: da13:3346 bh 85 wr_id 15e73ce0. [D 20:00:56.627027] openib_check_cq: completed send wr_id 15e732b8. [D 20:00:56.629461] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 0. [D 20:00:56.629488] post_sr: da13:3346 bh 78 len 16 wr_id 15e732d0. [D 20:00:56.629526] openib_check_cq: completed send wr_id 15e732d0. [D 20:00:56.629552] openib_check_cq: completed recv wr_id 15e73cf8. [D 20:00:56.629580] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab481a010 rkey 48006400 nic_wr_credit 1. [D 20:00:56.629607] openib_post_rr: da13:3346 bh 86 wr_id 15e73cf8. [D 20:00:56.629637] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:56.629666] ========================================================= [D 20:00:56.629691] PINT_process_request [D 20:00:56.629714] tiling 9 copies [D 20:00:56.629739] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.629765] lvl 0 el 0 blk 0 by 245366784 [D 20:00:56.629790] to 245366784 ta 234881024 fi 268435456 [D 20:00:56.629815] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.629839] basic type or contiguous data [D 20:00:56.629863] Request disp [D 20:00:56.629887] contig_offset = 245366784 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.629912] calling distribute [D 20:00:56.629936] PINT_distribute [D 20:00:56.629960] of 245366784 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 245366784 exfl 1 [D 20:00:56.629986] begin iteration loff: 245366784 [D 20:00:56.630011] segment exceeds byte limit [D 20:00:56.630035] file being extended [D 20:00:56.630059] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.630084] **********CALL***PROCESS********* [D 20:00:56.630107] segment of 10485760 sz 2097152 [D 20:00:56.630132] PINT_process_request [D 20:00:56.630156] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.630183] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.630208] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.630232] basic type or contiguous data [D 20:00:56.630256] Request disp [D 20:00:56.630280] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.630305] process a segment [D 20:00:56.630329] of 10485760 sz 2097152 [D 20:00:56.630353] add a segment 0 [D 20:00:56.630377] retval = 2097152 [D 20:00:56.630401] exiting distribute returned less than expected [D 20:00:56.630425] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.630452] *****RETURN***FROM***PROCESS***** [D 20:00:56.630475] end iteration [D 20:00:56.630499] done with segments or bytes [D 20:00:56.630523] of 247463936 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.630552] next loff: 247463936 next poff: 247463936 [D 20:00:56.630577] return value 2097152 [D 20:00:56.630600] finished [D 20:00:56.630624] retval = 2097152 [D 20:00:56.630648] exiting distribute returned less than expected [D 20:00:56.630673] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 247463936 fo 268435456 eof 0 [D 20:00:56.630699] ========================================================= [D 20:00:56.630723] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:56.630748] element 0: offset: 0xb4e63000, size: 2097152 [D 20:00:56.630776] post_sr: da13:3346 bh 79 len 32 wr_id 15e732e8. [D 20:00:56.632068] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 0. [D 20:00:56.632097] post_sr: da13:3346 bh 80 len 16 wr_id 15e73300. [D 20:00:56.632122] openib_check_cq: completed send wr_id 15e732e8. [D 20:00:56.632148] openib_check_cq: completed send wr_id 15e73300. [D 20:00:56.632178] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:56.632204] ========================================================= [D 20:00:56.632228] PINT_process_request [D 20:00:56.632252] tiling 10 copies [D 20:00:56.632276] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.632303] lvl 0 el 0 blk 0 by 278921216 [D 20:00:56.632328] to 278921216 ta 268435456 fi 301989888 [D 20:00:56.632352] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.632377] basic type or contiguous data [D 20:00:56.632401] Request disp [D 20:00:56.632425] contig_offset = 278921216 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.632451] calling distribute [D 20:00:56.632474] PINT_distribute [D 20:00:56.632498] of 278921216 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 278921216 exfl 1 [D 20:00:56.632524] begin iteration loff: 278921216 [D 20:00:56.632550] segment exceeds byte limit [D 20:00:56.632574] file being extended [D 20:00:56.632597] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.632621] **********CALL***PROCESS********* [D 20:00:56.632646] segment of 10485760 sz 2097152 [D 20:00:56.632670] PINT_process_request [D 20:00:56.632694] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.632720] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.632745] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.632770] basic type or contiguous data [D 20:00:56.632794] Request disp [D 20:00:56.632818] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.632843] process a segment [D 20:00:56.632866] of 10485760 sz 2097152 [D 20:00:56.632890] add a segment 0 [D 20:00:56.632914] retval = 2097152 [D 20:00:56.632938] exiting distribute returned less than expected [D 20:00:56.632963] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.632988] *****RETURN***FROM***PROCESS***** [D 20:00:56.633012] end iteration [D 20:00:56.633036] done with segments or bytes [D 20:00:56.633060] of 281018368 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.633086] next loff: 281018368 next poff: 281018368 [D 20:00:56.633111] return value 2097152 [D 20:00:56.633134] finished [D 20:00:56.633158] retval = 2097152 [D 20:00:56.633183] exiting distribute returned less than expected [D 20:00:56.633207] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 281018368 fo 301989888 eof 0 [D 20:00:56.633233] ========================================================= [D 20:00:56.633256] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:56.633282] element 0: offset: 0xb2e61000, size: 2097152 [D 20:00:56.633309] post_sr: da13:3346 bh 81 len 32 wr_id 15e73318. [D 20:00:56.633338] openib_check_cq: completed recv wr_id 15e73d10. [D 20:00:56.633367] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaab4a1b010 rkey 48006500 nic_wr_credit 1. [D 20:00:56.633395] openib_post_rr: da13:3346 bh 87 wr_id 15e73d10. [D 20:00:56.633420] openib_check_cq: completed send wr_id 15e73318. [D 20:00:56.634730] PINT_client_state_machine_test id 22 [D 20:00:56.635864] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 0. [D 20:00:56.635891] post_sr: da13:3346 bh 82 len 16 wr_id 15e73330. [D 20:00:56.635928] openib_check_cq: completed send wr_id 15e73330. [D 20:00:56.635958] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:56.635984] ========================================================= [D 20:00:56.636009] PINT_process_request [D 20:00:56.636033] tiling 11 copies [D 20:00:56.636057] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.636083] lvl 0 el 0 blk 0 by 312475648 [D 20:00:56.636108] to 312475648 ta 301989888 fi 335544320 [D 20:00:56.636133] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.636157] basic type or contiguous data [D 20:00:56.636181] Request disp [D 20:00:56.636205] contig_offset = 312475648 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.636230] calling distribute [D 20:00:56.636254] PINT_distribute [D 20:00:56.636278] of 312475648 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 312475648 exfl 1 [D 20:00:56.636304] begin iteration loff: 312475648 [D 20:00:56.636329] segment exceeds byte limit [D 20:00:56.636353] file being extended [D 20:00:56.636377] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.636402] **********CALL***PROCESS********* [D 20:00:56.636425] segment of 10485760 sz 2097152 [D 20:00:56.636450] PINT_process_request [D 20:00:56.636474] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.636500] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.636525] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.636549] basic type or contiguous data [D 20:00:56.636573] Request disp [D 20:00:56.636597] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.636623] process a segment [D 20:00:56.636647] of 10485760 sz 2097152 [D 20:00:56.636671] add a segment 0 [D 20:00:56.636694] retval = 2097152 [D 20:00:56.636719] exiting distribute returned less than expected [D 20:00:56.636743] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.636769] *****RETURN***FROM***PROCESS***** [D 20:00:56.636793] end iteration [D 20:00:56.636816] done with segments or bytes [D 20:00:56.636840] of 314572800 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.636866] next loff: 314572800 next poff: 314572800 [D 20:00:56.636891] return value 2097152 [D 20:00:56.636916] finished [D 20:00:56.636940] retval = 2097152 [D 20:00:56.636963] exiting distribute returned less than expected [D 20:00:56.636987] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 314572800 fo 335544320 eof 0 [D 20:00:56.637013] ========================================================= [D 20:00:56.637038] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:56.637063] element 0: offset: 0xb0e5f000, size: 2097152 [D 20:00:56.637089] post_sr: da13:3346 bh 83 len 32 wr_id 15e73348. [D 20:00:56.637125] openib_check_cq: completed send wr_id 15e73348. [D 20:00:56.644761] PINT_client_state_machine_test id 22 [D 20:00:56.654788] PINT_client_state_machine_test id 22 [D 20:00:56.664820] PINT_client_state_machine_test id 22 [D 20:00:56.674849] PINT_client_state_machine_test id 22 [D 20:00:56.693668] PINT_client_state_machine_test id 22 [D 20:00:56.705667] PINT_client_state_machine_test id 22 [D 20:00:56.717667] PINT_client_state_machine_test id 22 [D 20:00:56.729666] PINT_client_state_machine_test id 22 [D 20:00:56.741666] PINT_client_state_machine_test id 22 [D 20:00:56.753666] PINT_client_state_machine_test id 22 [D 20:00:56.765666] PINT_client_state_machine_test id 22 [D 20:00:56.777672] PINT_client_state_machine_test id 22 [D 20:00:56.789668] PINT_client_state_machine_test id 22 [D 20:00:56.801667] PINT_client_state_machine_test id 22 [D 20:00:56.813667] PINT_client_state_machine_test id 22 [D 20:00:56.825668] PINT_client_state_machine_test id 22 [D 20:00:56.837667] PINT_client_state_machine_test id 22 [D 20:00:56.849667] PINT_client_state_machine_test id 22 [D 20:00:56.861667] PINT_client_state_machine_test id 22 [D 20:00:56.873667] PINT_client_state_machine_test id 22 [D 20:00:56.885668] PINT_client_state_machine_test id 22 [D 20:00:56.897667] PINT_client_state_machine_test id 22 [D 20:00:56.909667] PINT_client_state_machine_test id 22 [D 20:00:56.921668] PINT_client_state_machine_test id 22 -[D 20:00:56.932520] openib_check_cq: completed recv wr_id 15e73d28. [D 20:00:56.932573] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab4c1c010 rkey 48006600 nic_wr_credit 1. [D 20:00:56.932601] openib_post_rr: da13:3346 bh 88 wr_id 15e73d28. [D 20:00:56.932644] PINT_client_state_machine_test id 22 [D 20:00:56.935068] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 0. [D 20:00:56.935096] post_sr: da13:3346 bh 84 len 16 wr_id 15e73360. [D 20:00:56.935123] openib_check_cq: completed recv wr_id 15e73d40. [D 20:00:56.935151] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaab4e1d010 rkey 48006700 nic_wr_credit 1. [D 20:00:56.935177] openib_post_rr: da13:3346 bh 89 wr_id 15e73d40. [D 20:00:56.935204] openib_check_cq: completed send wr_id 15e73360. [D 20:00:56.935235] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:56.935262] ========================================================= [D 20:00:56.935286] PINT_process_request [D 20:00:56.935312] tiling 12 copies [D 20:00:56.935336] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.935362] lvl 0 el 0 blk 0 by 346030080 [D 20:00:56.935387] to 346030080 ta 335544320 fi 369098752 [D 20:00:56.935412] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.935437] basic type or contiguous data [D 20:00:56.935461] Request disp [D 20:00:56.935485] contig_offset = 346030080 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.935510] calling distribute [D 20:00:56.935535] PINT_distribute [D 20:00:56.935559] of 346030080 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 346030080 exfl 1 [D 20:00:56.935586] begin iteration loff: 346030080 [D 20:00:56.935612] segment exceeds byte limit [D 20:00:56.935636] file being extended [D 20:00:56.935660] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.935685] **********CALL***PROCESS********* [D 20:00:56.935708] segment of 10485760 sz 2097152 [D 20:00:56.935732] PINT_process_request [D 20:00:56.935756] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.935783] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.935808] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.935832] basic type or contiguous data [D 20:00:56.935856] Request disp [D 20:00:56.935879] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.935905] process a segment [D 20:00:56.935929] of 10485760 sz 2097152 [D 20:00:56.935953] add a segment 0 [D 20:00:56.935976] retval = 2097152 [D 20:00:56.936000] exiting distribute returned less than expected [D 20:00:56.936025] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.936051] *****RETURN***FROM***PROCESS***** [D 20:00:56.936076] end iteration [D 20:00:56.936100] done with segments or bytes [D 20:00:56.936122] of 348127232 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.936148] next loff: 348127232 next poff: 348127232 [D 20:00:56.936173] return value 2097152 [D 20:00:56.936198] finished [D 20:00:56.936222] retval = 2097152 [D 20:00:56.936245] exiting distribute returned less than expected [D 20:00:56.936269] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 348127232 fo 369098752 eof 0 [D 20:00:56.936295] ========================================================= [D 20:00:56.936320] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:56.936346] element 0: offset: 0xaee5d000, size: 2097152 [D 20:00:56.936374] post_sr: da13:3346 bh 85 len 32 wr_id 15e73378. [D 20:00:56.937645] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 0. [D 20:00:56.937679] post_sr: da13:3346 bh 86 len 16 wr_id 15e73390. [D 20:00:56.937705] openib_check_cq: completed send wr_id 15e73378. [D 20:00:56.937730] openib_check_cq: completed send wr_id 15e73390. [D 20:00:56.937761] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:56.937787] ========================================================= [D 20:00:56.937810] PINT_process_request [D 20:00:56.937834] tiling 13 copies [D 20:00:56.937859] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.937885] lvl 0 el 0 blk 0 by 379584512 [D 20:00:56.937910] to 379584512 ta 369098752 fi 402653184 [D 20:00:56.937934] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.937959] basic type or contiguous data [D 20:00:56.937983] Request disp [D 20:00:56.938007] contig_offset = 379584512 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.938032] calling distribute [D 20:00:56.938055] PINT_distribute [D 20:00:56.938079] of 379584512 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 379584512 exfl 1 [D 20:00:56.938106] begin iteration loff: 379584512 [D 20:00:56.938131] segment exceeds byte limit [D 20:00:56.938155] file being extended [D 20:00:56.938179] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.938203] **********CALL***PROCESS********* [D 20:00:56.938227] segment of 10485760 sz 2097152 [D 20:00:56.938252] PINT_process_request [D 20:00:56.938276] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.938302] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.938326] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.938351] basic type or contiguous data [D 20:00:56.938375] Request disp [D 20:00:56.938399] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.938424] process a segment [D 20:00:56.938447] of 10485760 sz 2097152 [D 20:00:56.938471] add a segment 0 [D 20:00:56.938495] retval = 2097152 [D 20:00:56.938519] exiting distribute returned less than expected [D 20:00:56.938544] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.938570] *****RETURN***FROM***PROCESS***** [D 20:00:56.938593] end iteration [D 20:00:56.938617] done with segments or bytes [D 20:00:56.938641] of 381681664 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.938669] next loff: 381681664 next poff: 381681664 [D 20:00:56.938694] return value 2097152 [D 20:00:56.938718] finished [D 20:00:56.938742] retval = 2097152 [D 20:00:56.938766] exiting distribute returned less than expected [D 20:00:56.938790] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 381681664 fo 402653184 eof 0 [D 20:00:56.938816] ========================================================= [D 20:00:56.938840] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:56.938866] element 0: offset: 0xace5b000, size: 2097152 [D 20:00:56.938893] post_sr: da13:3346 bh 87 len 32 wr_id 15e733a8. [D 20:00:56.938928] openib_check_cq: completed send wr_id 15e733a8. [D 20:00:56.942679] PINT_client_state_machine_test id 22 [D 20:00:56.942981] openib_check_cq: completed recv wr_id 15e73d58. [D 20:00:56.943010] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaab501e010 rkey c8006800 nic_wr_credit 1. [D 20:00:56.943037] openib_post_rr: da13:3346 bh 90 wr_id 15e73d58. [D 20:00:56.943237] openib_check_cq: completed recv wr_id 15e73d70. [D 20:00:56.943265] openib_post_sr_rdmaw: ibv_post_send wr_id 15f07a30 to da13:3346 remote addr 2aaab521f010 rkey c8006900 nic_wr_credit 2. [D 20:00:56.943292] openib_post_rr: da13:3346 bh 91 wr_id 15e73d70. [D 20:00:56.945502] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 1. [D 20:00:56.945529] post_sr: da13:3346 bh 88 len 16 wr_id 15e733c0. [D 20:00:56.946087] openib_check_cq: completed recv wr_id 15e73d88. [D 20:00:56.946116] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab5420010 rkey c8006a00 nic_wr_credit 2. [D 20:00:56.946143] openib_post_rr: da13:3346 bh 92 wr_id 15e73d88. [D 20:00:56.947950] openib_check_cq: completed rdma wr_id 15f07a30, nic_wr_credit 1. [D 20:00:56.947978] post_sr: da13:3346 bh 89 len 16 wr_id 15e733d8. [D 20:00:56.948004] openib_check_cq: completed send wr_id 15e733c0. [D 20:00:56.948035] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:00:56.948060] ========================================================= [D 20:00:56.948084] PINT_process_request [D 20:00:56.948109] tiling 14 copies [D 20:00:56.948133] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.948160] lvl 0 el 0 blk 0 by 413138944 [D 20:00:56.948184] to 413138944 ta 402653184 fi 436207616 [D 20:00:56.948208] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.948233] basic type or contiguous data [D 20:00:56.948257] Request disp [D 20:00:56.948281] contig_offset = 413138944 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.948307] calling distribute [D 20:00:56.948330] PINT_distribute [D 20:00:56.948353] of 413138944 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 413138944 exfl 1 [D 20:00:56.948380] begin iteration loff: 413138944 [D 20:00:56.948405] segment exceeds byte limit [D 20:00:56.948429] file being extended [D 20:00:56.948452] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.948477] **********CALL***PROCESS********* [D 20:00:56.948501] segment of 10485760 sz 2097152 [D 20:00:56.948525] PINT_process_request [D 20:00:56.948549] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.948575] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.948599] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.948624] basic type or contiguous data [D 20:00:56.948648] Request disp [D 20:00:56.948676] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.948704] process a segment [D 20:00:56.948731] of 10485760 sz 2097152 [D 20:00:56.948759] add a segment 0 [D 20:00:56.948785] retval = 2097152 [D 20:00:56.948810] exiting distribute returned less than expected [D 20:00:56.948834] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.948860] *****RETURN***FROM***PROCESS***** [D 20:00:56.948884] end iteration [D 20:00:56.948908] done with segments or bytes [D 20:00:56.948931] of 415236096 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.948957] next loff: 415236096 next poff: 415236096 [D 20:00:56.948982] return value 2097152 [D 20:00:56.949006] finished [D 20:00:56.949029] retval = 2097152 [D 20:00:56.949053] exiting distribute returned less than expected [D 20:00:56.949077] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 415236096 fo 436207616 eof 0 [D 20:00:56.949103] ========================================================= [D 20:00:56.949128] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:00:56.949152] element 0: offset: 0xaae59000, size: 2097152 [D 20:00:56.949179] post_sr: da13:3346 bh 90 len 32 wr_id 15e733f0. [D 20:00:56.950038] openib_check_cq: completed recv wr_id 15e73da0. [D 20:00:56.950067] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab5621010 rkey c8006b00 nic_wr_credit 2. [D 20:00:56.950095] openib_post_rr: da13:3346 bh 93 wr_id 15e73da0. [D 20:00:56.950407] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 1. [D 20:00:56.950434] post_sr: da13:3346 bh 91 len 16 wr_id 15e73408. [D 20:00:56.950461] openib_check_cq: completed send wr_id 15e733d8. [D 20:00:56.950486] openib_check_cq: completed send wr_id 15e733f0. [D 20:00:56.950516] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:00:56.950542] ========================================================= [D 20:00:56.950566] PINT_process_request [D 20:00:56.950591] tiling 16 copies [D 20:00:56.950614] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.950640] lvl 0 el 0 blk 0 by 471859200 [D 20:00:56.950668] to 471859200 ta 469762048 fi 503316480 [D 20:00:56.950693] mto 2097152 mta 0 mfi 2097152 [D 20:00:56.950720] basic type or contiguous data [D 20:00:56.950745] Request disp [D 20:00:56.950769] contig_offset = 471859200 contig_size = 65011712 lvl_flag = 1 [D 20:00:56.950794] calling distribute [D 20:00:56.950818] PINT_distribute [D 20:00:56.950841] of 471859200 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 1 [D 20:00:56.950868] begin iteration loff: 471859200 [D 20:00:56.950893] segment exceeds byte limit [D 20:00:56.950917] file being extended [D 20:00:56.950940] client lstof 2097152 diff 0 sgof 2097152 [D 20:00:56.950964] **********CALL***PROCESS********* [D 20:00:56.950988] segment of 2097152 sz 2097152 [D 20:00:56.951013] PINT_process_request [D 20:00:56.951037] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.951063] lvl 0 el 0 blk 0 by 2097152 [D 20:00:56.951087] to 2097152 ta 2097152 fi 4194304 [D 20:00:56.951112] basic type or contiguous data [D 20:00:56.951137] Request disp [D 20:00:56.951161] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:00:56.951186] process a segment [D 20:00:56.951210] of 2097152 sz 2097152 [D 20:00:56.951234] add a segment 0 [D 20:00:56.951258] retval = 2097152 [D 20:00:56.951282] exiting distribute returned less than expected [D 20:00:56.951306] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:00:56.951332] *****RETURN***FROM***PROCESS***** [D 20:00:56.951355] end iteration [D 20:00:56.951379] done with segments or bytes [D 20:00:56.951403] of 473956352 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.951429] next loff: 473956352 next poff: 473956352 [D 20:00:56.951454] return value 2097152 [D 20:00:56.951478] finished [D 20:00:56.951501] retval = 2097152 [D 20:00:56.951525] exiting distribute returned less than expected [D 20:00:56.951549] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 473956352 fo 503316480 eof 0 [D 20:00:56.951575] ========================================================= [D 20:00:56.951600] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:00:56.951624] element 0: offset: 0xa6655000, size: 2097152 [D 20:00:56.951652] post_sr: da13:3346 bh 92 len 32 wr_id 15e73420. [D 20:00:56.952709] PINT_client_state_machine_test id 22 [D 20:00:56.952864] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 0. [D 20:00:56.952892] post_sr: da13:3346 bh 93 len 16 wr_id 15e73438. [D 20:00:56.952918] openib_check_cq: completed send wr_id 15e73408. [D 20:00:56.952943] openib_check_cq: completed send wr_id 15e73420. [D 20:00:56.952969] openib_check_cq: completed send wr_id 15e73438. [D 20:00:56.953000] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:00:56.953025] ========================================================= [D 20:00:56.953049] PINT_process_request [D 20:00:56.953073] tiling 15 copies [D 20:00:56.953098] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.953124] lvl 0 el 0 blk 0 by 446693376 [D 20:00:56.953148] to 446693376 ta 436207616 fi 469762048 [D 20:00:56.953173] mto 10485760 mta 8388608 mfi 10485760 [D 20:00:56.953198] basic type or contiguous data [D 20:00:56.953222] Request disp [D 20:00:56.953246] contig_offset = 446693376 contig_size = 56623104 lvl_flag = 1 [D 20:00:56.953270] calling distribute [D 20:00:56.953294] PINT_distribute [D 20:00:56.953318] of 446693376 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 446693376 exfl 1 [D 20:00:56.953345] begin iteration loff: 446693376 [D 20:00:56.953370] segment exceeds byte limit [D 20:00:56.953393] file being extended [D 20:00:56.953417] client lstof 10485760 diff 0 sgof 10485760 [D 20:00:56.953441] **********CALL***PROCESS********* [D 20:00:56.953465] segment of 10485760 sz 2097152 [D 20:00:56.953490] PINT_process_request [D 20:00:56.953514] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.953539] lvl 0 el 0 blk 0 by 10485760 [D 20:00:56.953564] to 10485760 ta 10485760 fi 12582912 [D 20:00:56.953591] basic type or contiguous data [D 20:00:56.953616] Request disp [D 20:00:56.953639] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:00:56.953664] process a segment [D 20:00:56.953688] of 10485760 sz 2097152 [D 20:00:56.953712] add a segment 0 [D 20:00:56.953737] retval = 2097152 [D 20:00:56.953761] exiting distribute returned less than expected [D 20:00:56.953784] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:00:56.953810] *****RETURN***FROM***PROCESS***** [D 20:00:56.953835] end iteration [D 20:00:56.953858] done with segments or bytes [D 20:00:56.953882] of 448790528 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.953907] next loff: 448790528 next poff: 448790528 [D 20:00:56.953932] return value 2097152 [D 20:00:56.953957] finished [D 20:00:56.953981] retval = 2097152 [D 20:00:56.954005] exiting distribute returned less than expected [D 20:00:56.954029] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 448790528 fo 469762048 eof 0 [D 20:00:56.954054] ========================================================= [D 20:00:56.954079] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:00:56.954104] element 0: offset: 0xa8e57000, size: 2097152 [D 20:00:56.954131] post_sr: da13:3346 bh 94 len 32 wr_id 15e73450. [D 20:00:56.954159] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:56.954185] ========================================================= [D 20:00:56.954210] PINT_process_request [D 20:00:56.954234] tiling 2 copies [D 20:00:56.954257] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.954283] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.954308] to 12582912 ta 0 fi 33554432 [D 20:00:56.954333] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.954359] basic type or contiguous data [D 20:00:56.954382] Request disp [D 20:00:56.954406] contig_offset = 12582912 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.954431] calling distribute [D 20:00:56.954455] PINT_distribute [D 20:00:56.954479] of 12582912 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 12582912 exfl 1 [D 20:00:56.954504] begin iteration loff: 12582912 [D 20:00:56.954530] segment exceeds byte limit [D 20:00:56.954554] file being extended [D 20:00:56.954578] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.954602] **********CALL***PROCESS********* [D 20:00:56.954626] segment of 12582912 sz 2097152 [D 20:00:56.954650] PINT_process_request [D 20:00:56.954677] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.954703] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.954728] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.954752] basic type or contiguous data [D 20:00:56.954776] Request disp [D 20:00:56.954800] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.954825] process a segment [D 20:00:56.954849] of 12582912 sz 2097152 [D 20:00:56.954873] add a segment 0 [D 20:00:56.954896] retval = 2097152 [D 20:00:56.954920] exiting distribute returned less than expected [D 20:00:56.954945] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.954971] *****RETURN***FROM***PROCESS***** [D 20:00:56.954995] end iteration [D 20:00:56.955018] done with segments or bytes [D 20:00:56.955042] of 14680064 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.955068] next loff: 14680064 next poff: 14680064 [D 20:00:56.955092] return value 2097152 [D 20:00:56.955117] finished [D 20:00:56.955141] retval = 2097152 [D 20:00:56.955164] exiting distribute returned less than expected [D 20:00:56.955188] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 14680064 fo 33554432 eof 0 [D 20:00:56.955214] ========================================================= [D 20:00:56.955239] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:56.955264] element 0: offset: 0xc3071000, size: 2097152 [D 20:00:56.955293] post_sr: da13:3346 bh 95 len 32 wr_id 15e73468. [D 20:00:56.955324] openib_check_cq: completed recv wr_id 15e73db8. [D 20:00:56.955352] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab5822010 rkey c8006c00 nic_wr_credit 1. [D 20:00:56.955380] openib_post_rr: da13:3346 bh 94 wr_id 15e73db8. [D 20:00:56.955406] openib_check_cq: completed send wr_id 15e73450. [D 20:00:56.955430] openib_check_cq: completed send wr_id 15e73468. [D 20:00:56.956820] openib_check_cq: completed recv wr_id 15e73dd0. [D 20:00:56.956850] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab5a23010 rkey c8006d00 nic_wr_credit 2. [D 20:00:56.956876] openib_post_rr: da13:3346 bh 95 wr_id 15e73dd0. [D 20:00:56.957847] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 1. [D 20:00:56.957874] post_sr: da13:3346 bh 96 len 16 wr_id 15e73480. [D 20:00:56.960302] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 0. [D 20:00:56.960330] post_sr: da13:3346 bh 97 len 16 wr_id 15e73498. [D 20:00:56.960355] openib_check_cq: completed send wr_id 15e73480. [D 20:00:56.960381] openib_check_cq: completed send wr_id 15e73498. [D 20:00:56.960412] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:56.960437] ========================================================= [D 20:00:56.960462] PINT_process_request [D 20:00:56.960486] tiling 3 copies [D 20:00:56.960510] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.960536] lvl 0 el 0 blk 0 by 46137344 [D 20:00:56.960560] to 46137344 ta 33554432 fi 67108864 [D 20:00:56.960585] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.960610] basic type or contiguous data [D 20:00:56.960634] Request disp [D 20:00:56.960658] contig_offset = 46137344 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.960687] calling distribute [D 20:00:56.960715] PINT_distribute [D 20:00:56.960742] of 46137344 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 46137344 exfl 1 [D 20:00:56.960773] begin iteration loff: 46137344 [D 20:00:56.960798] segment exceeds byte limit [D 20:00:56.960822] file being extended [D 20:00:56.960845] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.960870] **********CALL***PROCESS********* [D 20:00:56.960893] segment of 12582912 sz 2097152 [D 20:00:56.960918] PINT_process_request [D 20:00:56.960942] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.960968] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.960992] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.961017] basic type or contiguous data [D 20:00:56.961041] Request disp [D 20:00:56.961064] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.961090] process a segment [D 20:00:56.961112] of 12582912 sz 2097152 [D 20:00:56.961137] add a segment 0 [D 20:00:56.961161] retval = 2097152 [D 20:00:56.961185] exiting distribute returned less than expected [D 20:00:56.961209] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.961235] *****RETURN***FROM***PROCESS***** [D 20:00:56.961258] end iteration [D 20:00:56.961282] done with segments or bytes [D 20:00:56.961305] of 48234496 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.961331] next loff: 48234496 next poff: 48234496 [D 20:00:56.961356] return value 2097152 [D 20:00:56.961380] finished [D 20:00:56.961404] retval = 2097152 [D 20:00:56.961428] exiting distribute returned less than expected [D 20:00:56.961452] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 48234496 fo 67108864 eof 0 [D 20:00:56.961478] ========================================================= [D 20:00:56.961503] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:56.961527] element 0: offset: 0xc106f000, size: 2097152 [D 20:00:56.961554] post_sr: da13:3346 bh 98 len 32 wr_id 15e734b0. [D 20:00:56.961583] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:56.961608] ========================================================= [D 20:00:56.961635] PINT_process_request [D 20:00:56.961660] tiling 4 copies [D 20:00:56.961684] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.961710] lvl 0 el 0 blk 0 by 79691776 [D 20:00:56.961735] to 79691776 ta 67108864 fi 100663296 [D 20:00:56.961760] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.961785] basic type or contiguous data [D 20:00:56.961809] Request disp [D 20:00:56.961833] contig_offset = 79691776 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.961857] calling distribute [D 20:00:56.961881] PINT_distribute [D 20:00:56.961904] of 79691776 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 79691776 exfl 1 [D 20:00:56.961931] begin iteration loff: 79691776 [D 20:00:56.961956] segment exceeds byte limit [D 20:00:56.961979] file being extended [D 20:00:56.962003] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.962028] **********CALL***PROCESS********* [D 20:00:56.962051] segment of 12582912 sz 2097152 [D 20:00:56.962076] PINT_process_request [D 20:00:56.962100] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.962125] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.962150] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.962175] basic type or contiguous data [D 20:00:56.962199] Request disp [D 20:00:56.962223] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.962247] process a segment [D 20:00:56.962270] of 12582912 sz 2097152 [D 20:00:56.962295] add a segment 0 [D 20:00:56.962319] retval = 2097152 [D 20:00:56.962343] exiting distribute returned less than expected [D 20:00:56.962367] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.962392] *****RETURN***FROM***PROCESS***** [D 20:00:56.962417] end iteration [D 20:00:56.962440] done with segments or bytes [D 20:00:56.962464] of 81788928 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.962490] next loff: 81788928 next poff: 81788928 [D 20:00:56.962514] return value 2097152 [D 20:00:56.962538] finished [D 20:00:56.962562] retval = 2097152 [D 20:00:56.962586] exiting distribute returned less than expected [D 20:00:56.962610] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 81788928 fo 100663296 eof 0 [D 20:00:56.962635] ========================================================= [D 20:00:56.962662] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:56.962688] element 0: offset: 0xbf06d000, size: 2097152 [D 20:00:56.962715] post_sr: da13:3346 bh 99 len 32 wr_id 15e734c8. [D 20:00:56.962744] PINT_client_state_machine_test id 22 [D 20:00:56.962773] openib_check_cq: completed send wr_id 15e734b0. [D 20:00:56.962799] openib_check_cq: completed recv wr_id 15e73de8. [D 20:00:56.962826] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaab5c25010 rkey c8006e00 nic_wr_credit 1. [D 20:00:56.962854] openib_post_rr: da13:3346 bh 96 wr_id 15e73de8. [D 20:00:56.962880] openib_check_cq: completed send wr_id 15e734c8. [D 20:00:56.965264] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 0. [D 20:00:56.965291] post_sr: da13:3346 bh 0 len 16 wr_id 15e72b80. [D 20:00:56.965328] openib_check_cq: completed send wr_id 15e72b80. [D 20:00:56.965358] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:00:56.965384] ========================================================= [D 20:00:56.965409] PINT_process_request [D 20:00:56.965433] tiling 16 copies [D 20:00:56.965456] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.965483] lvl 0 el 0 blk 0 by 473956352 [D 20:00:56.965508] to 473956352 ta 469762048 fi 503316480 [D 20:00:56.965533] mto 4194304 mta 2097152 mfi 4194304 [D 20:00:56.965558] basic type or contiguous data [D 20:00:56.965581] Request disp [D 20:00:56.965604] contig_offset = 473956352 contig_size = 62914560 lvl_flag = 1 [D 20:00:56.965630] calling distribute [D 20:00:56.965654] PINT_distribute [D 20:00:56.965680] of 473956352 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 473956352 exfl 1 [D 20:00:56.965706] begin iteration loff: 473956352 [D 20:00:56.965731] segment exceeds byte limit [D 20:00:56.965755] file being extended [D 20:00:56.965779] client lstof 4194304 diff 0 sgof 4194304 [D 20:00:56.965804] **********CALL***PROCESS********* [D 20:00:56.965827] segment of 4194304 sz 2097152 [D 20:00:56.965851] PINT_process_request [D 20:00:56.965875] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.965901] lvl 0 el 0 blk 0 by 4194304 [D 20:00:56.965926] to 4194304 ta 4194304 fi 6291456 [D 20:00:56.965950] basic type or contiguous data [D 20:00:56.965974] Request disp [D 20:00:56.965998] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:00:56.966023] process a segment [D 20:00:56.966047] of 4194304 sz 2097152 [D 20:00:56.966071] add a segment 0 [D 20:00:56.966094] retval = 2097152 [D 20:00:56.966118] exiting distribute returned less than expected [D 20:00:56.966141] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:00:56.966168] *****RETURN***FROM***PROCESS***** [D 20:00:56.966192] end iteration [D 20:00:56.966216] done with segments or bytes [D 20:00:56.966238] of 476053504 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.966264] next loff: 476053504 next poff: 476053504 [D 20:00:56.966288] return value 2097152 [D 20:00:56.966313] finished [D 20:00:56.966336] retval = 2097152 [D 20:00:56.966360] exiting distribute returned less than expected [D 20:00:56.966383] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 476053504 fo 503316480 eof 0 [D 20:00:56.966410] ========================================================= [D 20:00:56.966434] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:00:56.966459] element 0: offset: 0xa6855000, size: 2097152 [D 20:00:56.966485] post_sr: da13:3346 bh 1 len 32 wr_id 15e72b98. [D 20:00:56.966521] openib_check_cq: completed send wr_id 15e72b98. [D 20:00:56.967876] openib_check_cq: completed recv wr_id 15e73e00. [D 20:00:56.967905] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaab5e26010 rkey c8006f00 nic_wr_credit 1. [D 20:00:56.967931] openib_post_rr: da13:3346 bh 97 wr_id 15e73e00. [D 20:00:56.970397] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 0. [D 20:00:56.970425] post_sr: da13:3346 bh 2 len 16 wr_id 15e72bb0. [D 20:00:56.970461] openib_check_cq: completed send wr_id 15e72bb0. [D 20:00:56.970492] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:56.970518] ========================================================= [D 20:00:56.970542] PINT_process_request [D 20:00:56.970566] tiling 5 copies [D 20:00:56.970590] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.970617] lvl 0 el 0 blk 0 by 113246208 [D 20:00:56.970642] to 113246208 ta 100663296 fi 134217728 [D 20:00:56.970668] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.970694] basic type or contiguous data [D 20:00:56.970718] Request disp [D 20:00:56.970742] contig_offset = 113246208 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.970767] calling distribute [D 20:00:56.970790] PINT_distribute [D 20:00:56.970814] of 113246208 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 113246208 exfl 1 [D 20:00:56.970841] begin iteration loff: 113246208 [D 20:00:56.970866] segment exceeds byte limit [D 20:00:56.970890] file being extended [D 20:00:56.970913] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.970938] **********CALL***PROCESS********* [D 20:00:56.970962] segment of 12582912 sz 2097152 [D 20:00:56.970986] PINT_process_request [D 20:00:56.971010] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.971035] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.971060] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.971085] basic type or contiguous data [D 20:00:56.971109] Request disp [D 20:00:56.971136] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.971160] process a segment [D 20:00:56.971184] of 12582912 sz 2097152 [D 20:00:56.971209] add a segment 0 [D 20:00:56.971233] retval = 2097152 [D 20:00:56.971257] exiting distribute returned less than expected [D 20:00:56.971281] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.971306] *****RETURN***FROM***PROCESS***** [D 20:00:56.971331] end iteration [D 20:00:56.971355] done with segments or bytes [D 20:00:56.971378] of 115343360 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.971404] next loff: 115343360 next poff: 115343360 [D 20:00:56.971428] return value 2097152 [D 20:00:56.971453] finished [D 20:00:56.971477] retval = 2097152 [D 20:00:56.971501] exiting distribute returned less than expected [D 20:00:56.971525] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 115343360 fo 134217728 eof 0 [D 20:00:56.971550] ========================================================= [D 20:00:56.971575] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:56.971600] element 0: offset: 0xbd06b000, size: 2097152 [D 20:00:56.971627] post_sr: da13:3346 bh 3 len 32 wr_id 15e72bc8. [D 20:00:56.971656] openib_check_cq: completed recv wr_id 15e73e18. [D 20:00:56.971685] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab6027010 rkey 48007000 nic_wr_credit 1. [D 20:00:56.971713] openib_post_rr: da13:3346 bh 98 wr_id 15e73e18. [D 20:00:56.971738] openib_check_cq: completed send wr_id 15e72bc8. [D 20:00:56.972774] PINT_client_state_machine_test id 22 [D 20:00:56.974177] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 0. [D 20:00:56.974205] post_sr: da13:3346 bh 4 len 16 wr_id 15e72be0. [D 20:00:56.974241] openib_check_cq: completed send wr_id 15e72be0. [D 20:00:56.974271] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:56.974297] ========================================================= [D 20:00:56.974321] PINT_process_request [D 20:00:56.974346] tiling 6 copies [D 20:00:56.974370] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.974395] lvl 0 el 0 blk 0 by 146800640 [D 20:00:56.974420] to 146800640 ta 134217728 fi 167772160 [D 20:00:56.974445] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.974470] basic type or contiguous data [D 20:00:56.974493] Request disp [D 20:00:56.974516] contig_offset = 146800640 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.974542] calling distribute [D 20:00:56.974566] PINT_distribute [D 20:00:56.974590] of 146800640 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 146800640 exfl 1 [D 20:00:56.974616] begin iteration loff: 146800640 [D 20:00:56.974641] segment exceeds byte limit [D 20:00:56.974667] file being extended [D 20:00:56.974691] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.974716] **********CALL***PROCESS********* [D 20:00:56.974740] segment of 12582912 sz 2097152 [D 20:00:56.974764] PINT_process_request [D 20:00:56.974788] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.974814] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.974839] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.974864] basic type or contiguous data [D 20:00:56.974887] Request disp [D 20:00:56.974911] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.974936] process a segment [D 20:00:56.974959] of 12582912 sz 2097152 [D 20:00:56.974984] add a segment 0 [D 20:00:56.975007] retval = 2097152 [D 20:00:56.975030] exiting distribute returned less than expected [D 20:00:56.975054] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.975081] *****RETURN***FROM***PROCESS***** [D 20:00:56.975105] end iteration [D 20:00:56.975129] done with segments or bytes [D 20:00:56.975152] of 148897792 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.975178] next loff: 148897792 next poff: 148897792 [D 20:00:56.975205] return value 2097152 [D 20:00:56.975230] finished [D 20:00:56.975254] retval = 2097152 [D 20:00:56.975277] exiting distribute returned less than expected [D 20:00:56.975301] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 148897792 fo 167772160 eof 0 [D 20:00:56.975327] ========================================================= [D 20:00:56.975351] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:56.975377] element 0: offset: 0xbb069000, size: 2097152 [D 20:00:56.975403] post_sr: da13:3346 bh 5 len 32 wr_id 15e72bf8. [D 20:00:56.975433] openib_check_cq: completed recv wr_id 15e73e30. [D 20:00:56.975461] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab6228010 rkey 48007100 nic_wr_credit 1. [D 20:00:56.975488] openib_post_rr: da13:3346 bh 99 wr_id 15e73e30. [D 20:00:56.975515] openib_check_cq: completed send wr_id 15e72bf8. [D 20:00:56.977955] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 0. [D 20:00:56.977983] post_sr: da13:3346 bh 6 len 16 wr_id 15e72c10. [D 20:00:56.978020] openib_check_cq: completed send wr_id 15e72c10. [D 20:00:56.978051] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:56.978076] ========================================================= [D 20:00:56.978100] PINT_process_request [D 20:00:56.978125] tiling 7 copies [D 20:00:56.978149] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.978175] lvl 0 el 0 blk 0 by 180355072 [D 20:00:56.978199] to 180355072 ta 167772160 fi 201326592 [D 20:00:56.978224] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.978249] basic type or contiguous data [D 20:00:56.978273] Request disp [D 20:00:56.978297] contig_offset = 180355072 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.978322] calling distribute [D 20:00:56.978346] PINT_distribute [D 20:00:56.978370] of 180355072 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 180355072 exfl 1 [D 20:00:56.978397] begin iteration loff: 180355072 [D 20:00:56.978422] segment exceeds byte limit [D 20:00:56.978446] file being extended [D 20:00:56.978468] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.978494] **********CALL***PROCESS********* [D 20:00:56.978517] segment of 12582912 sz 2097152 [D 20:00:56.978542] PINT_process_request [D 20:00:56.978566] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.978591] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.978616] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.978641] basic type or contiguous data [D 20:00:56.978667] Request disp [D 20:00:56.978692] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.978716] process a segment [D 20:00:56.978740] of 12582912 sz 2097152 [D 20:00:56.978764] add a segment 0 [D 20:00:56.978788] retval = 2097152 [D 20:00:56.978812] exiting distribute returned less than expected [D 20:00:56.978836] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.978862] *****RETURN***FROM***PROCESS***** [D 20:00:56.978886] end iteration [D 20:00:56.978910] done with segments or bytes [D 20:00:56.978933] of 182452224 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.978959] next loff: 182452224 next poff: 182452224 [D 20:00:56.978983] return value 2097152 [D 20:00:56.979007] finished [D 20:00:56.979031] retval = 2097152 [D 20:00:56.979055] exiting distribute returned less than expected [D 20:00:56.979079] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 182452224 fo 201326592 eof 0 [D 20:00:56.979104] ========================================================= [D 20:00:56.979129] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:56.979154] element 0: offset: 0xb9067000, size: 2097152 [D 20:00:56.979181] post_sr: da13:3346 bh 7 len 32 wr_id 15e72c28. [D 20:00:56.979216] openib_check_cq: completed send wr_id 15e72c28. [D 20:00:56.980546] openib_check_cq: completed recv wr_id 15e734e8. [D 20:00:56.980576] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab642a010 rkey 48007200 nic_wr_credit 1. [D 20:00:56.980603] openib_post_rr: da13:3346 bh 0 wr_id 15e734e8. [D 20:00:56.982802] PINT_client_state_machine_test id 22 [D 20:00:56.983069] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 0. [D 20:00:56.983097] post_sr: da13:3346 bh 8 len 16 wr_id 15e72c40. [D 20:00:56.983133] openib_check_cq: completed send wr_id 15e72c40. [D 20:00:56.983163] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:56.983189] ========================================================= [D 20:00:56.983213] PINT_process_request [D 20:00:56.983237] tiling 8 copies [D 20:00:56.983262] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.983287] lvl 0 el 0 blk 0 by 213909504 [D 20:00:56.983312] to 213909504 ta 201326592 fi 234881024 [D 20:00:56.983337] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.983362] basic type or contiguous data [D 20:00:56.983386] Request disp [D 20:00:56.983409] contig_offset = 213909504 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.983434] calling distribute [D 20:00:56.983458] PINT_distribute [D 20:00:56.983482] of 213909504 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 213909504 exfl 1 [D 20:00:56.983508] begin iteration loff: 213909504 [D 20:00:56.983532] segment exceeds byte limit [D 20:00:56.983556] file being extended [D 20:00:56.983580] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.983605] **********CALL***PROCESS********* [D 20:00:56.983628] segment of 12582912 sz 2097152 [D 20:00:56.983653] PINT_process_request [D 20:00:56.983676] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.983702] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.983726] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.983751] basic type or contiguous data [D 20:00:56.983775] Request disp [D 20:00:56.983798] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.983823] process a segment [D 20:00:56.983847] of 12582912 sz 2097152 [D 20:00:56.983871] add a segment 0 [D 20:00:56.983895] retval = 2097152 [D 20:00:56.983919] exiting distribute returned less than expected [D 20:00:56.983942] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.983968] *****RETURN***FROM***PROCESS***** [D 20:00:56.983993] end iteration [D 20:00:56.984016] done with segments or bytes [D 20:00:56.984040] of 216006656 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.984066] next loff: 216006656 next poff: 216006656 [D 20:00:56.984090] return value 2097152 [D 20:00:56.984114] finished [D 20:00:56.984138] retval = 2097152 [D 20:00:56.984162] exiting distribute returned less than expected [D 20:00:56.984186] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 216006656 fo 234881024 eof 0 [D 20:00:56.984211] ========================================================= [D 20:00:56.984235] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:56.984261] element 0: offset: 0xb7065000, size: 2097152 [D 20:00:56.984288] post_sr: da13:3346 bh 9 len 32 wr_id 15e72c58. [D 20:00:56.984318] openib_check_cq: completed recv wr_id 15e73500. [D 20:00:56.984345] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab662b010 rkey 48007300 nic_wr_credit 1. [D 20:00:56.984372] openib_post_rr: da13:3346 bh 1 wr_id 15e73500. [D 20:00:56.984397] openib_check_cq: completed send wr_id 15e72c58. [D 20:00:56.986839] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 0. [D 20:00:56.986866] post_sr: da13:3346 bh 10 len 16 wr_id 15e72c70. [D 20:00:56.986903] openib_check_cq: completed send wr_id 15e72c70. [D 20:00:56.986934] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:56.986960] ========================================================= [D 20:00:56.986985] PINT_process_request [D 20:00:56.987009] tiling 9 copies [D 20:00:56.987032] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.987061] lvl 0 el 0 blk 0 by 247463936 [D 20:00:56.987087] to 247463936 ta 234881024 fi 268435456 [D 20:00:56.987112] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.987136] basic type or contiguous data [D 20:00:56.987160] Request disp [D 20:00:56.987184] contig_offset = 247463936 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.987209] calling distribute [D 20:00:56.987233] PINT_distribute [D 20:00:56.987257] of 247463936 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 247463936 exfl 1 [D 20:00:56.987282] begin iteration loff: 247463936 [D 20:00:56.987307] segment exceeds byte limit [D 20:00:56.987331] file being extended [D 20:00:56.987355] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.987380] **********CALL***PROCESS********* [D 20:00:56.987403] segment of 12582912 sz 2097152 [D 20:00:56.987427] PINT_process_request [D 20:00:56.987451] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.987478] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.987503] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.987526] basic type or contiguous data [D 20:00:56.987550] Request disp [D 20:00:56.987574] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.987599] process a segment [D 20:00:56.987623] of 12582912 sz 2097152 [D 20:00:56.987648] add a segment 0 [D 20:00:56.987671] retval = 2097152 [D 20:00:56.987695] exiting distribute returned less than expected [D 20:00:56.987718] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.987745] *****RETURN***FROM***PROCESS***** [D 20:00:56.987769] end iteration [D 20:00:56.987792] done with segments or bytes [D 20:00:56.987815] of 249561088 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.987841] next loff: 249561088 next poff: 249561088 [D 20:00:56.987866] return value 2097152 [D 20:00:56.987891] finished [D 20:00:56.987915] retval = 2097152 [D 20:00:56.987938] exiting distribute returned less than expected [D 20:00:56.987962] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 249561088 fo 268435456 eof 0 [D 20:00:56.987988] ========================================================= [D 20:00:56.988012] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:56.988037] element 0: offset: 0xb5063000, size: 2097152 [D 20:00:56.988064] post_sr: da13:3346 bh 11 len 32 wr_id 15e72c88. [D 20:00:56.988100] openib_check_cq: completed send wr_id 15e72c88. [D 20:00:56.988209] openib_check_cq: completed recv wr_id 15e73518. [D 20:00:56.988236] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab682c010 rkey 48007400 nic_wr_credit 1. [D 20:00:56.988263] openib_post_rr: da13:3346 bh 2 wr_id 15e73518. [D 20:00:56.990729] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 0. [D 20:00:56.990757] post_sr: da13:3346 bh 12 len 16 wr_id 15e72ca0. [D 20:00:56.990794] openib_check_cq: completed send wr_id 15e72ca0. [D 20:00:56.990824] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:56.990850] ========================================================= [D 20:00:56.990874] PINT_process_request [D 20:00:56.990899] tiling 10 copies [D 20:00:56.990922] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.990949] lvl 0 el 0 blk 0 by 281018368 [D 20:00:56.990974] to 281018368 ta 268435456 fi 301989888 [D 20:00:56.990999] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.991023] basic type or contiguous data [D 20:00:56.991046] Request disp [D 20:00:56.991070] contig_offset = 281018368 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.991095] calling distribute [D 20:00:56.991119] PINT_distribute [D 20:00:56.991143] of 281018368 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 281018368 exfl 1 [D 20:00:56.991169] begin iteration loff: 281018368 [D 20:00:56.991194] segment exceeds byte limit [D 20:00:56.991218] file being extended [D 20:00:56.991244] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.991270] **********CALL***PROCESS********* [D 20:00:56.991293] segment of 12582912 sz 2097152 [D 20:00:56.991317] PINT_process_request [D 20:00:56.991341] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.991368] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.991393] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.991417] basic type or contiguous data [D 20:00:56.991441] Request disp [D 20:00:56.991464] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.991489] process a segment [D 20:00:56.991513] of 12582912 sz 2097152 [D 20:00:56.991537] add a segment 0 [D 20:00:56.991561] retval = 2097152 [D 20:00:56.991585] exiting distribute returned less than expected [D 20:00:56.991608] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.991635] *****RETURN***FROM***PROCESS***** [D 20:00:56.991659] end iteration [D 20:00:56.991683] done with segments or bytes [D 20:00:56.991705] of 283115520 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.991731] next loff: 283115520 next poff: 283115520 [D 20:00:56.991756] return value 2097152 [D 20:00:56.991780] finished [D 20:00:56.991804] retval = 2097152 [D 20:00:56.991827] exiting distribute returned less than expected [D 20:00:56.991851] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 283115520 fo 301989888 eof 0 [D 20:00:56.991878] ========================================================= [D 20:00:56.991902] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:56.991928] element 0: offset: 0xb3061000, size: 2097152 [D 20:00:56.991954] post_sr: da13:3346 bh 13 len 32 wr_id 15e72cb8. [D 20:00:56.991984] openib_check_cq: completed recv wr_id 15e73530. [D 20:00:56.992011] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaab6a2d010 rkey 48007500 nic_wr_credit 1. [D 20:00:56.992037] openib_post_rr: da13:3346 bh 3 wr_id 15e73530. [D 20:00:56.992064] openib_check_cq: completed send wr_id 15e72cb8. [D 20:00:56.992831] PINT_client_state_machine_test id 22 [D 20:00:56.994503] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 0. [D 20:00:56.994530] post_sr: da13:3346 bh 14 len 16 wr_id 15e72cd0. [D 20:00:56.994566] openib_check_cq: completed send wr_id 15e72cd0. [D 20:00:56.994597] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:56.994623] ========================================================= [D 20:00:56.994647] PINT_process_request [D 20:00:56.994673] tiling 11 copies [D 20:00:56.994698] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.994724] lvl 0 el 0 blk 0 by 314572800 [D 20:00:56.994749] to 314572800 ta 301989888 fi 335544320 [D 20:00:56.994773] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.994798] basic type or contiguous data [D 20:00:56.994822] Request disp [D 20:00:56.994846] contig_offset = 314572800 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.994871] calling distribute [D 20:00:56.994894] PINT_distribute [D 20:00:56.994918] of 314572800 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 314572800 exfl 1 [D 20:00:56.994944] begin iteration loff: 314572800 [D 20:00:56.994969] segment exceeds byte limit [D 20:00:56.994993] file being extended [D 20:00:56.995017] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.995041] **********CALL***PROCESS********* [D 20:00:56.995065] segment of 12582912 sz 2097152 [D 20:00:56.995089] PINT_process_request [D 20:00:56.995113] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.995139] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.995163] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.995188] basic type or contiguous data [D 20:00:56.995212] Request disp [D 20:00:56.995236] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.995261] process a segment [D 20:00:56.995285] of 12582912 sz 2097152 [D 20:00:56.995308] add a segment 0 [D 20:00:56.995337] retval = 2097152 [D 20:00:56.995362] exiting distribute returned less than expected [D 20:00:56.995386] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.995411] *****RETURN***FROM***PROCESS***** [D 20:00:56.995436] end iteration [D 20:00:56.995460] done with segments or bytes [D 20:00:56.995484] of 316669952 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.995509] next loff: 316669952 next poff: 316669952 [D 20:00:56.995533] return value 2097152 [D 20:00:56.995557] finished [D 20:00:56.995581] retval = 2097152 [D 20:00:56.995605] exiting distribute returned less than expected [D 20:00:56.995629] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 316669952 fo 335544320 eof 0 [D 20:00:56.995656] ========================================================= [D 20:00:56.995679] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:56.995704] element 0: offset: 0xb105f000, size: 2097152 [D 20:00:56.995732] post_sr: da13:3346 bh 15 len 32 wr_id 15e72ce8. [D 20:00:56.995761] openib_check_cq: completed recv wr_id 15e73548. [D 20:00:56.995788] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaab6c2e010 rkey 48007600 nic_wr_credit 1. [D 20:00:56.995815] openib_post_rr: da13:3346 bh 4 wr_id 15e73548. [D 20:00:56.995841] openib_check_cq: completed send wr_id 15e72ce8. [D 20:00:56.998281] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 0. [D 20:00:56.998309] post_sr: da13:3346 bh 16 len 16 wr_id 15e72d00. [D 20:00:56.998345] openib_check_cq: completed send wr_id 15e72d00. [D 20:00:56.998375] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:56.998401] ========================================================= [D 20:00:56.998425] PINT_process_request [D 20:00:56.998449] tiling 12 copies [D 20:00:56.998473] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.998498] lvl 0 el 0 blk 0 by 348127232 [D 20:00:56.998523] to 348127232 ta 335544320 fi 369098752 [D 20:00:56.998548] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:56.998573] basic type or contiguous data [D 20:00:56.998596] Request disp [D 20:00:56.998620] contig_offset = 348127232 contig_size = 54525952 lvl_flag = 1 [D 20:00:56.998644] calling distribute [D 20:00:56.998671] PINT_distribute [D 20:00:56.998695] of 348127232 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 348127232 exfl 1 [D 20:00:56.998722] begin iteration loff: 348127232 [D 20:00:56.998746] segment exceeds byte limit [D 20:00:56.998770] file being extended [D 20:00:56.998794] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:56.998819] **********CALL***PROCESS********* [D 20:00:56.998843] segment of 12582912 sz 2097152 [D 20:00:56.998867] PINT_process_request [D 20:00:56.998890] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:56.998916] lvl 0 el 0 blk 0 by 12582912 [D 20:00:56.998941] to 12582912 ta 12582912 fi 14680064 [D 20:00:56.998966] basic type or contiguous data [D 20:00:56.998990] Request disp [D 20:00:56.999013] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:56.999038] process a segment [D 20:00:56.999061] of 12582912 sz 2097152 [D 20:00:56.999086] add a segment 0 [D 20:00:56.999109] retval = 2097152 [D 20:00:56.999133] exiting distribute returned less than expected [D 20:00:56.999156] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:56.999183] *****RETURN***FROM***PROCESS***** [D 20:00:56.999207] end iteration [D 20:00:56.999230] done with segments or bytes [D 20:00:56.999254] of 350224384 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:56.999279] next loff: 350224384 next poff: 350224384 [D 20:00:56.999303] return value 2097152 [D 20:00:56.999327] finished [D 20:00:56.999351] retval = 2097152 [D 20:00:56.999375] exiting distribute returned less than expected [D 20:00:56.999399] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 350224384 fo 369098752 eof 0 [D 20:00:56.999427] ========================================================= [D 20:00:56.999452] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:56.999477] element 0: offset: 0xaf05d000, size: 2097152 [D 20:00:56.999505] post_sr: da13:3346 bh 17 len 32 wr_id 15e72d18. [D 20:00:56.999540] openib_check_cq: completed send wr_id 15e72d18. [D 20:00:57.001659] openib_check_cq: completed recv wr_id 15e73560. [D 20:00:57.001687] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab6e2f010 rkey 48007700 nic_wr_credit 1. [D 20:00:57.001713] openib_post_rr: da13:3346 bh 5 wr_id 15e73560. [D 20:00:57.002860] PINT_client_state_machine_test id 22 [D 20:00:57.004180] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 0. [D 20:00:57.004208] post_sr: da13:3346 bh 18 len 16 wr_id 15e72d30. [D 20:00:57.004244] openib_check_cq: completed send wr_id 15e72d30. [D 20:00:57.004275] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:57.004301] ========================================================= [D 20:00:57.004325] PINT_process_request [D 20:00:57.004349] tiling 13 copies [D 20:00:57.004373] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.004400] lvl 0 el 0 blk 0 by 381681664 [D 20:00:57.004425] to 381681664 ta 369098752 fi 402653184 [D 20:00:57.004449] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:57.004474] basic type or contiguous data [D 20:00:57.004498] Request disp [D 20:00:57.004522] contig_offset = 381681664 contig_size = 54525952 lvl_flag = 1 [D 20:00:57.004547] calling distribute [D 20:00:57.004570] PINT_distribute [D 20:00:57.004594] of 381681664 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 381681664 exfl 1 [D 20:00:57.004621] begin iteration loff: 381681664 [D 20:00:57.004646] segment exceeds byte limit [D 20:00:57.004673] file being extended [D 20:00:57.004701] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:57.004729] **********CALL***PROCESS********* [D 20:00:57.004756] segment of 12582912 sz 2097152 [D 20:00:57.004784] PINT_process_request [D 20:00:57.004808] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.004834] lvl 0 el 0 blk 0 by 12582912 [D 20:00:57.004858] to 12582912 ta 12582912 fi 14680064 [D 20:00:57.004883] basic type or contiguous data [D 20:00:57.004908] Request disp [D 20:00:57.004932] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:57.004957] process a segment [D 20:00:57.004981] of 12582912 sz 2097152 [D 20:00:57.005004] add a segment 0 [D 20:00:57.005028] retval = 2097152 [D 20:00:57.005052] exiting distribute returned less than expected [D 20:00:57.005076] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:57.005103] *****RETURN***FROM***PROCESS***** [D 20:00:57.005126] end iteration [D 20:00:57.005150] done with segments or bytes [D 20:00:57.005174] of 383778816 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.005200] next loff: 383778816 next poff: 383778816 [D 20:00:57.005224] return value 2097152 [D 20:00:57.005249] finished [D 20:00:57.005272] retval = 2097152 [D 20:00:57.005296] exiting distribute returned less than expected [D 20:00:57.005320] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 383778816 fo 402653184 eof 0 [D 20:00:57.005346] ========================================================= [D 20:00:57.005370] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:57.005395] element 0: offset: 0xad05b000, size: 2097152 [D 20:00:57.005422] post_sr: da13:3346 bh 19 len 32 wr_id 15e72d48. [D 20:00:57.005457] openib_check_cq: completed send wr_id 15e72d48. [D 20:00:57.012888] PINT_client_state_machine_test id 22 [D 20:00:57.022917] PINT_client_state_machine_test id 22 [D 20:00:57.032947] PINT_client_state_machine_test id 22 [D 20:00:57.042978] PINT_client_state_machine_test id 22 [D 20:00:57.058669] PINT_client_state_machine_test id 22 [D 20:00:57.070669] PINT_client_state_machine_test id 22 [D 20:00:57.082669] PINT_client_state_machine_test id 22 [D 20:00:57.094669] PINT_client_state_machine_test id 22 [D 20:00:57.106669] PINT_client_state_machine_test id 22 [D 20:00:57.118669] PINT_client_state_machine_test id 22 [D 20:00:57.130669] PINT_client_state_machine_test id 22 [D 20:00:57.142668] PINT_client_state_machine_test id 22 [D 20:00:57.170669] PINT_client_state_machine_test id 22 [D 20:00:57.182668] PINT_client_state_machine_test id 22 [D 20:00:57.194669] PINT_client_state_machine_test id 22 [D 20:00:57.206669] PINT_client_state_machine_test id 22 [D 20:00:57.218669] PINT_client_state_machine_test id 22 [D 20:00:57.230669] PINT_client_state_machine_test id 22 [D 20:00:57.242669] PINT_client_state_machine_test id 22 [D 20:00:57.254669] PINT_client_state_machine_test id 22 [D 20:00:57.266669] PINT_client_state_machine_test id 22 [D 20:00:57.278669] PINT_client_state_machine_test id 22 [D 20:00:57.290670] PINT_client_state_machine_test id 22 [D 20:00:57.302669] PINT_client_state_machine_test id 22 -[D 20:00:57.310647] openib_check_cq: completed recv wr_id 15e73578. [D 20:00:57.310685] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaab7030010 rkey c8007800 nic_wr_credit 1. [D 20:00:57.310716] openib_post_rr: da13:3346 bh 6 wr_id 15e73578. [D 20:00:57.312705] PINT_client_state_machine_test id 22 [D 20:00:57.313180] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 0. [D 20:00:57.313208] post_sr: da13:3346 bh 20 len 16 wr_id 15e72d60. [D 20:00:57.313245] openib_check_cq: completed send wr_id 15e72d60. [D 20:00:57.313276] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:00:57.313302] ========================================================= [D 20:00:57.313327] PINT_process_request [D 20:00:57.313352] tiling 14 copies [D 20:00:57.313376] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.313403] lvl 0 el 0 blk 0 by 415236096 [D 20:00:57.313427] to 415236096 ta 402653184 fi 436207616 [D 20:00:57.313452] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:57.313477] basic type or contiguous data [D 20:00:57.313502] Request disp [D 20:00:57.313526] contig_offset = 415236096 contig_size = 54525952 lvl_flag = 1 [D 20:00:57.313550] calling distribute [D 20:00:57.313574] PINT_distribute [D 20:00:57.313598] of 415236096 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 415236096 exfl 1 [D 20:00:57.313626] begin iteration loff: 415236096 [D 20:00:57.313651] segment exceeds byte limit [D 20:00:57.313674] file being extended [D 20:00:57.313698] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:57.313724] **********CALL***PROCESS********* [D 20:00:57.313748] segment of 12582912 sz 2097152 [D 20:00:57.313772] PINT_process_request [D 20:00:57.313796] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.313822] lvl 0 el 0 blk 0 by 12582912 [D 20:00:57.313847] to 12582912 ta 12582912 fi 14680064 [D 20:00:57.313872] basic type or contiguous data [D 20:00:57.313896] Request disp [D 20:00:57.313920] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:57.313945] process a segment [D 20:00:57.313968] of 12582912 sz 2097152 [D 20:00:57.313993] add a segment 0 [D 20:00:57.314017] retval = 2097152 [D 20:00:57.314041] exiting distribute returned less than expected [D 20:00:57.314064] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:57.314091] *****RETURN***FROM***PROCESS***** [D 20:00:57.314115] end iteration [D 20:00:57.314139] done with segments or bytes [D 20:00:57.314164] of 417333248 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.314195] next loff: 417333248 next poff: 417333248 [D 20:00:57.314224] return value 2097152 [D 20:00:57.314251] finished [D 20:00:57.314279] retval = 2097152 [D 20:00:57.314303] exiting distribute returned less than expected [D 20:00:57.314328] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 417333248 fo 436207616 eof 0 [D 20:00:57.314357] ========================================================= [D 20:00:57.314383] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:00:57.314408] element 0: offset: 0xab059000, size: 2097152 [D 20:00:57.314435] post_sr: da13:3346 bh 21 len 32 wr_id 15e72d78. [D 20:00:57.314467] openib_check_cq: completed recv wr_id 15e73590. [D 20:00:57.314494] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab7231010 rkey c8007900 nic_wr_credit 1. [D 20:00:57.314520] openib_post_rr: da13:3346 bh 7 wr_id 15e73590. [D 20:00:57.314546] openib_check_cq: completed send wr_id 15e72d78. [D 20:00:57.316984] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 0. [D 20:00:57.317011] post_sr: da13:3346 bh 22 len 16 wr_id 15e72d90. [D 20:00:57.317048] openib_check_cq: completed send wr_id 15e72d90. [D 20:00:57.317078] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:00:57.317104] ========================================================= [D 20:00:57.317129] PINT_process_request [D 20:00:57.317152] tiling 15 copies [D 20:00:57.317176] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.317203] lvl 0 el 0 blk 0 by 448790528 [D 20:00:57.317228] to 448790528 ta 436207616 fi 469762048 [D 20:00:57.317253] mto 12582912 mta 10485760 mfi 12582912 [D 20:00:57.317277] basic type or contiguous data [D 20:00:57.317301] Request disp [D 20:00:57.317325] contig_offset = 448790528 contig_size = 54525952 lvl_flag = 1 [D 20:00:57.317351] calling distribute [D 20:00:57.317375] PINT_distribute [D 20:00:57.317399] of 448790528 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 448790528 exfl 1 [D 20:00:57.317425] begin iteration loff: 448790528 [D 20:00:57.317450] segment exceeds byte limit [D 20:00:57.317474] file being extended [D 20:00:57.317498] client lstof 12582912 diff 0 sgof 12582912 [D 20:00:57.317523] **********CALL***PROCESS********* [D 20:00:57.317546] segment of 12582912 sz 2097152 [D 20:00:57.317570] PINT_process_request [D 20:00:57.317594] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.317621] lvl 0 el 0 blk 0 by 12582912 [D 20:00:57.317646] to 12582912 ta 12582912 fi 14680064 [D 20:00:57.317670] basic type or contiguous data [D 20:00:57.317694] Request disp [D 20:00:57.317718] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:00:57.317743] process a segment [D 20:00:57.317767] of 12582912 sz 2097152 [D 20:00:57.317791] add a segment 0 [D 20:00:57.317815] retval = 2097152 [D 20:00:57.317839] exiting distribute returned less than expected [D 20:00:57.317863] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:00:57.317890] *****RETURN***FROM***PROCESS***** [D 20:00:57.317914] end iteration [D 20:00:57.317937] done with segments or bytes [D 20:00:57.317961] of 450887680 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.317987] next loff: 450887680 next poff: 450887680 [D 20:00:57.318012] return value 2097152 [D 20:00:57.318037] finished [D 20:00:57.318060] retval = 2097152 [D 20:00:57.318084] exiting distribute returned less than expected [D 20:00:57.318108] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 450887680 fo 469762048 eof 0 [D 20:00:57.318134] ========================================================= [D 20:00:57.318159] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:00:57.318189] element 0: offset: 0xa9057000, size: 2097152 [D 20:00:57.318221] post_sr: da13:3346 bh 23 len 32 wr_id 15e72da8. [D 20:00:57.318261] openib_check_cq: completed send wr_id 15e72da8. [D 20:00:57.318806] openib_check_cq: completed recv wr_id 15e735a8. [D 20:00:57.318833] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab7432010 rkey c8007a00 nic_wr_credit 1. [D 20:00:57.318860] openib_post_rr: da13:3346 bh 8 wr_id 15e735a8. [D 20:00:57.321322] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 0. [D 20:00:57.321352] post_sr: da13:3346 bh 24 len 16 wr_id 15e72dc0. [D 20:00:57.321389] openib_check_cq: completed send wr_id 15e72dc0. [D 20:00:57.321419] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:57.321446] ========================================================= [D 20:00:57.321470] PINT_process_request [D 20:00:57.321494] tiling 2 copies [D 20:00:57.321518] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.321545] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.321570] to 14680064 ta 0 fi 33554432 [D 20:00:57.321595] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.321619] basic type or contiguous data [D 20:00:57.321643] Request disp [D 20:00:57.321667] contig_offset = 14680064 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.321693] calling distribute [D 20:00:57.321719] PINT_distribute [D 20:00:57.321743] of 14680064 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 14680064 exfl 1 [D 20:00:57.321769] begin iteration loff: 14680064 [D 20:00:57.321794] segment exceeds byte limit [D 20:00:57.321818] file being extended [D 20:00:57.321842] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.321866] **********CALL***PROCESS********* [D 20:00:57.321890] segment of 14680064 sz 2097152 [D 20:00:57.321914] PINT_process_request [D 20:00:57.321939] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.321965] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.321990] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.322014] basic type or contiguous data [D 20:00:57.322039] Request disp [D 20:00:57.322063] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.322088] process a segment [D 20:00:57.322112] of 14680064 sz 2097152 [D 20:00:57.322136] add a segment 0 [D 20:00:57.322160] retval = 2097152 [D 20:00:57.322190] exiting distribute returned less than expected [D 20:00:57.322217] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.322248] *****RETURN***FROM***PROCESS***** [D 20:00:57.322276] end iteration [D 20:00:57.322299] done with segments or bytes [D 20:00:57.322323] of 16777216 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.322349] next loff: 16777216 next poff: 16777216 [D 20:00:57.322374] return value 2097152 [D 20:00:57.322399] finished [D 20:00:57.322422] retval = 2097152 [D 20:00:57.322446] exiting distribute returned less than expected [D 20:00:57.322470] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 16777216 fo 33554432 eof 0 [D 20:00:57.322497] ========================================================= [D 20:00:57.322521] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:57.322546] element 0: offset: 0xc3271000, size: 2097152 [D 20:00:57.322573] post_sr: da13:3346 bh 25 len 32 wr_id 15e72dd8. [D 20:00:57.322604] openib_check_cq: completed recv wr_id 15e735c0. [D 20:00:57.322630] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab7633010 rkey c8007b00 nic_wr_credit 1. [D 20:00:57.322658] openib_post_rr: da13:3346 bh 9 wr_id 15e735c0. [D 20:00:57.322684] openib_check_cq: completed send wr_id 15e72dd8. [D 20:00:57.322731] PINT_client_state_machine_test id 22 [D 20:00:57.325125] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 0. [D 20:00:57.325153] post_sr: da13:3346 bh 26 len 16 wr_id 15e72df0. [D 20:00:57.325189] openib_check_cq: completed send wr_id 15e72df0. [D 20:00:57.325220] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:57.325246] ========================================================= [D 20:00:57.325270] PINT_process_request [D 20:00:57.325295] tiling 3 copies [D 20:00:57.325319] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.325345] lvl 0 el 0 blk 0 by 48234496 [D 20:00:57.325370] to 48234496 ta 33554432 fi 67108864 [D 20:00:57.325395] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.325420] basic type or contiguous data [D 20:00:57.325447] Request disp [D 20:00:57.325471] contig_offset = 48234496 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.325496] calling distribute [D 20:00:57.325521] PINT_distribute [D 20:00:57.325545] of 48234496 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 48234496 exfl 1 [D 20:00:57.325571] begin iteration loff: 48234496 [D 20:00:57.325596] segment exceeds byte limit [D 20:00:57.325620] file being extended [D 20:00:57.325644] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.325669] **********CALL***PROCESS********* [D 20:00:57.325693] segment of 14680064 sz 2097152 [D 20:00:57.325717] PINT_process_request [D 20:00:57.325741] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.325768] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.325793] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.325818] basic type or contiguous data [D 20:00:57.325841] Request disp [D 20:00:57.325865] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.325890] process a segment [D 20:00:57.325914] of 14680064 sz 2097152 [D 20:00:57.325939] add a segment 0 [D 20:00:57.325963] retval = 2097152 [D 20:00:57.325986] exiting distribute returned less than expected [D 20:00:57.326011] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.326037] *****RETURN***FROM***PROCESS***** [D 20:00:57.326062] end iteration [D 20:00:57.326086] done with segments or bytes [D 20:00:57.326109] of 50331648 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.326135] next loff: 50331648 next poff: 50331648 [D 20:00:57.326160] return value 2097152 [D 20:00:57.326189] finished [D 20:00:57.326216] retval = 2097152 [D 20:00:57.326244] exiting distribute returned less than expected [D 20:00:57.326272] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 50331648 fo 67108864 eof 0 [D 20:00:57.326299] ========================================================= [D 20:00:57.326324] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:57.326349] element 0: offset: 0xc126f000, size: 2097152 [D 20:00:57.326376] post_sr: da13:3346 bh 27 len 32 wr_id 15e72e08. [D 20:00:57.326413] openib_check_cq: completed send wr_id 15e72e08. [D 20:00:57.326539] openib_check_cq: completed recv wr_id 15e735d8. [D 20:00:57.326565] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab7834010 rkey c8007c00 nic_wr_credit 1. [D 20:00:57.326593] openib_post_rr: da13:3346 bh 10 wr_id 15e735d8. [D 20:00:57.329058] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 0. [D 20:00:57.329086] post_sr: da13:3346 bh 28 len 16 wr_id 15e72e20. [D 20:00:57.329122] openib_check_cq: completed send wr_id 15e72e20. [D 20:00:57.329153] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:57.329178] ========================================================= [D 20:00:57.329203] PINT_process_request [D 20:00:57.329227] tiling 4 copies [D 20:00:57.329252] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.329277] lvl 0 el 0 blk 0 by 81788928 [D 20:00:57.329303] to 81788928 ta 67108864 fi 100663296 [D 20:00:57.329328] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.329353] basic type or contiguous data [D 20:00:57.329377] Request disp [D 20:00:57.329401] contig_offset = 81788928 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.329425] calling distribute [D 20:00:57.329450] PINT_distribute [D 20:00:57.329474] of 81788928 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 81788928 exfl 1 [D 20:00:57.329501] begin iteration loff: 81788928 [D 20:00:57.329525] segment exceeds byte limit [D 20:00:57.329549] file being extended [D 20:00:57.329573] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.329598] **********CALL***PROCESS********* [D 20:00:57.329622] segment of 14680064 sz 2097152 [D 20:00:57.329646] PINT_process_request [D 20:00:57.329669] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.329699] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.329724] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.329749] basic type or contiguous data [D 20:00:57.329772] Request disp [D 20:00:57.329796] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.329822] process a segment [D 20:00:57.329846] of 14680064 sz 2097152 [D 20:00:57.329870] add a segment 0 [D 20:00:57.329895] retval = 2097152 [D 20:00:57.329918] exiting distribute returned less than expected [D 20:00:57.329942] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.329969] *****RETURN***FROM***PROCESS***** [D 20:00:57.329993] end iteration [D 20:00:57.330017] done with segments or bytes [D 20:00:57.330040] of 83886080 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.330066] next loff: 83886080 next poff: 83886080 [D 20:00:57.330091] return value 2097152 [D 20:00:57.330116] finished [D 20:00:57.330139] retval = 2097152 [D 20:00:57.330164] exiting distribute returned less than expected [D 20:00:57.330193] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 83886080 fo 100663296 eof 0 [D 20:00:57.330224] ========================================================= [D 20:00:57.330251] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:57.330281] element 0: offset: 0xbf26d000, size: 2097152 [D 20:00:57.330308] post_sr: da13:3346 bh 29 len 32 wr_id 15e72e38. [D 20:00:57.330344] openib_check_cq: completed send wr_id 15e72e38. [D 20:00:57.330559] openib_check_cq: completed recv wr_id 15e735f0. [D 20:00:57.330586] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaab7a35010 rkey c8007d00 nic_wr_credit 1. [D 20:00:57.330613] openib_post_rr: da13:3346 bh 11 wr_id 15e735f0. [D 20:00:57.332763] PINT_client_state_machine_test id 22 [D 20:00:57.333076] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 0. [D 20:00:57.333104] post_sr: da13:3346 bh 30 len 16 wr_id 15e72e50. [D 20:00:57.333141] openib_check_cq: completed send wr_id 15e72e50. [D 20:00:57.333171] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:57.333197] ========================================================= [D 20:00:57.333222] PINT_process_request [D 20:00:57.333246] tiling 5 copies [D 20:00:57.333271] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.333296] lvl 0 el 0 blk 0 by 115343360 [D 20:00:57.333322] to 115343360 ta 100663296 fi 134217728 [D 20:00:57.333347] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.333372] basic type or contiguous data [D 20:00:57.333395] Request disp [D 20:00:57.333419] contig_offset = 115343360 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.333445] calling distribute [D 20:00:57.333469] PINT_distribute [D 20:00:57.333493] of 115343360 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 115343360 exfl 1 [D 20:00:57.333519] begin iteration loff: 115343360 [D 20:00:57.333544] segment exceeds byte limit [D 20:00:57.333568] file being extended [D 20:00:57.333592] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.333617] **********CALL***PROCESS********* [D 20:00:57.333641] segment of 14680064 sz 2097152 [D 20:00:57.333665] PINT_process_request [D 20:00:57.333689] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.333715] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.333741] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.333765] basic type or contiguous data [D 20:00:57.333789] Request disp [D 20:00:57.333813] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.333838] process a segment [D 20:00:57.333862] of 14680064 sz 2097152 [D 20:00:57.333886] add a segment 0 [D 20:00:57.333910] retval = 2097152 [D 20:00:57.333934] exiting distribute returned less than expected [D 20:00:57.333958] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.333984] *****RETURN***FROM***PROCESS***** [D 20:00:57.334009] end iteration [D 20:00:57.334035] done with segments or bytes [D 20:00:57.334059] of 117440512 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.334085] next loff: 117440512 next poff: 117440512 [D 20:00:57.334110] return value 2097152 [D 20:00:57.334134] finished [D 20:00:57.334157] retval = 2097152 [D 20:00:57.334187] exiting distribute returned less than expected [D 20:00:57.334215] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 117440512 fo 134217728 eof 0 [D 20:00:57.334245] ========================================================= [D 20:00:57.334274] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:57.334300] element 0: offset: 0xbd26b000, size: 2097152 [D 20:00:57.334327] post_sr: da13:3346 bh 31 len 32 wr_id 15e72e68. [D 20:00:57.334358] openib_check_cq: completed recv wr_id 15e73608. [D 20:00:57.334384] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab7c36010 rkey c8007e00 nic_wr_credit 1. [D 20:00:57.334411] openib_post_rr: da13:3346 bh 12 wr_id 15e73608. [D 20:00:57.334438] openib_check_cq: completed send wr_id 15e72e68. [D 20:00:57.336876] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 0. [D 20:00:57.336904] post_sr: da13:3346 bh 32 len 16 wr_id 15e72e80. [D 20:00:57.336940] openib_check_cq: completed send wr_id 15e72e80. [D 20:00:57.336971] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:57.336997] ========================================================= [D 20:00:57.337021] PINT_process_request [D 20:00:57.337045] tiling 6 copies [D 20:00:57.337069] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.337096] lvl 0 el 0 blk 0 by 148897792 [D 20:00:57.337121] to 148897792 ta 134217728 fi 167772160 [D 20:00:57.337145] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.337170] basic type or contiguous data [D 20:00:57.337194] Request disp [D 20:00:57.337219] contig_offset = 148897792 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.337244] calling distribute [D 20:00:57.337267] PINT_distribute [D 20:00:57.337291] of 148897792 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 148897792 exfl 1 [D 20:00:57.337318] begin iteration loff: 148897792 [D 20:00:57.337343] segment exceeds byte limit [D 20:00:57.337368] file being extended [D 20:00:57.337391] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.337415] **********CALL***PROCESS********* [D 20:00:57.337439] segment of 14680064 sz 2097152 [D 20:00:57.337464] PINT_process_request [D 20:00:57.337489] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.337514] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.337539] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.337564] basic type or contiguous data [D 20:00:57.337588] Request disp [D 20:00:57.337613] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.337637] process a segment [D 20:00:57.337661] of 14680064 sz 2097152 [D 20:00:57.337686] add a segment 0 [D 20:00:57.337710] retval = 2097152 [D 20:00:57.337734] exiting distribute returned less than expected [D 20:00:57.337758] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.337784] *****RETURN***FROM***PROCESS***** [D 20:00:57.337808] end iteration [D 20:00:57.337833] done with segments or bytes [D 20:00:57.337857] of 150994944 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.337882] next loff: 150994944 next poff: 150994944 [D 20:00:57.337907] return value 2097152 [D 20:00:57.337931] finished [D 20:00:57.337955] retval = 2097152 [D 20:00:57.337979] exiting distribute returned less than expected [D 20:00:57.338003] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 150994944 fo 167772160 eof 0 [D 20:00:57.338029] ========================================================= [D 20:00:57.338053] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:57.338079] element 0: offset: 0xbb269000, size: 2097152 [D 20:00:57.338109] post_sr: da13:3346 bh 33 len 32 wr_id 15e72e98. [D 20:00:57.338139] openib_check_cq: completed recv wr_id 15e73620. [D 20:00:57.338169] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab7e37010 rkey c8007f00 nic_wr_credit 1. [D 20:00:57.338200] openib_post_rr: da13:3346 bh 13 wr_id 15e73620. [D 20:00:57.338231] openib_check_cq: completed send wr_id 15e72e98. [D 20:00:57.340667] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 0. [D 20:00:57.340695] post_sr: da13:3346 bh 34 len 16 wr_id 15e72eb0. [D 20:00:57.340732] openib_check_cq: completed send wr_id 15e72eb0. [D 20:00:57.340763] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:57.340788] ========================================================= [D 20:00:57.340812] PINT_process_request [D 20:00:57.340837] tiling 7 copies [D 20:00:57.340861] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.340888] lvl 0 el 0 blk 0 by 182452224 [D 20:00:57.340912] to 182452224 ta 167772160 fi 201326592 [D 20:00:57.340937] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.340962] basic type or contiguous data [D 20:00:57.340986] Request disp [D 20:00:57.341011] contig_offset = 182452224 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.341035] calling distribute [D 20:00:57.341059] PINT_distribute [D 20:00:57.341083] of 182452224 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 182452224 exfl 1 [D 20:00:57.341111] begin iteration loff: 182452224 [D 20:00:57.341135] segment exceeds byte limit [D 20:00:57.341159] file being extended [D 20:00:57.341183] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.341208] **********CALL***PROCESS********* [D 20:00:57.341232] segment of 14680064 sz 2097152 [D 20:00:57.341257] PINT_process_request [D 20:00:57.341280] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.341307] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.341332] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.341357] basic type or contiguous data [D 20:00:57.341381] Request disp [D 20:00:57.341404] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.341429] process a segment [D 20:00:57.341453] of 14680064 sz 2097152 [D 20:00:57.341478] add a segment 0 [D 20:00:57.341502] retval = 2097152 [D 20:00:57.341526] exiting distribute returned less than expected [D 20:00:57.341550] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.341576] *****RETURN***FROM***PROCESS***** [D 20:00:57.341600] end iteration [D 20:00:57.341624] done with segments or bytes [D 20:00:57.341648] of 184549376 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.341673] next loff: 184549376 next poff: 184549376 [D 20:00:57.341698] return value 2097152 [D 20:00:57.341725] finished [D 20:00:57.341749] retval = 2097152 [D 20:00:57.341773] exiting distribute returned less than expected [D 20:00:57.341796] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 184549376 fo 201326592 eof 0 [D 20:00:57.341823] ========================================================= [D 20:00:57.341847] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:57.341873] element 0: offset: 0xb9267000, size: 2097152 [D 20:00:57.341900] post_sr: da13:3346 bh 35 len 32 wr_id 15e72ec8. [D 20:00:57.341930] openib_check_cq: completed recv wr_id 15e73638. [D 20:00:57.341958] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab8038010 rkey 48008000 nic_wr_credit 1. [D 20:00:57.341984] openib_post_rr: da13:3346 bh 14 wr_id 15e73638. [D 20:00:57.342010] openib_check_cq: completed send wr_id 15e72ec8. [D 20:00:57.342792] PINT_client_state_machine_test id 22 [D 20:00:57.344448] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 0. [D 20:00:57.344474] post_sr: da13:3346 bh 36 len 16 wr_id 15e72ee0. [D 20:00:57.344511] openib_check_cq: completed send wr_id 15e72ee0. [D 20:00:57.344541] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:57.344570] ========================================================= [D 20:00:57.344594] PINT_process_request [D 20:00:57.344619] tiling 8 copies [D 20:00:57.344643] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.344670] lvl 0 el 0 blk 0 by 216006656 [D 20:00:57.344695] to 216006656 ta 201326592 fi 234881024 [D 20:00:57.344719] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.344744] basic type or contiguous data [D 20:00:57.344768] Request disp [D 20:00:57.344792] contig_offset = 216006656 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.344818] calling distribute [D 20:00:57.344841] PINT_distribute [D 20:00:57.344865] of 216006656 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 216006656 exfl 1 [D 20:00:57.344891] begin iteration loff: 216006656 [D 20:00:57.344916] segment exceeds byte limit [D 20:00:57.344941] file being extended [D 20:00:57.344964] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.344988] **********CALL***PROCESS********* [D 20:00:57.345012] segment of 14680064 sz 2097152 [D 20:00:57.345037] PINT_process_request [D 20:00:57.345061] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.345087] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.345111] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.345136] basic type or contiguous data [D 20:00:57.345160] Request disp [D 20:00:57.345184] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.345209] process a segment [D 20:00:57.345232] of 14680064 sz 2097152 [D 20:00:57.345256] add a segment 0 [D 20:00:57.345280] retval = 2097152 [D 20:00:57.345305] exiting distribute returned less than expected [D 20:00:57.345329] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.345355] *****RETURN***FROM***PROCESS***** [D 20:00:57.345379] end iteration [D 20:00:57.345403] done with segments or bytes [D 20:00:57.345426] of 218103808 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.345453] next loff: 218103808 next poff: 218103808 [D 20:00:57.345478] return value 2097152 [D 20:00:57.345501] finished [D 20:00:57.345525] retval = 2097152 [D 20:00:57.345549] exiting distribute returned less than expected [D 20:00:57.345573] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 218103808 fo 234881024 eof 0 [D 20:00:57.345600] ========================================================= [D 20:00:57.345623] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:57.345648] element 0: offset: 0xb7265000, size: 2097152 [D 20:00:57.345676] post_sr: da13:3346 bh 37 len 32 wr_id 15e72ef8. [D 20:00:57.345711] openib_check_cq: completed send wr_id 15e72ef8. [D 20:00:57.345895] openib_check_cq: completed recv wr_id 15e73650. [D 20:00:57.345923] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab8239010 rkey 48008100 nic_wr_credit 1. [D 20:00:57.345950] openib_post_rr: da13:3346 bh 15 wr_id 15e73650. [D 20:00:57.348415] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 0. [D 20:00:57.348443] post_sr: da13:3346 bh 38 len 16 wr_id 15e72f10. [D 20:00:57.348480] openib_check_cq: completed send wr_id 15e72f10. [D 20:00:57.348510] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:57.348536] ========================================================= [D 20:00:57.348561] PINT_process_request [D 20:00:57.348585] tiling 9 copies [D 20:00:57.348608] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.348635] lvl 0 el 0 blk 0 by 249561088 [D 20:00:57.348660] to 249561088 ta 234881024 fi 268435456 [D 20:00:57.348686] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.348711] basic type or contiguous data [D 20:00:57.348734] Request disp [D 20:00:57.348758] contig_offset = 249561088 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.348784] calling distribute [D 20:00:57.348808] PINT_distribute [D 20:00:57.348832] of 249561088 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 249561088 exfl 1 [D 20:00:57.348861] begin iteration loff: 249561088 [D 20:00:57.348886] segment exceeds byte limit [D 20:00:57.348910] file being extended [D 20:00:57.348934] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.348958] **********CALL***PROCESS********* [D 20:00:57.348982] segment of 14680064 sz 2097152 [D 20:00:57.349007] PINT_process_request [D 20:00:57.349031] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.349057] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.349081] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.349106] basic type or contiguous data [D 20:00:57.349130] Request disp [D 20:00:57.349154] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.349179] process a segment [D 20:00:57.349203] of 14680064 sz 2097152 [D 20:00:57.349226] add a segment 0 [D 20:00:57.349251] retval = 2097152 [D 20:00:57.349275] exiting distribute returned less than expected [D 20:00:57.349299] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.349325] *****RETURN***FROM***PROCESS***** [D 20:00:57.349349] end iteration [D 20:00:57.349373] done with segments or bytes [D 20:00:57.349397] of 251658240 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.349423] next loff: 251658240 next poff: 251658240 [D 20:00:57.349447] return value 2097152 [D 20:00:57.349472] finished [D 20:00:57.349495] retval = 2097152 [D 20:00:57.349519] exiting distribute returned less than expected [D 20:00:57.349543] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 251658240 fo 268435456 eof 0 [D 20:00:57.349569] ========================================================= [D 20:00:57.349594] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:57.349619] element 0: offset: 0xb5263000, size: 2097152 [D 20:00:57.349646] post_sr: da13:3346 bh 39 len 32 wr_id 15e72f28. [D 20:00:57.349681] openib_check_cq: completed send wr_id 15e72f28. [D 20:00:57.351155] openib_check_cq: completed recv wr_id 15e73668. [D 20:00:57.351183] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab843a010 rkey 48008200 nic_wr_credit 1. [D 20:00:57.351210] openib_post_rr: da13:3346 bh 16 wr_id 15e73668. [D 20:00:57.352822] PINT_client_state_machine_test id 22 [D 20:00:57.353675] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 0. [D 20:00:57.353703] post_sr: da13:3346 bh 40 len 16 wr_id 15e72f40. [D 20:00:57.353739] openib_check_cq: completed send wr_id 15e72f40. [D 20:00:57.353771] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:57.353796] ========================================================= [D 20:00:57.353821] PINT_process_request [D 20:00:57.353845] tiling 10 copies [D 20:00:57.353869] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.353896] lvl 0 el 0 blk 0 by 283115520 [D 20:00:57.353920] to 283115520 ta 268435456 fi 301989888 [D 20:00:57.353945] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.353970] basic type or contiguous data [D 20:00:57.353994] Request disp [D 20:00:57.354018] contig_offset = 283115520 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.354043] calling distribute [D 20:00:57.354067] PINT_distribute [D 20:00:57.354091] of 283115520 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 283115520 exfl 1 [D 20:00:57.354118] begin iteration loff: 283115520 [D 20:00:57.354143] segment exceeds byte limit [D 20:00:57.354170] file being extended [D 20:00:57.354198] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.354227] **********CALL***PROCESS********* [D 20:00:57.354253] segment of 14680064 sz 2097152 [D 20:00:57.354282] PINT_process_request [D 20:00:57.354306] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.354332] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.354357] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.354382] basic type or contiguous data [D 20:00:57.354406] Request disp [D 20:00:57.354433] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.354458] process a segment [D 20:00:57.354482] of 14680064 sz 2097152 [D 20:00:57.354506] add a segment 0 [D 20:00:57.354531] retval = 2097152 [D 20:00:57.354555] exiting distribute returned less than expected [D 20:00:57.354578] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.354605] *****RETURN***FROM***PROCESS***** [D 20:00:57.354630] end iteration [D 20:00:57.354654] done with segments or bytes [D 20:00:57.354678] of 285212672 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.354703] next loff: 285212672 next poff: 285212672 [D 20:00:57.354728] return value 2097152 [D 20:00:57.354752] finished [D 20:00:57.354776] retval = 2097152 [D 20:00:57.354800] exiting distribute returned less than expected [D 20:00:57.354825] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 285212672 fo 301989888 eof 0 [D 20:00:57.354850] ========================================================= [D 20:00:57.354875] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:57.354900] element 0: offset: 0xb3261000, size: 2097152 [D 20:00:57.354927] post_sr: da13:3346 bh 41 len 32 wr_id 15e72f58. [D 20:00:57.354958] openib_check_cq: completed recv wr_id 15e73680. [D 20:00:57.354985] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaab863b010 rkey 48008300 nic_wr_credit 1. [D 20:00:57.355011] openib_post_rr: da13:3346 bh 17 wr_id 15e73680. [D 20:00:57.355038] openib_check_cq: completed send wr_id 15e72f58. [D 20:00:57.357390] openib_check_cq: completed recv wr_id 15e73698. [D 20:00:57.357417] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaab883c010 rkey 48008400 nic_wr_credit 2. [D 20:00:57.357443] openib_post_rr: da13:3346 bh 18 wr_id 15e73698. [D 20:00:57.357480] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 1. [D 20:00:57.357507] post_sr: da13:3346 bh 42 len 16 wr_id 15e72f70. [D 20:00:57.359933] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 0. [D 20:00:57.359961] post_sr: da13:3346 bh 43 len 16 wr_id 15e72f88. [D 20:00:57.359988] openib_check_cq: completed send wr_id 15e72f70. [D 20:00:57.360013] openib_check_cq: completed send wr_id 15e72f88. [D 20:00:57.360044] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:57.360070] ========================================================= [D 20:00:57.360094] PINT_process_request [D 20:00:57.360118] tiling 11 copies [D 20:00:57.360142] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.360172] lvl 0 el 0 blk 0 by 316669952 [D 20:00:57.360197] to 316669952 ta 301989888 fi 335544320 [D 20:00:57.360221] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.360247] basic type or contiguous data [D 20:00:57.360271] Request disp [D 20:00:57.360295] contig_offset = 316669952 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.360320] calling distribute [D 20:00:57.360344] PINT_distribute [D 20:00:57.360367] of 316669952 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 316669952 exfl 1 [D 20:00:57.360394] begin iteration loff: 316669952 [D 20:00:57.360420] segment exceeds byte limit [D 20:00:57.360444] file being extended [D 20:00:57.360468] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.360493] **********CALL***PROCESS********* [D 20:00:57.360517] segment of 14680064 sz 2097152 [D 20:00:57.360542] PINT_process_request [D 20:00:57.360566] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.360592] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.360617] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.360642] basic type or contiguous data [D 20:00:57.360666] Request disp [D 20:00:57.360690] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.360714] process a segment [D 20:00:57.360738] of 14680064 sz 2097152 [D 20:00:57.360763] add a segment 0 [D 20:00:57.360787] retval = 2097152 [D 20:00:57.360814] exiting distribute returned less than expected [D 20:00:57.360837] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.360864] *****RETURN***FROM***PROCESS***** [D 20:00:57.360888] end iteration [D 20:00:57.360913] done with segments or bytes [D 20:00:57.360937] of 318767104 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.360962] next loff: 318767104 next poff: 318767104 [D 20:00:57.360987] return value 2097152 [D 20:00:57.361011] finished [D 20:00:57.361035] retval = 2097152 [D 20:00:57.361059] exiting distribute returned less than expected [D 20:00:57.361083] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 318767104 fo 335544320 eof 0 [D 20:00:57.361108] ========================================================= [D 20:00:57.361133] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:57.361159] element 0: offset: 0xb125f000, size: 2097152 [D 20:00:57.361186] post_sr: da13:3346 bh 44 len 32 wr_id 15e72fa0. [D 20:00:57.361214] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:57.361240] ========================================================= [D 20:00:57.361265] PINT_process_request [D 20:00:57.361289] tiling 12 copies [D 20:00:57.361312] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.361339] lvl 0 el 0 blk 0 by 350224384 [D 20:00:57.361364] to 350224384 ta 335544320 fi 369098752 [D 20:00:57.361389] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.361413] basic type or contiguous data [D 20:00:57.361437] Request disp [D 20:00:57.361461] contig_offset = 350224384 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.361486] calling distribute [D 20:00:57.361510] PINT_distribute [D 20:00:57.361534] of 350224384 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 350224384 exfl 1 [D 20:00:57.361560] begin iteration loff: 350224384 [D 20:00:57.361585] segment exceeds byte limit [D 20:00:57.361609] file being extended [D 20:00:57.361633] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.361658] **********CALL***PROCESS********* [D 20:00:57.361681] segment of 14680064 sz 2097152 [D 20:00:57.361708] PINT_process_request [D 20:00:57.361732] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.361758] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.361783] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.361807] basic type or contiguous data [D 20:00:57.361831] Request disp [D 20:00:57.361855] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.361880] process a segment [D 20:00:57.361904] of 14680064 sz 2097152 [D 20:00:57.361928] add a segment 0 [D 20:00:57.361952] retval = 2097152 [D 20:00:57.361976] exiting distribute returned less than expected [D 20:00:57.362000] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.362027] *****RETURN***FROM***PROCESS***** [D 20:00:57.362051] end iteration [D 20:00:57.362074] done with segments or bytes [D 20:00:57.362098] of 352321536 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.362124] next loff: 352321536 next poff: 352321536 [D 20:00:57.362149] return value 2097152 [D 20:00:57.362177] finished [D 20:00:57.362204] retval = 2097152 [D 20:00:57.362231] exiting distribute returned less than expected [D 20:00:57.362258] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 352321536 fo 369098752 eof 0 [D 20:00:57.362288] ========================================================= [D 20:00:57.362313] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:57.362338] element 0: offset: 0xaf25d000, size: 2097152 [D 20:00:57.362364] post_sr: da13:3346 bh 45 len 32 wr_id 15e72fb8. [D 20:00:57.362395] openib_check_cq: completed recv wr_id 15e736b0. [D 20:00:57.362421] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaab8a3d010 rkey 48008500 nic_wr_credit 1. [D 20:00:57.362449] openib_post_rr: da13:3346 bh 19 wr_id 15e736b0. [D 20:00:57.362478] openib_check_cq: completed send wr_id 15e72fa0. [D 20:00:57.362504] openib_check_cq: completed send wr_id 15e72fb8. [D 20:00:57.362850] PINT_client_state_machine_test id 22 [D 20:00:57.364913] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 0. [D 20:00:57.364940] post_sr: da13:3346 bh 46 len 16 wr_id 15e72fd0. [D 20:00:57.364977] openib_check_cq: completed send wr_id 15e72fd0. [D 20:00:57.365008] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:57.365034] ========================================================= [D 20:00:57.365058] PINT_process_request [D 20:00:57.365082] tiling 13 copies [D 20:00:57.365106] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.365133] lvl 0 el 0 blk 0 by 383778816 [D 20:00:57.365158] to 383778816 ta 369098752 fi 402653184 [D 20:00:57.365182] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.365207] basic type or contiguous data [D 20:00:57.365231] Request disp [D 20:00:57.365255] contig_offset = 383778816 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.365281] calling distribute [D 20:00:57.365304] PINT_distribute [D 20:00:57.365328] of 383778816 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 383778816 exfl 1 [D 20:00:57.365355] begin iteration loff: 383778816 [D 20:00:57.365380] segment exceeds byte limit [D 20:00:57.365404] file being extended [D 20:00:57.365427] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.365452] **********CALL***PROCESS********* [D 20:00:57.365476] segment of 14680064 sz 2097152 [D 20:00:57.365501] PINT_process_request [D 20:00:57.365525] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.365550] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.365575] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.365600] basic type or contiguous data [D 20:00:57.365624] Request disp [D 20:00:57.365648] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.365674] process a segment [D 20:00:57.365697] of 14680064 sz 2097152 [D 20:00:57.365721] add a segment 0 [D 20:00:57.365745] retval = 2097152 [D 20:00:57.365769] exiting distribute returned less than expected [D 20:00:57.365793] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.365819] *****RETURN***FROM***PROCESS***** [D 20:00:57.365843] end iteration [D 20:00:57.365867] done with segments or bytes [D 20:00:57.365891] of 385875968 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.365917] next loff: 385875968 next poff: 385875968 [D 20:00:57.365942] return value 2097152 [D 20:00:57.365965] finished [D 20:00:57.365989] retval = 2097152 [D 20:00:57.366013] exiting distribute returned less than expected [D 20:00:57.366037] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 385875968 fo 402653184 eof 0 [D 20:00:57.366064] ========================================================= [D 20:00:57.366087] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:57.366113] element 0: offset: 0xad25b000, size: 2097152 [D 20:00:57.366140] post_sr: da13:3346 bh 47 len 32 wr_id 15e72fe8. [D 20:00:57.366178] openib_check_cq: completed send wr_id 15e72fe8. [D 20:00:57.366968] openib_check_cq: completed recv wr_id 15e736c8. [D 20:00:57.366995] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab8c3e010 rkey 48008600 nic_wr_credit 1. [D 20:00:57.367022] openib_post_rr: da13:3346 bh 20 wr_id 15e736c8. [D 20:00:57.369485] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 0. [D 20:00:57.369512] post_sr: da13:3346 bh 48 len 16 wr_id 15e73000. [D 20:00:57.369549] openib_check_cq: completed send wr_id 15e73000. [D 20:00:57.369580] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:00:57.369606] ========================================================= [D 20:00:57.369630] PINT_process_request [D 20:00:57.369654] tiling 14 copies [D 20:00:57.369679] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.369708] lvl 0 el 0 blk 0 by 417333248 [D 20:00:57.369733] to 417333248 ta 402653184 fi 436207616 [D 20:00:57.369758] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.369783] basic type or contiguous data [D 20:00:57.369807] Request disp [D 20:00:57.369831] contig_offset = 417333248 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.369856] calling distribute [D 20:00:57.369880] PINT_distribute [D 20:00:57.369905] of 417333248 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 417333248 exfl 1 [D 20:00:57.369931] begin iteration loff: 417333248 [D 20:00:57.369956] segment exceeds byte limit [D 20:00:57.369980] file being extended [D 20:00:57.370004] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.370028] **********CALL***PROCESS********* [D 20:00:57.370052] segment of 14680064 sz 2097152 [D 20:00:57.370077] PINT_process_request [D 20:00:57.370100] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.370127] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.370152] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.370181] basic type or contiguous data [D 20:00:57.370208] Request disp [D 20:00:57.370236] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.370265] process a segment [D 20:00:57.370290] of 14680064 sz 2097152 [D 20:00:57.370315] add a segment 0 [D 20:00:57.370339] retval = 2097152 [D 20:00:57.370363] exiting distribute returned less than expected [D 20:00:57.370388] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.370413] *****RETURN***FROM***PROCESS***** [D 20:00:57.370438] end iteration [D 20:00:57.370462] done with segments or bytes [D 20:00:57.370486] of 419430400 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.370512] next loff: 419430400 next poff: 419430400 [D 20:00:57.370536] return value 2097152 [D 20:00:57.370560] finished [D 20:00:57.370584] retval = 2097152 [D 20:00:57.370608] exiting distribute returned less than expected [D 20:00:57.370632] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 419430400 fo 436207616 eof 0 [D 20:00:57.370657] ========================================================= [D 20:00:57.370682] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:00:57.370708] element 0: offset: 0xab259000, size: 2097152 [D 20:00:57.370735] post_sr: da13:3346 bh 49 len 32 wr_id 15e73018. [D 20:00:57.370765] openib_check_cq: completed recv wr_id 15e736e0. [D 20:00:57.370792] openib_post_sr_rdmaw: ibv_post_send wr_id 15f07a30 to da13:3346 remote addr 2aaab8e3f010 rkey 48008700 nic_wr_credit 1. [D 20:00:57.370819] openib_post_rr: da13:3346 bh 21 wr_id 15e736e0. [D 20:00:57.370844] openib_check_cq: completed send wr_id 15e73018. [D 20:00:57.372879] PINT_client_state_machine_test id 22 [D 20:00:57.372910] sm control op 300 [D 20:00:57.372936] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:00:57.372961] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:00:57.372991] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:00:57.373018] PINT_client_state_machine_test id 22 [D 20:00:57.373242] openib_check_cq: completed rdma wr_id 15f07a30, nic_wr_credit 0. [D 20:00:57.373270] post_sr: da13:3346 bh 50 len 16 wr_id 15e73030. [D 20:00:57.373306] openib_check_cq: completed send wr_id 15e73030. [D 20:00:57.373336] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:00:57.373362] ========================================================= [D 20:00:57.373387] PINT_process_request [D 20:00:57.373411] tiling 16 copies [D 20:00:57.373435] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.373462] lvl 0 el 0 blk 0 by 476053504 [D 20:00:57.373487] to 476053504 ta 469762048 fi 503316480 [D 20:00:57.373512] mto 6291456 mta 4194304 mfi 6291456 [D 20:00:57.373537] basic type or contiguous data [D 20:00:57.373560] Request disp [D 20:00:57.373593] contig_offset = 476053504 contig_size = 60817408 lvl_flag = 1 [D 20:00:57.373620] calling distribute [D 20:00:57.373644] PINT_distribute [D 20:00:57.373668] of 476053504 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 1 [D 20:00:57.373695] begin iteration loff: 476053504 [D 20:00:57.373720] segment exceeds byte limit [D 20:00:57.373744] file being extended [D 20:00:57.373767] client lstof 6291456 diff 0 sgof 6291456 [D 20:00:57.373792] **********CALL***PROCESS********* [D 20:00:57.373816] segment of 6291456 sz 2097152 [D 20:00:57.373841] PINT_process_request [D 20:00:57.373866] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.373891] lvl 0 el 0 blk 0 by 6291456 [D 20:00:57.373916] to 6291456 ta 6291456 fi 8388608 [D 20:00:57.373941] basic type or contiguous data [D 20:00:57.373965] Request disp [D 20:00:57.373989] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:00:57.374014] process a segment [D 20:00:57.374037] of 6291456 sz 2097152 [D 20:00:57.374062] add a segment 0 [D 20:00:57.374086] retval = 2097152 [D 20:00:57.374111] exiting distribute returned less than expected [D 20:00:57.374135] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:00:57.374161] *****RETURN***FROM***PROCESS***** [D 20:00:57.374191] end iteration [D 20:00:57.374218] done with segments or bytes [D 20:00:57.374246] of 478150656 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.374276] next loff: 478150656 next poff: 478150656 [D 20:00:57.374300] return value 2097152 [D 20:00:57.374325] finished [D 20:00:57.374349] retval = 2097152 [D 20:00:57.374373] exiting distribute returned less than expected [D 20:00:57.374397] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 478150656 fo 503316480 eof 0 [D 20:00:57.374423] ========================================================= [D 20:00:57.374448] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:00:57.374473] element 0: offset: 0xa6a55000, size: 2097152 [D 20:00:57.374500] post_sr: da13:3346 bh 51 len 32 wr_id 15e73048. [D 20:00:57.374530] openib_check_cq: completed recv wr_id 15e736f8. [D 20:00:57.374558] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab9040010 rkey c8008800 nic_wr_credit 1. [D 20:00:57.374585] openib_post_rr: da13:3346 bh 22 wr_id 15e736f8. [D 20:00:57.374611] openib_check_cq: completed send wr_id 15e73048. [D 20:00:57.377050] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 0. [D 20:00:57.377078] post_sr: da13:3346 bh 52 len 16 wr_id 15e73060. [D 20:00:57.377114] openib_check_cq: completed send wr_id 15e73060. [D 20:00:57.377144] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:00:57.377170] ========================================================= [D 20:00:57.377195] PINT_process_request [D 20:00:57.377219] tiling 15 copies [D 20:00:57.377243] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.377269] lvl 0 el 0 blk 0 by 450887680 [D 20:00:57.377295] to 450887680 ta 436207616 fi 469762048 [D 20:00:57.377320] mto 14680064 mta 12582912 mfi 14680064 [D 20:00:57.377345] basic type or contiguous data [D 20:00:57.377368] Request disp [D 20:00:57.377392] contig_offset = 450887680 contig_size = 52428800 lvl_flag = 1 [D 20:00:57.377418] calling distribute [D 20:00:57.377442] PINT_distribute [D 20:00:57.377466] of 450887680 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 450887680 exfl 1 [D 20:00:57.377492] begin iteration loff: 450887680 [D 20:00:57.377517] segment exceeds byte limit [D 20:00:57.377541] file being extended [D 20:00:57.377565] client lstof 14680064 diff 0 sgof 14680064 [D 20:00:57.377590] **********CALL***PROCESS********* [D 20:00:57.377614] segment of 14680064 sz 2097152 [D 20:00:57.377638] PINT_process_request [D 20:00:57.377662] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.377688] lvl 0 el 0 blk 0 by 14680064 [D 20:00:57.377717] to 14680064 ta 14680064 fi 16777216 [D 20:00:57.377741] basic type or contiguous data [D 20:00:57.377765] Request disp [D 20:00:57.377789] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:00:57.377815] process a segment [D 20:00:57.377839] of 14680064 sz 2097152 [D 20:00:57.377862] add a segment 0 [D 20:00:57.377886] retval = 2097152 [D 20:00:57.377911] exiting distribute returned less than expected [D 20:00:57.377935] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:00:57.377962] *****RETURN***FROM***PROCESS***** [D 20:00:57.377985] end iteration [D 20:00:57.378009] done with segments or bytes [D 20:00:57.378033] of 452984832 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.378059] next loff: 452984832 next poff: 452984832 [D 20:00:57.378084] return value 2097152 [D 20:00:57.378108] finished [D 20:00:57.378132] retval = 2097152 [D 20:00:57.378157] exiting distribute returned less than expected [D 20:00:57.378185] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 452984832 fo 469762048 eof 0 [D 20:00:57.378215] ========================================================= [D 20:00:57.378243] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:00:57.378273] element 0: offset: 0xa9257000, size: 2097152 [D 20:00:57.378300] post_sr: da13:3346 bh 53 len 32 wr_id 15e73078. [D 20:00:57.378336] openib_check_cq: completed send wr_id 15e73078. [D 20:00:57.378818] openib_check_cq: completed recv wr_id 15e73710. [D 20:00:57.378845] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab9241010 rkey c8008900 nic_wr_credit 1. [D 20:00:57.378871] openib_post_rr: da13:3346 bh 23 wr_id 15e73710. [D 20:00:57.381335] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 0. [D 20:00:57.381361] post_sr: da13:3346 bh 54 len 16 wr_id 15e73090. [D 20:00:57.381399] openib_check_cq: completed send wr_id 15e73090. [D 20:00:57.381429] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:57.381455] ========================================================= [D 20:00:57.381480] PINT_process_request [D 20:00:57.381504] tiling 2 copies [D 20:00:57.381527] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.381554] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.381579] to 16777216 ta 0 fi 33554432 [D 20:00:57.381604] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.381628] basic type or contiguous data [D 20:00:57.381652] Request disp [D 20:00:57.381676] contig_offset = 16777216 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.381704] calling distribute [D 20:00:57.381728] PINT_distribute [D 20:00:57.381752] of 16777216 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 16777216 exfl 1 [D 20:00:57.381778] begin iteration loff: 16777216 [D 20:00:57.381804] segment exceeds byte limit [D 20:00:57.381828] file being extended [D 20:00:57.381852] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.381876] **********CALL***PROCESS********* [D 20:00:57.381900] segment of 16777216 sz 2097152 [D 20:00:57.381925] PINT_process_request [D 20:00:57.381949] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.381975] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.381999] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.382024] basic type or contiguous data [D 20:00:57.382049] Request disp [D 20:00:57.382073] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.382098] process a segment [D 20:00:57.382122] of 16777216 sz 2097152 [D 20:00:57.382146] add a segment 0 [D 20:00:57.382173] retval = 2097152 [D 20:00:57.382202] exiting distribute returned less than expected [D 20:00:57.382228] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.382259] *****RETURN***FROM***PROCESS***** [D 20:00:57.382287] end iteration [D 20:00:57.382310] done with segments or bytes [D 20:00:57.382334] of 18874368 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.382363] next loff: 18874368 next poff: 18874368 [D 20:00:57.382389] return value 2097152 [D 20:00:57.382412] finished [D 20:00:57.382436] retval = 2097152 [D 20:00:57.382460] exiting distribute returned less than expected [D 20:00:57.382485] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 18874368 fo 33554432 eof 0 [D 20:00:57.382511] ========================================================= [D 20:00:57.382535] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:57.382561] element 0: offset: 0xc3471000, size: 2097152 [D 20:00:57.382588] post_sr: da13:3346 bh 55 len 32 wr_id 15e730a8. [D 20:00:57.382618] openib_check_cq: completed recv wr_id 15e73728. [D 20:00:57.382645] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab9442010 rkey c8008a00 nic_wr_credit 1. [D 20:00:57.382672] openib_post_rr: da13:3346 bh 24 wr_id 15e73728. [D 20:00:57.382699] openib_check_cq: completed send wr_id 15e730a8. [D 20:00:57.383049] PINT_client_state_machine_test id 22 [D 20:00:57.385140] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 0. [D 20:00:57.385168] post_sr: da13:3346 bh 56 len 16 wr_id 15e730c0. [D 20:00:57.385204] openib_check_cq: completed send wr_id 15e730c0. [D 20:00:57.385234] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:57.385260] ========================================================= [D 20:00:57.385285] PINT_process_request [D 20:00:57.385309] tiling 3 copies [D 20:00:57.385334] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.385359] lvl 0 el 0 blk 0 by 50331648 [D 20:00:57.385384] to 50331648 ta 33554432 fi 67108864 [D 20:00:57.385409] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.385435] basic type or contiguous data [D 20:00:57.385458] Request disp [D 20:00:57.385482] contig_offset = 50331648 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.385507] calling distribute [D 20:00:57.385531] PINT_distribute [D 20:00:57.385555] of 50331648 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 50331648 exfl 1 [D 20:00:57.385581] begin iteration loff: 50331648 [D 20:00:57.385606] segment exceeds byte limit [D 20:00:57.385630] file being extended [D 20:00:57.385654] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.385679] **********CALL***PROCESS********* [D 20:00:57.385703] segment of 16777216 sz 2097152 [D 20:00:57.385727] PINT_process_request [D 20:00:57.385751] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.385777] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.385802] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.385827] basic type or contiguous data [D 20:00:57.385850] Request disp [D 20:00:57.385874] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.385899] process a segment [D 20:00:57.385923] of 16777216 sz 2097152 [D 20:00:57.385948] add a segment 0 [D 20:00:57.385972] retval = 2097152 [D 20:00:57.385995] exiting distribute returned less than expected [D 20:00:57.386019] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.386046] *****RETURN***FROM***PROCESS***** [D 20:00:57.386070] end iteration [D 20:00:57.386094] done with segments or bytes [D 20:00:57.386117] of 52428800 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.386143] next loff: 52428800 next poff: 52428800 [D 20:00:57.386172] return value 2097152 [D 20:00:57.386199] finished [D 20:00:57.386227] retval = 2097152 [D 20:00:57.386254] exiting distribute returned less than expected [D 20:00:57.386281] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 52428800 fo 67108864 eof 0 [D 20:00:57.386308] ========================================================= [D 20:00:57.386332] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:57.386358] element 0: offset: 0xc146f000, size: 2097152 [D 20:00:57.386384] post_sr: da13:3346 bh 57 len 32 wr_id 15e730d8. [D 20:00:57.386415] openib_check_cq: completed recv wr_id 15e73740. [D 20:00:57.386445] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab9643010 rkey c8008b00 nic_wr_credit 1. [D 20:00:57.386471] openib_post_rr: da13:3346 bh 25 wr_id 15e73740. [D 20:00:57.386498] openib_check_cq: completed send wr_id 15e730d8. [D 20:00:57.388493] openib_check_cq: completed recv wr_id 15e73758. [D 20:00:57.388521] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab9844010 rkey c8008c00 nic_wr_credit 2. [D 20:00:57.388548] openib_post_rr: da13:3346 bh 26 wr_id 15e73758. [D 20:00:57.388939] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 1. [D 20:00:57.388966] post_sr: da13:3346 bh 58 len 16 wr_id 15e730f0. [D 20:00:57.391366] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 0. [D 20:00:57.391395] post_sr: da13:3346 bh 59 len 16 wr_id 15e73108. [D 20:00:57.391422] openib_check_cq: completed send wr_id 15e730f0. [D 20:00:57.391448] openib_check_cq: completed send wr_id 15e73108. [D 20:00:57.391477] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:57.391503] ========================================================= [D 20:00:57.391528] PINT_process_request [D 20:00:57.391551] tiling 4 copies [D 20:00:57.391576] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.391602] lvl 0 el 0 blk 0 by 83886080 [D 20:00:57.391627] to 83886080 ta 67108864 fi 100663296 [D 20:00:57.391652] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.391676] basic type or contiguous data [D 20:00:57.391702] Request disp [D 20:00:57.391727] contig_offset = 83886080 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.391753] calling distribute [D 20:00:57.391777] PINT_distribute [D 20:00:57.391800] of 83886080 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 83886080 exfl 1 [D 20:00:57.391827] begin iteration loff: 83886080 [D 20:00:57.391852] segment exceeds byte limit [D 20:00:57.391876] file being extended [D 20:00:57.391900] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.391924] **********CALL***PROCESS********* [D 20:00:57.391949] segment of 16777216 sz 2097152 [D 20:00:57.391973] PINT_process_request [D 20:00:57.391997] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.392024] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.392048] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.392073] basic type or contiguous data [D 20:00:57.392097] Request disp [D 20:00:57.392121] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.392147] process a segment [D 20:00:57.392173] of 16777216 sz 2097152 [D 20:00:57.392198] add a segment 0 [D 20:00:57.392222] retval = 2097152 [D 20:00:57.392246] exiting distribute returned less than expected [D 20:00:57.392270] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.392296] *****RETURN***FROM***PROCESS***** [D 20:00:57.392320] end iteration [D 20:00:57.392344] done with segments or bytes [D 20:00:57.392368] of 85983232 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.392394] next loff: 85983232 next poff: 85983232 [D 20:00:57.392418] return value 2097152 [D 20:00:57.392443] finished [D 20:00:57.392467] retval = 2097152 [D 20:00:57.392491] exiting distribute returned less than expected [D 20:00:57.392515] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 85983232 fo 100663296 eof 0 [D 20:00:57.392541] ========================================================= [D 20:00:57.392566] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:57.392591] element 0: offset: 0xbf46d000, size: 2097152 [D 20:00:57.392618] post_sr: da13:3346 bh 60 len 32 wr_id 15e73120. [D 20:00:57.392646] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:00:57.392672] ========================================================= [D 20:00:57.392697] PINT_process_request [D 20:00:57.392721] tiling 16 copies [D 20:00:57.392745] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.392774] lvl 0 el 0 blk 0 by 478150656 [D 20:00:57.392799] to 478150656 ta 469762048 fi 503316480 [D 20:00:57.392824] mto 8388608 mta 6291456 mfi 8388608 [D 20:00:57.392848] basic type or contiguous data [D 20:00:57.392872] Request disp [D 20:00:57.392896] contig_offset = 478150656 contig_size = 58720256 lvl_flag = 1 [D 20:00:57.392922] calling distribute [D 20:00:57.392946] PINT_distribute [D 20:00:57.392970] of 478150656 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 1 [D 20:00:57.392996] begin iteration loff: 478150656 [D 20:00:57.393021] segment exceeds byte limit [D 20:00:57.393045] file being extended [D 20:00:57.393069] client lstof 8388608 diff 0 sgof 8388608 [D 20:00:57.393094] **********CALL***PROCESS********* [D 20:00:57.393117] segment of 8388608 sz 2097152 [D 20:00:57.393141] PINT_process_request [D 20:00:57.393166] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.393192] lvl 0 el 0 blk 0 by 8388608 [D 20:00:57.393217] to 8388608 ta 8388608 fi 10485760 [D 20:00:57.393241] basic type or contiguous data [D 20:00:57.393265] Request disp [D 20:00:57.393289] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:00:57.393314] process a segment [D 20:00:57.393339] of 8388608 sz 2097152 [D 20:00:57.393362] add a segment 0 [D 20:00:57.393387] retval = 2097152 [D 20:00:57.393411] exiting distribute returned less than expected [D 20:00:57.393435] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:00:57.393462] *****RETURN***FROM***PROCESS***** [D 20:00:57.393486] end iteration [D 20:00:57.393509] done with segments or bytes [D 20:00:57.393533] of 480247808 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.393559] next loff: 480247808 next poff: 480247808 [D 20:00:57.393584] return value 2097152 [D 20:00:57.393608] finished [D 20:00:57.393631] retval = 2097152 [D 20:00:57.393656] exiting distribute returned less than expected [D 20:00:57.393680] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 480247808 fo 503316480 eof 0 [D 20:00:57.393706] ========================================================= [D 20:00:57.393731] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:00:57.393756] element 0: offset: 0xa6c55000, size: 2097152 [D 20:00:57.393783] post_sr: da13:3346 bh 61 len 32 wr_id 15e73138. [D 20:00:57.393813] PINT_client_state_machine_test id 22 [D 20:00:57.393841] openib_check_cq: completed send wr_id 15e73120. [D 20:00:57.393867] openib_check_cq: completed send wr_id 15e73138. [D 20:00:57.403841] PINT_client_state_machine_test id 22 [D 20:00:57.413871] PINT_client_state_machine_test id 22 [D 20:00:57.423900] PINT_client_state_machine_test id 22 [D 20:00:57.433933] PINT_client_state_machine_test id 22 [D 20:00:57.448172] PINT_client_state_machine_test id 22 [D 20:00:57.460170] PINT_client_state_machine_test id 22 [D 20:00:57.472170] PINT_client_state_machine_test id 22 [D 20:00:57.484174] PINT_client_state_machine_test id 22 [D 20:00:57.496170] PINT_client_state_machine_test id 22 [D 20:00:57.508171] PINT_client_state_machine_test id 22 [D 20:00:57.520172] PINT_client_state_machine_test id 22 [D 20:00:57.532171] PINT_client_state_machine_test id 22 [D 20:00:57.544171] PINT_client_state_machine_test id 22 [D 20:00:57.556171] PINT_client_state_machine_test id 22 [D 20:00:57.568171] PINT_client_state_machine_test id 22 [D 20:00:57.580171] PINT_client_state_machine_test id 22 [D 20:00:57.592188] PINT_client_state_machine_test id 22 -[D 20:00:57.599599] openib_check_cq: completed recv wr_id 15e73770. [D 20:00:57.599634] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaab9a45010 rkey c8008d00 nic_wr_credit 1. [D 20:00:57.599665] openib_post_rr: da13:3346 bh 27 wr_id 15e73770. [D 20:00:57.602139] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 0. [D 20:00:57.602167] post_sr: da13:3346 bh 62 len 16 wr_id 15e73150. [D 20:00:57.602207] openib_check_cq: completed send wr_id 15e73150. [D 20:00:57.602238] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:57.602264] ========================================================= [D 20:00:57.602288] PINT_process_request [D 20:00:57.602312] tiling 5 copies [D 20:00:57.602336] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.602363] lvl 0 el 0 blk 0 by 117440512 [D 20:00:57.602388] to 117440512 ta 100663296 fi 134217728 [D 20:00:57.602413] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.602437] basic type or contiguous data [D 20:00:57.602462] Request disp [D 20:00:57.602486] contig_offset = 117440512 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.602511] calling distribute [D 20:00:57.602535] PINT_distribute [D 20:00:57.602560] of 117440512 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 117440512 exfl 1 [D 20:00:57.602586] begin iteration loff: 117440512 [D 20:00:57.602611] segment exceeds byte limit [D 20:00:57.602636] file being extended [D 20:00:57.602660] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.602689] **********CALL***PROCESS********* [D 20:00:57.602717] segment of 16777216 sz 2097152 [D 20:00:57.602745] PINT_process_request [D 20:00:57.602772] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.602799] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.602825] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.602849] basic type or contiguous data [D 20:00:57.602873] Request disp [D 20:00:57.602897] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.602922] process a segment [D 20:00:57.602946] of 16777216 sz 2097152 [D 20:00:57.602971] add a segment 0 [D 20:00:57.602994] retval = 2097152 [D 20:00:57.603018] exiting distribute returned less than expected [D 20:00:57.603043] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.603069] *****RETURN***FROM***PROCESS***** [D 20:00:57.603094] end iteration [D 20:00:57.603117] done with segments or bytes [D 20:00:57.603141] of 119537664 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.603167] next loff: 119537664 next poff: 119537664 [D 20:00:57.603192] return value 2097152 [D 20:00:57.603216] finished [D 20:00:57.603240] retval = 2097152 [D 20:00:57.603263] exiting distribute returned less than expected [D 20:00:57.603287] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 119537664 fo 134217728 eof 0 [D 20:00:57.603314] ========================================================= [D 20:00:57.603339] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:57.603363] element 0: offset: 0xbd46b000, size: 2097152 [D 20:00:57.603391] post_sr: da13:3346 bh 63 len 32 wr_id 15e73168. [D 20:00:57.603421] PINT_client_state_machine_test id 22 [D 20:00:57.603449] openib_check_cq: completed send wr_id 15e73168. [D 20:00:57.604277] openib_check_cq: completed recv wr_id 15e73788. [D 20:00:57.604304] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab9c46010 rkey c8008e00 nic_wr_credit 1. [D 20:00:57.604331] openib_post_rr: da13:3346 bh 28 wr_id 15e73788. [D 20:00:57.606800] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 0. [D 20:00:57.606828] post_sr: da13:3346 bh 64 len 16 wr_id 15e73180. [D 20:00:57.606864] openib_check_cq: completed send wr_id 15e73180. [D 20:00:57.606895] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:57.606921] ========================================================= [D 20:00:57.606945] PINT_process_request [D 20:00:57.606969] tiling 6 copies [D 20:00:57.606994] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.607021] lvl 0 el 0 blk 0 by 150994944 [D 20:00:57.607046] to 150994944 ta 134217728 fi 167772160 [D 20:00:57.607070] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.607095] basic type or contiguous data [D 20:00:57.607119] Request disp [D 20:00:57.607144] contig_offset = 150994944 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.607172] calling distribute [D 20:00:57.607196] PINT_distribute [D 20:00:57.607220] of 150994944 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 150994944 exfl 1 [D 20:00:57.607247] begin iteration loff: 150994944 [D 20:00:57.607272] segment exceeds byte limit [D 20:00:57.607295] file being extended [D 20:00:57.607319] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.607344] **********CALL***PROCESS********* [D 20:00:57.607368] segment of 16777216 sz 2097152 [D 20:00:57.607393] PINT_process_request [D 20:00:57.607417] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.607443] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.607468] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.607493] basic type or contiguous data [D 20:00:57.607517] Request disp [D 20:00:57.607541] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.607565] process a segment [D 20:00:57.607589] of 16777216 sz 2097152 [D 20:00:57.607614] add a segment 0 [D 20:00:57.607638] retval = 2097152 [D 20:00:57.607663] exiting distribute returned less than expected [D 20:00:57.607686] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.607712] *****RETURN***FROM***PROCESS***** [D 20:00:57.607737] end iteration [D 20:00:57.607761] done with segments or bytes [D 20:00:57.607785] of 153092096 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.607810] next loff: 153092096 next poff: 153092096 [D 20:00:57.607835] return value 2097152 [D 20:00:57.607859] finished [D 20:00:57.607884] retval = 2097152 [D 20:00:57.607908] exiting distribute returned less than expected [D 20:00:57.607932] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 153092096 fo 167772160 eof 0 [D 20:00:57.607958] ========================================================= [D 20:00:57.607983] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:57.608008] element 0: offset: 0xbb469000, size: 2097152 [D 20:00:57.608036] post_sr: da13:3346 bh 65 len 32 wr_id 15e73198. [D 20:00:57.608071] openib_check_cq: completed send wr_id 15e73198. [D 20:00:57.609317] openib_check_cq: completed recv wr_id 15e737a0. [D 20:00:57.609344] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab9e47010 rkey c8008f00 nic_wr_credit 1. [D 20:00:57.609372] openib_post_rr: da13:3346 bh 29 wr_id 15e737a0. [D 20:00:57.611233] openib_check_cq: completed recv wr_id 15e737b8. [D 20:00:57.611261] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaaba048010 rkey 48009000 nic_wr_credit 2. [D 20:00:57.611289] openib_post_rr: da13:3346 bh 30 wr_id 15e737b8. [D 20:00:57.611842] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 1. [D 20:00:57.611869] post_sr: da13:3346 bh 66 len 16 wr_id 15e731b0. [D 20:00:57.613448] PINT_client_state_machine_test id 22 [D 20:00:57.614268] openib_check_cq: completed recv wr_id 15e737d0. [D 20:00:57.614295] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaaba249010 rkey 48009100 nic_wr_credit 2. [D 20:00:57.614322] openib_post_rr: da13:3346 bh 31 wr_id 15e737d0. [D 20:00:57.614348] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 1. [D 20:00:57.614375] post_sr: da13:3346 bh 67 len 16 wr_id 15e731c8. [D 20:00:57.614401] openib_check_cq: completed send wr_id 15e731b0. [D 20:00:57.614431] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:57.614457] ========================================================= [D 20:00:57.614481] PINT_process_request [D 20:00:57.614505] tiling 7 copies [D 20:00:57.614529] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.614556] lvl 0 el 0 blk 0 by 184549376 [D 20:00:57.614581] to 184549376 ta 167772160 fi 201326592 [D 20:00:57.614605] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.614630] basic type or contiguous data [D 20:00:57.614654] Request disp [D 20:00:57.614685] contig_offset = 184549376 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.614714] calling distribute [D 20:00:57.614741] PINT_distribute [D 20:00:57.614769] of 184549376 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 184549376 exfl 1 [D 20:00:57.614796] begin iteration loff: 184549376 [D 20:00:57.614821] segment exceeds byte limit [D 20:00:57.614846] file being extended [D 20:00:57.614870] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.614894] **********CALL***PROCESS********* [D 20:00:57.614918] segment of 16777216 sz 2097152 [D 20:00:57.614942] PINT_process_request [D 20:00:57.614967] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.614993] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.615017] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.615042] basic type or contiguous data [D 20:00:57.615066] Request disp [D 20:00:57.615090] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.615115] process a segment [D 20:00:57.615139] of 16777216 sz 2097152 [D 20:00:57.615163] add a segment 0 [D 20:00:57.615187] retval = 2097152 [D 20:00:57.615211] exiting distribute returned less than expected [D 20:00:57.615235] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.615262] *****RETURN***FROM***PROCESS***** [D 20:00:57.615286] end iteration [D 20:00:57.615310] done with segments or bytes [D 20:00:57.615333] of 186646528 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.615360] next loff: 186646528 next poff: 186646528 [D 20:00:57.615385] return value 2097152 [D 20:00:57.615408] finished [D 20:00:57.615432] retval = 2097152 [D 20:00:57.615456] exiting distribute returned less than expected [D 20:00:57.615481] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 186646528 fo 201326592 eof 0 [D 20:00:57.615507] ========================================================= [D 20:00:57.615531] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:57.615557] element 0: offset: 0xb9467000, size: 2097152 [D 20:00:57.615584] post_sr: da13:3346 bh 68 len 32 wr_id 15e731e0. [D 20:00:57.616789] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 0. [D 20:00:57.616817] post_sr: da13:3346 bh 69 len 16 wr_id 15e731f8. [D 20:00:57.616844] openib_check_cq: completed send wr_id 15e731c8. [D 20:00:57.616868] openib_check_cq: completed send wr_id 15e731e0. [D 20:00:57.616894] openib_check_cq: completed send wr_id 15e731f8. [D 20:00:57.616924] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:57.616950] ========================================================= [D 20:00:57.616974] PINT_process_request [D 20:00:57.616998] tiling 8 copies [D 20:00:57.617023] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.617050] lvl 0 el 0 blk 0 by 218103808 [D 20:00:57.617074] to 218103808 ta 201326592 fi 234881024 [D 20:00:57.617099] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.617124] basic type or contiguous data [D 20:00:57.617148] Request disp [D 20:00:57.617172] contig_offset = 218103808 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.617197] calling distribute [D 20:00:57.617221] PINT_distribute [D 20:00:57.617245] of 218103808 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 218103808 exfl 1 [D 20:00:57.617271] begin iteration loff: 218103808 [D 20:00:57.617297] segment exceeds byte limit [D 20:00:57.617321] file being extended [D 20:00:57.617344] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.617369] **********CALL***PROCESS********* [D 20:00:57.617393] segment of 16777216 sz 2097152 [D 20:00:57.617418] PINT_process_request [D 20:00:57.617442] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.617467] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.617492] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.617517] basic type or contiguous data [D 20:00:57.617541] Request disp [D 20:00:57.617565] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.617592] process a segment [D 20:00:57.617617] of 16777216 sz 2097152 [D 20:00:57.617641] add a segment 0 [D 20:00:57.617665] retval = 2097152 [D 20:00:57.617690] exiting distribute returned less than expected [D 20:00:57.617713] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.617740] *****RETURN***FROM***PROCESS***** [D 20:00:57.617764] end iteration [D 20:00:57.617788] done with segments or bytes [D 20:00:57.617812] of 220200960 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.617837] next loff: 220200960 next poff: 220200960 [D 20:00:57.617862] return value 2097152 [D 20:00:57.617887] finished [D 20:00:57.617911] retval = 2097152 [D 20:00:57.617935] exiting distribute returned less than expected [D 20:00:57.617959] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 220200960 fo 234881024 eof 0 [D 20:00:57.617985] ========================================================= [D 20:00:57.618009] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:57.618034] element 0: offset: 0xb7465000, size: 2097152 [D 20:00:57.618062] post_sr: da13:3346 bh 70 len 32 wr_id 15e73210. [D 20:00:57.618090] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:57.618115] ========================================================= [D 20:00:57.618140] PINT_process_request [D 20:00:57.618165] tiling 9 copies [D 20:00:57.618188] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.618215] lvl 0 el 0 blk 0 by 251658240 [D 20:00:57.618240] to 251658240 ta 234881024 fi 268435456 [D 20:00:57.618265] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.618289] basic type or contiguous data [D 20:00:57.618313] Request disp [D 20:00:57.618337] contig_offset = 251658240 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.618363] calling distribute [D 20:00:57.618387] PINT_distribute [D 20:00:57.618411] of 251658240 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 251658240 exfl 1 [D 20:00:57.618436] begin iteration loff: 251658240 [D 20:00:57.618462] segment exceeds byte limit [D 20:00:57.618486] file being extended [D 20:00:57.618510] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.618535] **********CALL***PROCESS********* [D 20:00:57.618558] segment of 16777216 sz 2097152 [D 20:00:57.618583] PINT_process_request [D 20:00:57.618607] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.618633] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.618658] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.618687] basic type or contiguous data [D 20:00:57.618715] Request disp [D 20:00:57.618741] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.618771] process a segment [D 20:00:57.618796] of 16777216 sz 2097152 [D 20:00:57.618820] add a segment 0 [D 20:00:57.618844] retval = 2097152 [D 20:00:57.618868] exiting distribute returned less than expected [D 20:00:57.618892] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.618919] *****RETURN***FROM***PROCESS***** [D 20:00:57.618944] end iteration [D 20:00:57.618967] done with segments or bytes [D 20:00:57.618990] of 253755392 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.619017] next loff: 253755392 next poff: 253755392 [D 20:00:57.619042] return value 2097152 [D 20:00:57.619066] finished [D 20:00:57.619090] retval = 2097152 [D 20:00:57.619113] exiting distribute returned less than expected [D 20:00:57.619138] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 253755392 fo 268435456 eof 0 [D 20:00:57.619164] ========================================================= [D 20:00:57.619189] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:57.619214] element 0: offset: 0xb5463000, size: 2097152 [D 20:00:57.619241] post_sr: da13:3346 bh 71 len 32 wr_id 15e73228. [D 20:00:57.619272] openib_check_cq: completed recv wr_id 15e737e8. [D 20:00:57.619300] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaaba44a010 rkey 48009200 nic_wr_credit 1. [D 20:00:57.619328] openib_post_rr: da13:3346 bh 32 wr_id 15e737e8. [D 20:00:57.619354] openib_check_cq: completed send wr_id 15e73210. [D 20:00:57.619379] openib_check_cq: completed send wr_id 15e73228. [D 20:00:57.621148] openib_check_cq: completed recv wr_id 15e73800. [D 20:00:57.621174] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaaba64b010 rkey 48009300 nic_wr_credit 2. [D 20:00:57.621202] openib_post_rr: da13:3346 bh 33 wr_id 15e73800. [D 20:00:57.621799] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 1. [D 20:00:57.621826] post_sr: da13:3346 bh 72 len 16 wr_id 15e73240. [D 20:00:57.623476] PINT_client_state_machine_test id 22 [D 20:00:57.624257] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 0. [D 20:00:57.624284] post_sr: da13:3346 bh 73 len 16 wr_id 15e73258. [D 20:00:57.624310] openib_check_cq: completed send wr_id 15e73240. [D 20:00:57.624336] openib_check_cq: completed send wr_id 15e73258. [D 20:00:57.624366] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:57.624391] ========================================================= [D 20:00:57.624416] PINT_process_request [D 20:00:57.624440] tiling 10 copies [D 20:00:57.624464] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.624490] lvl 0 el 0 blk 0 by 285212672 [D 20:00:57.624516] to 285212672 ta 268435456 fi 301989888 [D 20:00:57.624541] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.624565] basic type or contiguous data [D 20:00:57.624589] Request disp [D 20:00:57.624613] contig_offset = 285212672 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.624638] calling distribute [D 20:00:57.624662] PINT_distribute [D 20:00:57.624689] of 285212672 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 285212672 exfl 1 [D 20:00:57.624714] begin iteration loff: 285212672 [D 20:00:57.624740] segment exceeds byte limit [D 20:00:57.624764] file being extended [D 20:00:57.624788] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.624813] **********CALL***PROCESS********* [D 20:00:57.624836] segment of 16777216 sz 2097152 [D 20:00:57.624861] PINT_process_request [D 20:00:57.624886] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.624912] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.624937] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.624961] basic type or contiguous data [D 20:00:57.624986] Request disp [D 20:00:57.625009] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.625035] process a segment [D 20:00:57.625059] of 16777216 sz 2097152 [D 20:00:57.625082] add a segment 0 [D 20:00:57.625106] retval = 2097152 [D 20:00:57.625131] exiting distribute returned less than expected [D 20:00:57.625155] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.625181] *****RETURN***FROM***PROCESS***** [D 20:00:57.625205] end iteration [D 20:00:57.625229] done with segments or bytes [D 20:00:57.625253] of 287309824 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.625279] next loff: 287309824 next poff: 287309824 [D 20:00:57.625303] return value 2097152 [D 20:00:57.625328] finished [D 20:00:57.625351] retval = 2097152 [D 20:00:57.625375] exiting distribute returned less than expected [D 20:00:57.625399] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 287309824 fo 301989888 eof 0 [D 20:00:57.625426] ========================================================= [D 20:00:57.625450] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:57.625475] element 0: offset: 0xb3461000, size: 2097152 [D 20:00:57.625502] post_sr: da13:3346 bh 74 len 32 wr_id 15e73270. [D 20:00:57.625531] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:57.625556] ========================================================= [D 20:00:57.625583] PINT_process_request [D 20:00:57.625608] tiling 11 copies [D 20:00:57.625632] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.625658] lvl 0 el 0 blk 0 by 318767104 [D 20:00:57.625683] to 318767104 ta 301989888 fi 335544320 [D 20:00:57.625708] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.625733] basic type or contiguous data [D 20:00:57.625757] Request disp [D 20:00:57.625781] contig_offset = 318767104 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.625805] calling distribute [D 20:00:57.625829] PINT_distribute [D 20:00:57.625853] of 318767104 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 318767104 exfl 1 [D 20:00:57.625880] begin iteration loff: 318767104 [D 20:00:57.625905] segment exceeds byte limit [D 20:00:57.625928] file being extended [D 20:00:57.625953] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.625977] **********CALL***PROCESS********* [D 20:00:57.626001] segment of 16777216 sz 2097152 [D 20:00:57.626026] PINT_process_request [D 20:00:57.626049] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.626076] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.626101] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.626126] basic type or contiguous data [D 20:00:57.626150] Request disp [D 20:00:57.626173] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.626199] process a segment [D 20:00:57.626223] of 16777216 sz 2097152 [D 20:00:57.626247] add a segment 0 [D 20:00:57.626271] retval = 2097152 [D 20:00:57.626295] exiting distribute returned less than expected [D 20:00:57.626318] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.626345] *****RETURN***FROM***PROCESS***** [D 20:00:57.626369] end iteration [D 20:00:57.626393] done with segments or bytes [D 20:00:57.626417] of 320864256 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.626442] next loff: 320864256 next poff: 320864256 [D 20:00:57.626467] return value 2097152 [D 20:00:57.626492] finished [D 20:00:57.626515] retval = 2097152 [D 20:00:57.626539] exiting distribute returned less than expected [D 20:00:57.626564] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 320864256 fo 335544320 eof 0 [D 20:00:57.626589] ========================================================= [D 20:00:57.626614] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:57.626639] element 0: offset: 0xb145f000, size: 2097152 [D 20:00:57.626667] post_sr: da13:3346 bh 75 len 32 wr_id 15e73288. [D 20:00:57.626702] openib_check_cq: completed recv wr_id 15e73818. [D 20:00:57.626732] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaaba84c010 rkey 48009400 nic_wr_credit 1. [D 20:00:57.626763] openib_post_rr: da13:3346 bh 34 wr_id 15e73818. [D 20:00:57.626791] openib_check_cq: completed send wr_id 15e73270. [D 20:00:57.626816] openib_check_cq: completed send wr_id 15e73288. [D 20:00:57.628547] openib_check_cq: completed recv wr_id 15e73830. [D 20:00:57.628574] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaabaa4d010 rkey 48009500 nic_wr_credit 2. [D 20:00:57.628600] openib_post_rr: da13:3346 bh 35 wr_id 15e73830. [D 20:00:57.629231] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 1. [D 20:00:57.629258] post_sr: da13:3346 bh 76 len 16 wr_id 15e732a0. [D 20:00:57.631690] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 0. [D 20:00:57.631719] post_sr: da13:3346 bh 77 len 16 wr_id 15e732b8. [D 20:00:57.631745] openib_check_cq: completed send wr_id 15e732a0. [D 20:00:57.631771] openib_check_cq: completed send wr_id 15e732b8. [D 20:00:57.631803] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:57.631829] ========================================================= [D 20:00:57.631853] PINT_process_request [D 20:00:57.631876] tiling 12 copies [D 20:00:57.631901] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.631927] lvl 0 el 0 blk 0 by 352321536 [D 20:00:57.631955] to 352321536 ta 335544320 fi 369098752 [D 20:00:57.631980] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.632005] basic type or contiguous data [D 20:00:57.632029] Request disp [D 20:00:57.632053] contig_offset = 352321536 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.632078] calling distribute [D 20:00:57.632102] PINT_distribute [D 20:00:57.632125] of 352321536 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 352321536 exfl 1 [D 20:00:57.632152] begin iteration loff: 352321536 [D 20:00:57.632178] segment exceeds byte limit [D 20:00:57.632202] file being extended [D 20:00:57.632225] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.632250] **********CALL***PROCESS********* [D 20:00:57.632274] segment of 16777216 sz 2097152 [D 20:00:57.632299] PINT_process_request [D 20:00:57.632323] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.632349] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.632373] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.632398] basic type or contiguous data [D 20:00:57.632422] Request disp [D 20:00:57.632446] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.632471] process a segment [D 20:00:57.632494] of 16777216 sz 2097152 [D 20:00:57.632518] add a segment 0 [D 20:00:57.632542] retval = 2097152 [D 20:00:57.632567] exiting distribute returned less than expected [D 20:00:57.632591] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.632616] *****RETURN***FROM***PROCESS***** [D 20:00:57.632641] end iteration [D 20:00:57.632667] done with segments or bytes [D 20:00:57.632692] of 354418688 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.632718] next loff: 354418688 next poff: 354418688 [D 20:00:57.632742] return value 2097152 [D 20:00:57.632767] finished [D 20:00:57.632791] retval = 2097152 [D 20:00:57.632815] exiting distribute returned less than expected [D 20:00:57.632839] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 354418688 fo 369098752 eof 0 [D 20:00:57.632864] ========================================================= [D 20:00:57.632889] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:57.632914] element 0: offset: 0xaf45d000, size: 2097152 [D 20:00:57.632942] post_sr: da13:3346 bh 78 len 32 wr_id 15e732d0. [D 20:00:57.632969] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:57.632995] ========================================================= [D 20:00:57.633020] PINT_process_request [D 20:00:57.633044] tiling 13 copies [D 20:00:57.633068] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.633094] lvl 0 el 0 blk 0 by 385875968 [D 20:00:57.633118] to 385875968 ta 369098752 fi 402653184 [D 20:00:57.633143] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.633168] basic type or contiguous data [D 20:00:57.633192] Request disp [D 20:00:57.633215] contig_offset = 385875968 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.633241] calling distribute [D 20:00:57.633265] PINT_distribute [D 20:00:57.633289] of 385875968 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 385875968 exfl 1 [D 20:00:57.633315] begin iteration loff: 385875968 [D 20:00:57.633339] segment exceeds byte limit [D 20:00:57.633364] file being extended [D 20:00:57.633388] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.633412] **********CALL***PROCESS********* [D 20:00:57.633436] segment of 16777216 sz 2097152 [D 20:00:57.633460] PINT_process_request [D 20:00:57.633484] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.633510] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.633535] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.633560] basic type or contiguous data [D 20:00:57.633584] Request disp [D 20:00:57.633607] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.633632] process a segment [D 20:00:57.633656] of 16777216 sz 2097152 [D 20:00:57.633683] add a segment 0 [D 20:00:57.633708] retval = 2097152 [D 20:00:57.633731] exiting distribute returned less than expected [D 20:00:57.633755] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.633781] *****RETURN***FROM***PROCESS***** [D 20:00:57.633806] end iteration [D 20:00:57.633830] done with segments or bytes [D 20:00:57.633853] of 387973120 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.633878] next loff: 387973120 next poff: 387973120 [D 20:00:57.633903] return value 2097152 [D 20:00:57.633928] finished [D 20:00:57.633952] retval = 2097152 [D 20:00:57.633975] exiting distribute returned less than expected [D 20:00:57.633999] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 387973120 fo 402653184 eof 0 [D 20:00:57.634025] ========================================================= [D 20:00:57.634050] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:57.634075] element 0: offset: 0xad45b000, size: 2097152 [D 20:00:57.634101] post_sr: da13:3346 bh 79 len 32 wr_id 15e732e8. [D 20:00:57.634131] PINT_client_state_machine_test id 22 [D 20:00:57.634160] openib_check_cq: completed send wr_id 15e732d0. [D 20:00:57.634186] openib_check_cq: completed recv wr_id 15e73848. [D 20:00:57.634212] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaabac4f010 rkey 48009600 nic_wr_credit 1. [D 20:00:57.634239] openib_post_rr: da13:3346 bh 36 wr_id 15e73848. [D 20:00:57.634265] openib_check_cq: completed send wr_id 15e732e8. [D 20:00:57.635815] openib_check_cq: completed recv wr_id 15e73860. [D 20:00:57.635843] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaabae50010 rkey 48009700 nic_wr_credit 2. [D 20:00:57.635869] openib_post_rr: da13:3346 bh 37 wr_id 15e73860. [D 20:00:57.636161] openib_check_cq: completed recv wr_id 15e73878. [D 20:00:57.636188] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaaac6bb010 rkey 48002400 nic_wr_credit 3. [D 20:00:57.636215] openib_post_rr: da13:3346 bh 38 wr_id 15e73878. [D 20:00:57.636459] openib_check_cq: completed recv wr_id 15e73890. [D 20:00:57.636486] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaaac8bc010 rkey 48002500 nic_wr_credit 4. [D 20:00:57.636512] openib_post_rr: da13:3346 bh 39 wr_id 15e73890. [D 20:00:57.636706] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 3. [D 20:00:57.636734] post_sr: da13:3346 bh 80 len 16 wr_id 15e73300. [D 20:00:57.636767] openib_check_cq: completed recv wr_id 15e738a8. [D 20:00:57.636794] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaaacabd010 rkey 48002600 nic_wr_credit 4. [D 20:00:57.636820] openib_post_rr: da13:3346 bh 40 wr_id 15e738a8. [D 20:00:57.637093] openib_check_cq: completed recv wr_id 15e738c0. [D 20:00:57.637120] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaaaccbe010 rkey 48002700 nic_wr_credit 5. [D 20:00:57.637146] openib_post_rr: da13:3346 bh 41 wr_id 15e738c0. [D 20:00:57.637419] openib_check_cq: completed recv wr_id 15e738d8. [D 20:00:57.637446] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaaacebf010 rkey c8002800 nic_wr_credit 6. [D 20:00:57.637473] openib_post_rr: da13:3346 bh 42 wr_id 15e738d8. [D 20:00:57.639166] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 5. [D 20:00:57.639194] post_sr: da13:3346 bh 81 len 16 wr_id 15e73318. [D 20:00:57.639517] openib_check_cq: completed recv wr_id 15e738f0. [D 20:00:57.639544] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaaad0c0010 rkey c8002900 nic_wr_credit 6. [D 20:00:57.639572] openib_post_rr: da13:3346 bh 43 wr_id 15e738f0. [D 20:00:57.639808] openib_check_cq: completed recv wr_id 15e73908. [D 20:00:57.639835] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaaad2c1010 rkey c8002a00 nic_wr_credit 7. [D 20:00:57.639862] openib_post_rr: da13:3346 bh 44 wr_id 15e73908. [D 20:00:57.640284] openib_check_cq: completed recv wr_id 15e73920. [D 20:00:57.640312] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaaad4c2010 rkey c8002b00 nic_wr_credit 8. [D 20:00:57.640338] openib_post_rr: da13:3346 bh 45 wr_id 15e73920. [D 20:00:57.640583] openib_check_cq: completed recv wr_id 15e73938. [D 20:00:57.640610] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaaad6c3010 rkey c8002c00 nic_wr_credit 9. [D 20:00:57.640637] openib_post_rr: da13:3346 bh 46 wr_id 15e73938. [D 20:00:57.640862] openib_check_cq: completed recv wr_id 15e73950. [D 20:00:57.640890] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaaad8c4010 rkey c8002d00 nic_wr_credit 10. [D 20:00:57.640916] openib_post_rr: da13:3346 bh 47 wr_id 15e73950. [D 20:00:57.641148] openib_check_cq: completed recv wr_id 15e73968. [D 20:00:57.641176] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaaadac5010 rkey c8002e00 nic_wr_credit 11. [D 20:00:57.641202] openib_post_rr: da13:3346 bh 48 wr_id 15e73968. [D 20:00:57.641629] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 10. [D 20:00:57.641655] post_sr: da13:3346 bh 82 len 16 wr_id 15e73330. [D 20:00:57.644093] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 9. [D 20:00:57.644121] post_sr: da13:3346 bh 83 len 16 wr_id 15e73348. [D 20:00:57.644147] openib_check_cq: completed send wr_id 15e73300. [D 20:00:57.644177] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:00:57.644203] ========================================================= [D 20:00:57.644227] PINT_process_request [D 20:00:57.644251] tiling 14 copies [D 20:00:57.644276] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.644302] lvl 0 el 0 blk 0 by 419430400 [D 20:00:57.644328] to 419430400 ta 402653184 fi 436207616 [D 20:00:57.644352] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.644377] basic type or contiguous data [D 20:00:57.644401] Request disp [D 20:00:57.644424] contig_offset = 419430400 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.644450] calling distribute [D 20:00:57.644474] PINT_distribute [D 20:00:57.644497] of 419430400 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 419430400 exfl 1 [D 20:00:57.644524] begin iteration loff: 419430400 [D 20:00:57.644549] segment exceeds byte limit [D 20:00:57.644573] file being extended [D 20:00:57.644597] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.644621] **********CALL***PROCESS********* [D 20:00:57.644645] segment of 16777216 sz 2097152 [D 20:00:57.644672] PINT_process_request [D 20:00:57.644697] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.644722] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.644747] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.644772] basic type or contiguous data [D 20:00:57.644796] Request disp [D 20:00:57.644820] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.644845] process a segment [D 20:00:57.644868] of 16777216 sz 2097152 [D 20:00:57.644893] add a segment 0 [D 20:00:57.644917] retval = 2097152 [D 20:00:57.644941] exiting distribute returned less than expected [D 20:00:57.644965] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.644991] *****RETURN***FROM***PROCESS***** [D 20:00:57.645016] end iteration [D 20:00:57.645040] done with segments or bytes [D 20:00:57.645063] of 421527552 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.645089] next loff: 421527552 next poff: 421527552 [D 20:00:57.645113] return value 2097152 [D 20:00:57.645138] finished [D 20:00:57.645162] retval = 2097152 [D 20:00:57.645186] exiting distribute returned less than expected [D 20:00:57.645210] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 421527552 fo 436207616 eof 0 [D 20:00:57.645236] ========================================================= [D 20:00:57.645260] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:00:57.645290] element 0: offset: 0xab459000, size: 2097152 [D 20:00:57.645318] post_sr: da13:3346 bh 84 len 32 wr_id 15e73360. [D 20:00:57.645348] PINT_client_state_machine_test id 22 [D 20:00:57.646556] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 8. [D 20:00:57.646584] post_sr: da13:3346 bh 85 len 16 wr_id 15e73378. [D 20:00:57.649014] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 7. [D 20:00:57.649042] post_sr: da13:3346 bh 86 len 16 wr_id 15e73390. [D 20:00:57.651469] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 6. [D 20:00:57.651496] post_sr: da13:3346 bh 87 len 16 wr_id 15e733a8. [D 20:00:57.651523] openib_check_cq: completed send wr_id 15e73318. [D 20:00:57.651554] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:00:57.651579] ========================================================= [D 20:00:57.651603] PINT_process_request [D 20:00:57.651627] tiling 15 copies [D 20:00:57.651652] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.651679] lvl 0 el 0 blk 0 by 452984832 [D 20:00:57.651703] to 452984832 ta 436207616 fi 469762048 [D 20:00:57.651728] mto 16777216 mta 14680064 mfi 16777216 [D 20:00:57.651753] basic type or contiguous data [D 20:00:57.651777] Request disp [D 20:00:57.651803] contig_offset = 452984832 contig_size = 50331648 lvl_flag = 1 [D 20:00:57.651827] calling distribute [D 20:00:57.651852] PINT_distribute [D 20:00:57.651875] of 452984832 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 452984832 exfl 1 [D 20:00:57.651902] begin iteration loff: 452984832 [D 20:00:57.651926] segment exceeds byte limit [D 20:00:57.651951] file being extended [D 20:00:57.651975] client lstof 16777216 diff 0 sgof 16777216 [D 20:00:57.651999] **********CALL***PROCESS********* [D 20:00:57.652023] segment of 16777216 sz 2097152 [D 20:00:57.652048] PINT_process_request [D 20:00:57.652071] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.652098] lvl 0 el 0 blk 0 by 16777216 [D 20:00:57.652123] to 16777216 ta 16777216 fi 18874368 [D 20:00:57.652148] basic type or contiguous data [D 20:00:57.652172] Request disp [D 20:00:57.652194] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:00:57.652219] process a segment [D 20:00:57.652243] of 16777216 sz 2097152 [D 20:00:57.652268] add a segment 0 [D 20:00:57.652292] retval = 2097152 [D 20:00:57.652316] exiting distribute returned less than expected [D 20:00:57.652339] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:00:57.652366] *****RETURN***FROM***PROCESS***** [D 20:00:57.652390] end iteration [D 20:00:57.652414] done with segments or bytes [D 20:00:57.652438] of 455081984 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.652463] next loff: 455081984 next poff: 455081984 [D 20:00:57.652488] return value 2097152 [D 20:00:57.652512] finished [D 20:00:57.652536] retval = 2097152 [D 20:00:57.652560] exiting distribute returned less than expected [D 20:00:57.652584] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 455081984 fo 469762048 eof 0 [D 20:00:57.652609] ========================================================= [D 20:00:57.652634] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:00:57.652659] element 0: offset: 0xa9457000, size: 2097152 [D 20:00:57.652688] post_sr: da13:3346 bh 88 len 32 wr_id 15e733c0. [D 20:00:57.653930] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 5. [D 20:00:57.653958] post_sr: da13:3346 bh 89 len 16 wr_id 15e733d8. [D 20:00:57.655376] PINT_client_state_machine_test id 22 [D 20:00:57.656388] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 4. [D 20:00:57.656415] post_sr: da13:3346 bh 90 len 16 wr_id 15e733f0. [D 20:00:57.658857] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 3. [D 20:00:57.659118] post_sr: da13:3346 bh 91 len 16 wr_id 15e73408. [D 20:00:57.661315] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 2. [D 20:00:57.661345] post_sr: da13:3346 bh 92 len 16 wr_id 15e73420. [D 20:00:57.663776] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 1. [D 20:00:57.663804] post_sr: da13:3346 bh 93 len 16 wr_id 15e73438. [D 20:00:57.665407] PINT_client_state_machine_test id 22 [D 20:00:57.666235] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 0. [D 20:00:57.666263] post_sr: da13:3346 bh 94 len 16 wr_id 15e73450. [D 20:00:57.666290] openib_check_cq: completed send wr_id 15e73330. [D 20:00:57.666316] openib_check_cq: completed send wr_id 15e73348. [D 20:00:57.666340] openib_check_cq: completed send wr_id 15e73360. [D 20:00:57.666366] openib_check_cq: completed send wr_id 15e73378. [D 20:00:57.666391] openib_check_cq: completed send wr_id 15e73390. [D 20:00:57.666417] openib_check_cq: completed send wr_id 15e733a8. [D 20:00:57.666441] openib_check_cq: completed send wr_id 15e733c0. [D 20:00:57.666467] openib_check_cq: completed send wr_id 15e733d8. [D 20:00:57.666492] openib_check_cq: completed send wr_id 15e733f0. [D 20:00:57.666518] openib_check_cq: completed send wr_id 15e73408. [D 20:00:57.666543] openib_check_cq: completed send wr_id 15e73420. [D 20:00:57.666568] openib_check_cq: completed send wr_id 15e73438. [D 20:00:57.666593] openib_check_cq: completed send wr_id 15e73450. [D 20:00:57.666625] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:00:57.666650] ========================================================= [D 20:00:57.666678] PINT_process_request [D 20:00:57.666706] tiling 2 copies [D 20:00:57.666733] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.666764] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.666790] to 18874368 ta 0 fi 33554432 [D 20:00:57.666815] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.666840] basic type or contiguous data [D 20:00:57.666864] Request disp [D 20:00:57.666889] contig_offset = 18874368 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.666913] calling distribute [D 20:00:57.666937] PINT_distribute [D 20:00:57.666961] of 18874368 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 18874368 exfl 1 [D 20:00:57.666987] begin iteration loff: 18874368 [D 20:00:57.667013] segment exceeds byte limit [D 20:00:57.667037] file being extended [D 20:00:57.667060] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.667085] **********CALL***PROCESS********* [D 20:00:57.667109] segment of 18874368 sz 2097152 [D 20:00:57.667134] PINT_process_request [D 20:00:57.667158] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.667186] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.667347] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.667373] basic type or contiguous data [D 20:00:57.667397] Request disp [D 20:00:57.667421] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.667447] process a segment [D 20:00:57.667470] of 18874368 sz 2097152 [D 20:00:57.667494] add a segment 0 [D 20:00:57.667519] retval = 2097152 [D 20:00:57.667543] exiting distribute returned less than expected [D 20:00:57.667584] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.667611] *****RETURN***FROM***PROCESS***** [D 20:00:57.667635] end iteration [D 20:00:57.667659] done with segments or bytes [D 20:00:57.667683] of 20971520 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.667710] next loff: 20971520 next poff: 20971520 [D 20:00:57.667735] return value 2097152 [D 20:00:57.667758] finished [D 20:00:57.667782] retval = 2097152 [D 20:00:57.667807] exiting distribute returned less than expected [D 20:00:57.667831] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 20971520 fo 33554432 eof 0 [D 20:00:57.667857] ========================================================= [D 20:00:57.667881] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:00:57.667907] element 0: offset: 0xc3671000, size: 2097152 [D 20:00:57.667934] post_sr: da13:3346 bh 95 len 32 wr_id 15e73468. [D 20:00:57.667965] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:00:57.667991] ========================================================= [D 20:00:57.668016] PINT_process_request [D 20:00:57.668040] tiling 3 copies [D 20:00:57.668064] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.668090] lvl 0 el 0 blk 0 by 52428800 [D 20:00:57.668115] to 52428800 ta 33554432 fi 67108864 [D 20:00:57.668140] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.668165] basic type or contiguous data [D 20:00:57.668189] Request disp [D 20:00:57.668212] contig_offset = 52428800 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.668237] calling distribute [D 20:00:57.668261] PINT_distribute [D 20:00:57.668285] of 52428800 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 52428800 exfl 1 [D 20:00:57.668312] begin iteration loff: 52428800 [D 20:00:57.668336] segment exceeds byte limit [D 20:00:57.668360] file being extended [D 20:00:57.668384] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.668409] **********CALL***PROCESS********* [D 20:00:57.668433] segment of 18874368 sz 2097152 [D 20:00:57.668457] PINT_process_request [D 20:00:57.668481] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.668507] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.668532] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.668557] basic type or contiguous data [D 20:00:57.668581] Request disp [D 20:00:57.668604] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.668629] process a segment [D 20:00:57.668653] of 18874368 sz 2097152 [D 20:00:57.668681] add a segment 0 [D 20:00:57.668704] retval = 2097152 [D 20:00:57.668728] exiting distribute returned less than expected [D 20:00:57.668752] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.668779] *****RETURN***FROM***PROCESS***** [D 20:00:57.668803] end iteration [D 20:00:57.668827] done with segments or bytes [D 20:00:57.668850] of 54525952 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.668876] next loff: 54525952 next poff: 54525952 [D 20:00:57.668901] return value 2097152 [D 20:00:57.668926] finished [D 20:00:57.668949] retval = 2097152 [D 20:00:57.668973] exiting distribute returned less than expected [D 20:00:57.668997] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 54525952 fo 67108864 eof 0 [D 20:00:57.669023] ========================================================= [D 20:00:57.669048] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:00:57.669073] element 0: offset: 0xc166f000, size: 2097152 [D 20:00:57.669100] post_sr: da13:3346 bh 96 len 32 wr_id 15e73480. [D 20:00:57.669128] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:00:57.669154] ========================================================= [D 20:00:57.669178] PINT_process_request [D 20:00:57.669202] tiling 4 copies [D 20:00:57.669226] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.669252] lvl 0 el 0 blk 0 by 85983232 [D 20:00:57.669277] to 85983232 ta 67108864 fi 100663296 [D 20:00:57.669301] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.669326] basic type or contiguous data [D 20:00:57.669350] Request disp [D 20:00:57.669374] contig_offset = 85983232 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.669400] calling distribute [D 20:00:57.669423] PINT_distribute [D 20:00:57.669446] of 85983232 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 85983232 exfl 1 [D 20:00:57.669473] begin iteration loff: 85983232 [D 20:00:57.669498] segment exceeds byte limit [D 20:00:57.669522] file being extended [D 20:00:57.669546] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.669570] **********CALL***PROCESS********* [D 20:00:57.669594] segment of 18874368 sz 2097152 [D 20:00:57.669619] PINT_process_request [D 20:00:57.669643] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.669677] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.669702] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.669727] basic type or contiguous data [D 20:00:57.669751] Request disp [D 20:00:57.669775] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.669800] process a segment [D 20:00:57.669824] of 18874368 sz 2097152 [D 20:00:57.669848] add a segment 0 [D 20:00:57.669872] retval = 2097152 [D 20:00:57.669896] exiting distribute returned less than expected [D 20:00:57.669920] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.669946] *****RETURN***FROM***PROCESS***** [D 20:00:57.669970] end iteration [D 20:00:57.669994] done with segments or bytes [D 20:00:57.670018] of 88080384 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.670043] next loff: 88080384 next poff: 88080384 [D 20:00:57.670068] return value 2097152 [D 20:00:57.670092] finished [D 20:00:57.670116] retval = 2097152 [D 20:00:57.670140] exiting distribute returned less than expected [D 20:00:57.670164] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 88080384 fo 100663296 eof 0 [D 20:00:57.670190] ========================================================= [D 20:00:57.670214] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:00:57.670240] element 0: offset: 0xbf66d000, size: 2097152 [D 20:00:57.670267] post_sr: da13:3346 bh 97 len 32 wr_id 15e73498. [D 20:00:57.670295] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:00:57.670321] ========================================================= [D 20:00:57.670346] PINT_process_request [D 20:00:57.670370] tiling 5 copies [D 20:00:57.670393] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.670420] lvl 0 el 0 blk 0 by 119537664 [D 20:00:57.670445] to 119537664 ta 100663296 fi 134217728 [D 20:00:57.670470] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.670495] basic type or contiguous data [D 20:00:57.670518] Request disp [D 20:00:57.670542] contig_offset = 119537664 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.670567] calling distribute [D 20:00:57.670591] PINT_distribute [D 20:00:57.670615] of 119537664 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 119537664 exfl 1 [D 20:00:57.670641] begin iteration loff: 119537664 [D 20:00:57.670667] segment exceeds byte limit [D 20:00:57.670696] file being extended [D 20:00:57.670722] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.670751] **********CALL***PROCESS********* [D 20:00:57.670779] segment of 18874368 sz 2097152 [D 20:00:57.670804] PINT_process_request [D 20:00:57.670827] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.670853] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.670879] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.670904] basic type or contiguous data [D 20:00:57.670928] Request disp [D 20:00:57.670951] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.670976] process a segment [D 20:00:57.671000] of 18874368 sz 2097152 [D 20:00:57.671024] add a segment 0 [D 20:00:57.671048] retval = 2097152 [D 20:00:57.671072] exiting distribute returned less than expected [D 20:00:57.671096] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.671122] *****RETURN***FROM***PROCESS***** [D 20:00:57.671147] end iteration [D 20:00:57.671171] done with segments or bytes [D 20:00:57.671194] of 121634816 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.671219] next loff: 121634816 next poff: 121634816 [D 20:00:57.671245] return value 2097152 [D 20:00:57.671269] finished [D 20:00:57.671293] retval = 2097152 [D 20:00:57.671317] exiting distribute returned less than expected [D 20:00:57.671341] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 121634816 fo 134217728 eof 0 [D 20:00:57.671367] ========================================================= [D 20:00:57.671392] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:00:57.671420] element 0: offset: 0xbd66b000, size: 2097152 [D 20:00:57.671447] post_sr: da13:3346 bh 98 len 32 wr_id 15e734b0. [D 20:00:57.671476] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:00:57.671502] ========================================================= [D 20:00:57.671525] PINT_process_request [D 20:00:57.671549] tiling 6 copies [D 20:00:57.671574] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.671600] lvl 0 el 0 blk 0 by 153092096 [D 20:00:57.671626] to 153092096 ta 134217728 fi 167772160 [D 20:00:57.671650] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.671675] basic type or contiguous data [D 20:00:57.671699] Request disp [D 20:00:57.671723] contig_offset = 153092096 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.671749] calling distribute [D 20:00:57.671772] PINT_distribute [D 20:00:57.671797] of 153092096 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 153092096 exfl 1 [D 20:00:57.671824] begin iteration loff: 153092096 [D 20:00:57.671849] segment exceeds byte limit [D 20:00:57.671874] file being extended [D 20:00:57.671897] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.671922] **********CALL***PROCESS********* [D 20:00:57.671946] segment of 18874368 sz 2097152 [D 20:00:57.671970] PINT_process_request [D 20:00:57.671995] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.672020] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.672045] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.672070] basic type or contiguous data [D 20:00:57.672094] Request disp [D 20:00:57.672118] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.672142] process a segment [D 20:00:57.672166] of 18874368 sz 2097152 [D 20:00:57.672191] add a segment 0 [D 20:00:57.672215] retval = 2097152 [D 20:00:57.672239] exiting distribute returned less than expected [D 20:00:57.672263] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.672289] *****RETURN***FROM***PROCESS***** [D 20:00:57.672313] end iteration [D 20:00:57.672337] done with segments or bytes [D 20:00:57.672360] of 155189248 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.672387] next loff: 155189248 next poff: 155189248 [D 20:00:57.672411] return value 2097152 [D 20:00:57.672435] finished [D 20:00:57.672459] retval = 2097152 [D 20:00:57.672483] exiting distribute returned less than expected [D 20:00:57.672507] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 155189248 fo 167772160 eof 0 [D 20:00:57.672533] ========================================================= [D 20:00:57.672557] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:00:57.672582] element 0: offset: 0xbb669000, size: 2097152 [D 20:00:57.672610] post_sr: da13:3346 bh 99 len 32 wr_id 15e734c8. [D 20:00:57.672639] openib_check_cq: completed send wr_id 15e73468. [D 20:00:57.672668] openib_check_cq: completed send wr_id 15e73480. [D 20:00:57.672694] openib_check_cq: completed send wr_id 15e73498. [D 20:00:57.672719] openib_check_cq: completed send wr_id 15e734b0. [D 20:00:57.672744] openib_check_cq: completed send wr_id 15e734c8. [D 20:00:57.672776] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:00:57.672801] ========================================================= [D 20:00:57.672825] PINT_process_request [D 20:00:57.672849] tiling 7 copies [D 20:00:57.672874] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.672900] lvl 0 el 0 blk 0 by 186646528 [D 20:00:57.672924] to 186646528 ta 167772160 fi 201326592 [D 20:00:57.672949] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.672975] basic type or contiguous data [D 20:00:57.672999] Request disp [D 20:00:57.673023] contig_offset = 186646528 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.673048] calling distribute [D 20:00:57.673071] PINT_distribute [D 20:00:57.673098] of 186646528 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 186646528 exfl 1 [D 20:00:57.673125] begin iteration loff: 186646528 [D 20:00:57.673150] segment exceeds byte limit [D 20:00:57.673173] file being extended [D 20:00:57.673197] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.673222] **********CALL***PROCESS********* [D 20:00:57.673246] segment of 18874368 sz 2097152 [D 20:00:57.673271] PINT_process_request [D 20:00:57.673295] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.673320] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.673346] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.673371] basic type or contiguous data [D 20:00:57.673395] Request disp [D 20:00:57.673418] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.673443] process a segment [D 20:00:57.673467] of 18874368 sz 2097152 [D 20:00:57.673491] add a segment 0 [D 20:00:57.673516] retval = 2097152 [D 20:00:57.673540] exiting distribute returned less than expected [D 20:00:57.673563] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.673589] *****RETURN***FROM***PROCESS***** [D 20:00:57.673614] end iteration [D 20:00:57.673638] done with segments or bytes [D 20:00:57.673662] of 188743680 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.673687] next loff: 188743680 next poff: 188743680 [D 20:00:57.673711] return value 2097152 [D 20:00:57.673736] finished [D 20:00:57.673760] retval = 2097152 [D 20:00:57.673784] exiting distribute returned less than expected [D 20:00:57.673808] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 188743680 fo 201326592 eof 0 [D 20:00:57.673834] ========================================================= [D 20:00:57.673858] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:00:57.673884] element 0: offset: 0xb9667000, size: 2097152 [D 20:00:57.673911] post_sr: da13:3346 bh 0 len 32 wr_id 15e72b80. [D 20:00:57.673939] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:00:57.673965] ========================================================= [D 20:00:57.673990] PINT_process_request [D 20:00:57.674014] tiling 8 copies [D 20:00:57.674037] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.674064] lvl 0 el 0 blk 0 by 220200960 [D 20:00:57.674089] to 220200960 ta 201326592 fi 234881024 [D 20:00:57.674114] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.674139] basic type or contiguous data [D 20:00:57.674162] Request disp [D 20:00:57.674186] contig_offset = 220200960 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.674212] calling distribute [D 20:00:57.674236] PINT_distribute [D 20:00:57.674260] of 220200960 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 220200960 exfl 1 [D 20:00:57.674285] begin iteration loff: 220200960 [D 20:00:57.674310] segment exceeds byte limit [D 20:00:57.674334] file being extended [D 20:00:57.674358] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.674383] **********CALL***PROCESS********* [D 20:00:57.674407] segment of 18874368 sz 2097152 [D 20:00:57.674431] PINT_process_request [D 20:00:57.674455] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.674482] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.674506] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.674530] basic type or contiguous data [D 20:00:57.674554] Request disp [D 20:00:57.674578] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.674603] process a segment [D 20:00:57.674627] of 18874368 sz 2097152 [D 20:00:57.674651] add a segment 0 [D 20:00:57.674678] retval = 2097152 [D 20:00:57.674706] exiting distribute returned less than expected [D 20:00:57.674734] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.674763] *****RETURN***FROM***PROCESS***** [D 20:00:57.674790] end iteration [D 20:00:57.674814] done with segments or bytes [D 20:00:57.674838] of 222298112 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.674866] next loff: 222298112 next poff: 222298112 [D 20:00:57.674891] return value 2097152 [D 20:00:57.674916] finished [D 20:00:57.674940] retval = 2097152 [D 20:00:57.674963] exiting distribute returned less than expected [D 20:00:57.674987] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 222298112 fo 234881024 eof 0 [D 20:00:57.675013] ========================================================= [D 20:00:57.675038] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:00:57.675063] element 0: offset: 0xb7665000, size: 2097152 [D 20:00:57.675090] post_sr: da13:3346 bh 1 len 32 wr_id 15e72b98. [D 20:00:57.675118] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:00:57.675144] ========================================================= [D 20:00:57.675168] PINT_process_request [D 20:00:57.675193] tiling 9 copies [D 20:00:57.675217] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.675243] lvl 0 el 0 blk 0 by 253755392 [D 20:00:57.675268] to 253755392 ta 234881024 fi 268435456 [D 20:00:57.675292] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.675317] basic type or contiguous data [D 20:00:57.675342] Request disp [D 20:00:57.675366] contig_offset = 253755392 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.675392] calling distribute [D 20:00:57.675415] PINT_distribute [D 20:00:57.675439] of 253755392 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 253755392 exfl 1 [D 20:00:57.675466] begin iteration loff: 253755392 [D 20:00:57.675491] segment exceeds byte limit [D 20:00:57.675515] file being extended [D 20:00:57.675539] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.675563] **********CALL***PROCESS********* [D 20:00:57.675587] segment of 18874368 sz 2097152 [D 20:00:57.675612] PINT_process_request [D 20:00:57.675636] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.675662] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.675686] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.675711] basic type or contiguous data [D 20:00:57.675735] Request disp [D 20:00:57.675760] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.675785] process a segment [D 20:00:57.675808] of 18874368 sz 2097152 [D 20:00:57.675832] add a segment 0 [D 20:00:57.675856] retval = 2097152 [D 20:00:57.675880] exiting distribute returned less than expected [D 20:00:57.675904] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.675930] *****RETURN***FROM***PROCESS***** [D 20:00:57.675954] end iteration [D 20:00:57.675979] done with segments or bytes [D 20:00:57.676002] of 255852544 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.676028] next loff: 255852544 next poff: 255852544 [D 20:00:57.676053] return value 2097152 [D 20:00:57.676077] finished [D 20:00:57.676101] retval = 2097152 [D 20:00:57.676125] exiting distribute returned less than expected [D 20:00:57.676149] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 255852544 fo 268435456 eof 0 [D 20:00:57.676176] ========================================================= [D 20:00:57.676199] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:00:57.676225] element 0: offset: 0xb5663000, size: 2097152 [D 20:00:57.676252] post_sr: da13:3346 bh 2 len 32 wr_id 15e72bb0. [D 20:00:57.676280] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:00:57.676306] ========================================================= [D 20:00:57.676331] PINT_process_request [D 20:00:57.676355] tiling 10 copies [D 20:00:57.676379] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.676405] lvl 0 el 0 blk 0 by 287309824 [D 20:00:57.676429] to 287309824 ta 268435456 fi 301989888 [D 20:00:57.676454] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.676480] basic type or contiguous data [D 20:00:57.676504] Request disp [D 20:00:57.676529] contig_offset = 287309824 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.676555] calling distribute [D 20:00:57.676579] PINT_distribute [D 20:00:57.676603] of 287309824 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 287309824 exfl 1 [D 20:00:57.676630] begin iteration loff: 287309824 [D 20:00:57.676654] segment exceeds byte limit [D 20:00:57.676681] file being extended [D 20:00:57.676705] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.676730] **********CALL***PROCESS********* [D 20:00:57.676753] segment of 18874368 sz 2097152 [D 20:00:57.676778] PINT_process_request [D 20:00:57.676802] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.676829] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.676854] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.676878] basic type or contiguous data [D 20:00:57.676903] Request disp [D 20:00:57.676927] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.676952] process a segment [D 20:00:57.676976] of 18874368 sz 2097152 [D 20:00:57.677000] add a segment 0 [D 20:00:57.677023] retval = 2097152 [D 20:00:57.677048] exiting distribute returned less than expected [D 20:00:57.677072] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.677098] *****RETURN***FROM***PROCESS***** [D 20:00:57.677123] end iteration [D 20:00:57.677146] done with segments or bytes [D 20:00:57.677170] of 289406976 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.677196] next loff: 289406976 next poff: 289406976 [D 20:00:57.677221] return value 2097152 [D 20:00:57.677246] finished [D 20:00:57.677269] retval = 2097152 [D 20:00:57.677293] exiting distribute returned less than expected [D 20:00:57.677317] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 289406976 fo 301989888 eof 0 [D 20:00:57.677344] ========================================================= [D 20:00:57.677368] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:00:57.677392] element 0: offset: 0xb3661000, size: 2097152 [D 20:00:57.677420] post_sr: da13:3346 bh 3 len 32 wr_id 15e72bc8. [D 20:00:57.677449] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:00:57.677474] ========================================================= [D 20:00:57.677498] PINT_process_request [D 20:00:57.677522] tiling 11 copies [D 20:00:57.677547] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.677591] lvl 0 el 0 blk 0 by 320864256 [D 20:00:57.677616] to 320864256 ta 301989888 fi 335544320 [D 20:00:57.677640] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.677665] basic type or contiguous data [D 20:00:57.677690] Request disp [D 20:00:57.677714] contig_offset = 320864256 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.677739] calling distribute [D 20:00:57.677763] PINT_distribute [D 20:00:57.677787] of 320864256 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 320864256 exfl 1 [D 20:00:57.677814] begin iteration loff: 320864256 [D 20:00:57.677839] segment exceeds byte limit [D 20:00:57.677863] file being extended [D 20:00:57.677886] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.677911] **********CALL***PROCESS********* [D 20:00:57.677935] segment of 18874368 sz 2097152 [D 20:00:57.677960] PINT_process_request [D 20:00:57.677984] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.678010] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.678035] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.678060] basic type or contiguous data [D 20:00:57.678084] Request disp [D 20:00:57.678108] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.678133] process a segment [D 20:00:57.678156] of 18874368 sz 2097152 [D 20:00:57.678181] add a segment 0 [D 20:00:57.678205] retval = 2097152 [D 20:00:57.678230] exiting distribute returned less than expected [D 20:00:57.678254] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.678282] *****RETURN***FROM***PROCESS***** [D 20:00:57.678307] end iteration [D 20:00:57.678331] done with segments or bytes [D 20:00:57.678355] of 322961408 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.678380] next loff: 322961408 next poff: 322961408 [D 20:00:57.678405] return value 2097152 [D 20:00:57.678430] finished [D 20:00:57.678453] retval = 2097152 [D 20:00:57.678478] exiting distribute returned less than expected [D 20:00:57.678502] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 322961408 fo 335544320 eof 0 [D 20:00:57.678527] ========================================================= [D 20:00:57.678552] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:00:57.678577] element 0: offset: 0xb165f000, size: 2097152 [D 20:00:57.678604] post_sr: da13:3346 bh 4 len 32 wr_id 15e72be0. [D 20:00:57.678633] PINT_client_state_machine_test id 22 [D 20:00:57.678662] openib_check_cq: completed send wr_id 15e72b80. [D 20:00:57.678693] openib_check_cq: completed send wr_id 15e72b98. [D 20:00:57.678722] openib_check_cq: completed send wr_id 15e72bb0. [D 20:00:57.678750] openib_check_cq: completed send wr_id 15e72bc8. [D 20:00:57.678779] openib_check_cq: completed send wr_id 15e72be0. [D 20:00:57.678810] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:00:57.678835] ========================================================= [D 20:00:57.678859] PINT_process_request [D 20:00:57.678884] tiling 12 copies [D 20:00:57.678908] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.678935] lvl 0 el 0 blk 0 by 354418688 [D 20:00:57.678959] to 354418688 ta 335544320 fi 369098752 [D 20:00:57.678984] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:57.679009] basic type or contiguous data [D 20:00:57.679033] Request disp [D 20:00:57.679058] contig_offset = 354418688 contig_size = 48234496 lvl_flag = 1 [D 20:00:57.679082] calling distribute [D 20:00:57.679106] PINT_distribute [D 20:00:57.679130] of 354418688 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 354418688 exfl 1 [D 20:00:57.679157] begin iteration loff: 354418688 [D 20:00:57.679182] segment exceeds byte limit [D 20:00:57.679205] file being extended [D 20:00:57.679229] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:57.679254] **********CALL***PROCESS********* [D 20:00:57.679278] segment of 18874368 sz 2097152 [D 20:00:57.679303] PINT_process_request [D 20:00:57.679327] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:57.679352] lvl 0 el 0 blk 0 by 18874368 [D 20:00:57.679377] to 18874368 ta 18874368 fi 20971520 [D 20:00:57.679402] basic type or contiguous data [D 20:00:57.679426] Request disp [D 20:00:57.679450] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:57.679475] process a segment [D 20:00:57.679499] of 18874368 sz 2097152 [D 20:00:57.679523] add a segment 0 [D 20:00:57.679547] retval = 2097152 [D 20:00:57.679572] exiting distribute returned less than expected [D 20:00:57.679595] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:57.679622] *****RETURN***FROM***PROCESS***** [D 20:00:57.679646] end iteration [D 20:00:57.679670] done with segments or bytes [D 20:00:57.679694] of 356515840 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:57.679720] next loff: 356515840 next poff: 356515840 [D 20:00:57.679745] return value 2097152 [D 20:00:57.679769] finished [D 20:00:57.679793] retval = 2097152 [D 20:00:57.679817] exiting distribute returned less than expected [D 20:00:57.679841] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 356515840 fo 369098752 eof 0 [D 20:00:57.679867] ========================================================= [D 20:00:57.679892] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:00:57.679918] element 0: offset: 0xaf65d000, size: 2097152 [D 20:00:57.679944] post_sr: da13:3346 bh 5 len 32 wr_id 15e72bf8. [D 20:00:57.679980] openib_check_cq: completed send wr_id 15e72bf8. [D 20:00:57.688663] PINT_client_state_machine_test id 22 [D 20:00:57.698698] PINT_client_state_machine_test id 22 [D 20:00:57.708729] PINT_client_state_machine_test id 22 [D 20:00:57.718759] PINT_client_state_machine_test id 22 [D 20:00:57.736673] PINT_client_state_machine_test id 22 [D 20:00:57.748672] PINT_client_state_machine_test id 22 [D 20:00:57.760672] PINT_client_state_machine_test id 22 [D 20:00:57.772675] PINT_client_state_machine_test id 22 [D 20:00:57.784672] PINT_client_state_machine_test id 22 [D 20:00:57.796672] PINT_client_state_machine_test id 22 [D 20:00:57.808672] PINT_client_state_machine_test id 22 [D 20:00:57.820672] PINT_client_state_machine_test id 22 [D 20:00:57.832673] PINT_client_state_machine_test id 22 [D 20:00:57.844672] PINT_client_state_machine_test id 22 [D 20:00:57.856672] PINT_client_state_machine_test id 22 [D 20:00:57.868672] PINT_client_state_machine_test id 22 [D 20:00:57.880672] PINT_client_state_machine_test id 22 [D 20:00:57.892672] PINT_client_state_machine_test id 22 [D 20:00:57.904674] PINT_client_state_machine_test id 22 [D 20:00:57.916672] PINT_client_state_machine_test id 22 [D 20:00:57.928673] PINT_client_state_machine_test id 22 [D 20:00:57.940672] PINT_client_state_machine_test id 22 [D 20:00:57.952672] PINT_client_state_machine_test id 22 [D 20:00:57.964673] PINT_client_state_machine_test id 22 [D 20:00:57.976673] PINT_client_state_machine_test id 22 [D 20:00:57.988673] PINT_client_state_machine_test id 22 [D 20:00:58.000673] PINT_client_state_machine_test id 22 [D 20:00:58.012674] PINT_client_state_machine_test id 22 [D 20:00:58.024674] PINT_client_state_machine_test id 22 [D 20:00:58.036673] PINT_client_state_machine_test id 22 [D 20:00:58.048673] PINT_client_state_machine_test id 22 [D 20:00:58.060673] PINT_client_state_machine_test id 22 [D 20:00:58.072673] PINT_client_state_machine_test id 22 [D 20:00:58.084673] PINT_client_state_machine_test id 22 [D 20:00:58.096674] PINT_client_state_machine_test id 22 [D 20:00:58.108674] PINT_client_state_machine_test id 22 [D 20:00:58.120675] PINT_client_state_machine_test id 22 [D 20:00:58.132674] PINT_client_state_machine_test id 22 [D 20:00:58.144674] PINT_client_state_machine_test id 22 [D 20:00:58.156674] PINT_client_state_machine_test id 22 [D 20:00:58.168674] PINT_client_state_machine_test id 22 [D 20:00:58.180674] PINT_client_state_machine_test id 22 [D 20:00:58.192674] PINT_client_state_machine_test id 22 [D 20:00:58.204674] PINT_client_state_machine_test id 22 [D 20:00:58.216675] PINT_client_state_machine_test id 22 [D 20:00:58.228674] PINT_client_state_machine_test id 22 [D 20:00:58.240674] PINT_client_state_machine_test id 22 [D 20:00:58.252675] PINT_client_state_machine_test id 22 [D 20:00:58.264675] PINT_client_state_machine_test id 22 [D 20:00:58.276675] PINT_client_state_machine_test id 22 [D 20:00:58.288675] PINT_client_state_machine_test id 22 [D 20:00:58.300675] PINT_client_state_machine_test id 22 [D 20:00:58.312675] PINT_client_state_machine_test id 22 [D 20:00:58.324675] PINT_client_state_machine_test id 22 [D 20:00:58.336675] PINT_client_state_machine_test id 22 [D 20:00:58.348674] PINT_client_state_machine_test id 22 [D 20:00:58.360674] PINT_client_state_machine_test id 22 [D 20:00:58.372674] PINT_client_state_machine_test id 22 [D 20:00:58.384675] PINT_client_state_machine_test id 22 [D 20:00:58.384707] sm control op 300 [D 20:00:58.384733] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:00:58.384758] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:00:58.384787] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:00:58.384813] PINT_client_state_machine_test id 22 [D 20:00:58.396675] PINT_client_state_machine_test id 22 [D 20:00:58.408676] PINT_client_state_machine_test id 22 [D 20:00:58.420675] PINT_client_state_machine_test id 22 [D 20:00:58.432675] PINT_client_state_machine_test id 22 [D 20:00:58.444675] PINT_client_state_machine_test id 22 [D 20:00:58.456675] PINT_client_state_machine_test id 22 [D 20:00:58.468675] PINT_client_state_machine_test id 22 [D 20:00:58.480676] PINT_client_state_machine_test id 22 [D 20:00:58.492675] PINT_client_state_machine_test id 22 [D 20:00:58.504676] PINT_client_state_machine_test id 22 [D 20:00:58.516676] PINT_client_state_machine_test id 22 [D 20:00:58.528676] PINT_client_state_machine_test id 22 [D 20:00:58.540676] PINT_client_state_machine_test id 22 [D 20:00:58.552676] PINT_client_state_machine_test id 22 [D 20:00:58.564676] PINT_client_state_machine_test id 22 [D 20:00:58.576676] PINT_client_state_machine_test id 22 [D 20:00:58.588676] PINT_client_state_machine_test id 22 [D 20:00:58.600677] PINT_client_state_machine_test id 22 [D 20:00:58.612676] PINT_client_state_machine_test id 22 [D 20:00:58.624676] PINT_client_state_machine_test id 22 [D 20:00:58.636676] PINT_client_state_machine_test id 22 [D 20:00:58.648676] PINT_client_state_machine_test id 22 [D 20:00:58.660676] PINT_client_state_machine_test id 22 [D 20:00:58.672677] PINT_client_state_machine_test id 22 [D 20:00:58.684677] PINT_client_state_machine_test id 22 [D 20:00:58.696678] PINT_client_state_machine_test id 22 [D 20:00:58.708677] PINT_client_state_machine_test id 22 [D 20:00:58.720676] PINT_client_state_machine_test id 22 [D 20:00:58.732680] PINT_client_state_machine_test id 22 [D 20:00:58.744677] PINT_client_state_machine_test id 22 [D 20:00:58.756677] PINT_client_state_machine_test id 22 [D 20:00:58.768677] PINT_client_state_machine_test id 22 [D 20:00:58.780676] PINT_client_state_machine_test id 22 [D 20:00:58.792677] PINT_client_state_machine_test id 22 [D 20:00:58.804677] PINT_client_state_machine_test id 22 [D 20:00:58.816677] PINT_client_state_machine_test id 22 [D 20:00:58.828679] PINT_client_state_machine_test id 22 [D 20:00:58.840677] PINT_client_state_machine_test id 22 [D 20:00:58.852678] PINT_client_state_machine_test id 22 [D 20:00:58.864678] PINT_client_state_machine_test id 22 [D 20:00:58.876677] PINT_client_state_machine_test id 22 [D 20:00:58.888678] PINT_client_state_machine_test id 22 [D 20:00:58.900677] PINT_client_state_machine_test id 22 [D 20:00:58.912678] PINT_client_state_machine_test id 22 [D 20:00:58.924677] PINT_client_state_machine_test id 22 [D 20:00:58.936677] PINT_client_state_machine_test id 22 [D 20:00:58.948678] PINT_client_state_machine_test id 22 [D 20:00:58.960678] PINT_client_state_machine_test id 22 [D 20:00:58.972678] PINT_client_state_machine_test id 22 [D 20:00:58.984679] PINT_client_state_machine_test id 22 [D 20:00:58.996704] PINT_client_state_machine_test id 22 [D 20:00:59.008679] PINT_client_state_machine_test id 22 [D 20:00:59.020678] PINT_client_state_machine_test id 22 [D 20:00:59.032678] PINT_client_state_machine_test id 22 [D 20:00:59.044678] PINT_client_state_machine_test id 22 [D 20:00:59.056678] PINT_client_state_machine_test id 22 [D 20:00:59.068678] PINT_client_state_machine_test id 22 [D 20:00:59.080679] PINT_client_state_machine_test id 22 [D 20:00:59.092679] PINT_client_state_machine_test id 22 [D 20:00:59.104680] PINT_client_state_machine_test id 22 [D 20:00:59.116681] PINT_client_state_machine_test id 22 [D 20:00:59.128679] PINT_client_state_machine_test id 22 [D 20:00:59.140679] PINT_client_state_machine_test id 22 [D 20:00:59.152678] PINT_client_state_machine_test id 22 [D 20:00:59.164680] PINT_client_state_machine_test id 22 [D 20:00:59.176680] PINT_client_state_machine_test id 22 [D 20:00:59.188680] PINT_client_state_machine_test id 22 [D 20:00:59.200680] PINT_client_state_machine_test id 22 [D 20:00:59.212680] PINT_client_state_machine_test id 22 [D 20:00:59.224680] PINT_client_state_machine_test id 22 [D 20:00:59.236681] PINT_client_state_machine_test id 22 [D 20:00:59.248681] PINT_client_state_machine_test id 22 [D 20:00:59.260681] PINT_client_state_machine_test id 22 [D 20:00:59.272681] PINT_client_state_machine_test id 22 [D 20:00:59.284681] PINT_client_state_machine_test id 22 [D 20:00:59.296680] PINT_client_state_machine_test id 22 [D 20:00:59.308680] PINT_client_state_machine_test id 22 [D 20:00:59.320680] PINT_client_state_machine_test id 22 [D 20:00:59.332681] PINT_client_state_machine_test id 22 [D 20:00:59.344680] PINT_client_state_machine_test id 22 [D 20:00:59.356680] PINT_client_state_machine_test id 22 [D 20:00:59.368684] PINT_client_state_machine_test id 22 [D 20:00:59.380681] PINT_client_state_machine_test id 22 [D 20:00:59.392681] PINT_client_state_machine_test id 22 [D 20:00:59.392714] sm control op 300 [D 20:00:59.392740] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:00:59.392765] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:00:59.392794] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:00:59.392821] PINT_client_state_machine_test id 22 [D 20:00:59.404682] PINT_client_state_machine_test id 22 [D 20:00:59.416681] PINT_client_state_machine_test id 22 [D 20:00:59.428681] PINT_client_state_machine_test id 22 [D 20:00:59.440681] PINT_client_state_machine_test id 22 [D 20:00:59.452681] PINT_client_state_machine_test id 22 [D 20:00:59.464682] PINT_client_state_machine_test id 22 [D 20:00:59.476681] PINT_client_state_machine_test id 22 [D 20:00:59.488684] PINT_client_state_machine_test id 22 [D 20:00:59.500683] PINT_client_state_machine_test id 22 [D 20:00:59.512681] PINT_client_state_machine_test id 22 [D 20:00:59.524684] PINT_client_state_machine_test id 22 [D 20:00:59.536682] PINT_client_state_machine_test id 22 [D 20:00:59.548682] PINT_client_state_machine_test id 22 [D 20:00:59.560683] PINT_client_state_machine_test id 22 [D 20:00:59.572702] PINT_client_state_machine_test id 22 [D 20:00:59.584682] PINT_client_state_machine_test id 22 [D 20:00:59.596682] PINT_client_state_machine_test id 22 [D 20:00:59.608682] PINT_client_state_machine_test id 22 [D 20:00:59.620682] PINT_client_state_machine_test id 22 [D 20:00:59.632682] PINT_client_state_machine_test id 22 [D 20:00:59.644682] PINT_client_state_machine_test id 22 [D 20:00:59.656682] PINT_client_state_machine_test id 22 [D 20:00:59.668683] PINT_client_state_machine_test id 22 [D 20:00:59.680683] PINT_client_state_machine_test id 22 [D 20:00:59.692683] PINT_client_state_machine_test id 22 [D 20:00:59.704683] PINT_client_state_machine_test id 22 [D 20:00:59.716683] PINT_client_state_machine_test id 22 [D 20:00:59.728683] PINT_client_state_machine_test id 22 [D 20:00:59.740684] PINT_client_state_machine_test id 22 [D 20:00:59.752684] PINT_client_state_machine_test id 22 [D 20:00:59.764682] PINT_client_state_machine_test id 22 [D 20:00:59.776683] PINT_client_state_machine_test id 22 [D 20:00:59.788682] PINT_client_state_machine_test id 22 [D 20:00:59.800683] PINT_client_state_machine_test id 22 [D 20:00:59.812684] PINT_client_state_machine_test id 22 [D 20:00:59.824684] PINT_client_state_machine_test id 22 [D 20:00:59.836683] PINT_client_state_machine_test id 22 [D 20:00:59.848684] PINT_client_state_machine_test id 22 [D 20:00:59.860683] PINT_client_state_machine_test id 22 [D 20:00:59.872684] PINT_client_state_machine_test id 22 [D 20:00:59.884683] PINT_client_state_machine_test id 22 [D 20:00:59.896683] PINT_client_state_machine_test id 22 [D 20:00:59.908683] PINT_client_state_machine_test id 22 [D 20:00:59.920683] PINT_client_state_machine_test id 22 -[D 20:00:59.923109] openib_check_cq: completed recv wr_id 15e73980. [D 20:00:59.923141] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaaadcc6010 rkey c8002f00 nic_wr_credit 1. [D 20:00:59.923172] openib_post_rr: da13:3346 bh 49 wr_id 15e73980. [D 20:00:59.925641] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 0. [D 20:00:59.925668] post_sr: da13:3346 bh 6 len 16 wr_id 15e72c10. [D 20:00:59.925705] openib_check_cq: completed send wr_id 15e72c10. [D 20:00:59.925736] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:00:59.925762] ========================================================= [D 20:00:59.925786] PINT_process_request [D 20:00:59.925811] tiling 13 copies [D 20:00:59.925835] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:59.925866] lvl 0 el 0 blk 0 by 387973120 [D 20:00:59.925891] to 387973120 ta 369098752 fi 402653184 [D 20:00:59.925916] mto 18874368 mta 16777216 mfi 18874368 [D 20:00:59.925942] basic type or contiguous data [D 20:00:59.925966] Request disp [D 20:00:59.925990] contig_offset = 387973120 contig_size = 48234496 lvl_flag = 1 [D 20:00:59.926015] calling distribute [D 20:00:59.926039] PINT_distribute [D 20:00:59.926063] of 387973120 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 387973120 exfl 1 [D 20:00:59.926090] begin iteration loff: 387973120 [D 20:00:59.926115] segment exceeds byte limit [D 20:00:59.926139] file being extended [D 20:00:59.926163] client lstof 18874368 diff 0 sgof 18874368 [D 20:00:59.926188] **********CALL***PROCESS********* [D 20:00:59.926212] segment of 18874368 sz 2097152 [D 20:00:59.926237] PINT_process_request [D 20:00:59.926260] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:00:59.926287] lvl 0 el 0 blk 0 by 18874368 [D 20:00:59.926312] to 18874368 ta 18874368 fi 20971520 [D 20:00:59.926337] basic type or contiguous data [D 20:00:59.926361] Request disp [D 20:00:59.926384] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:00:59.926409] process a segment [D 20:00:59.926433] of 18874368 sz 2097152 [D 20:00:59.926458] add a segment 0 [D 20:00:59.926482] retval = 2097152 [D 20:00:59.926506] exiting distribute returned less than expected [D 20:00:59.926530] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:00:59.926556] *****RETURN***FROM***PROCESS***** [D 20:00:59.926581] end iteration [D 20:00:59.926605] done with segments or bytes [D 20:00:59.926629] of 390070272 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:00:59.926654] next loff: 390070272 next poff: 390070272 [D 20:00:59.926681] return value 2097152 [D 20:00:59.926710] finished [D 20:00:59.926737] retval = 2097152 [D 20:00:59.926765] exiting distribute returned less than expected [D 20:00:59.926793] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 390070272 fo 402653184 eof 0 [D 20:00:59.926819] ========================================================= [D 20:00:59.926844] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:00:59.926870] element 0: offset: 0xad65b000, size: 2097152 [D 20:00:59.926898] post_sr: da13:3346 bh 7 len 32 wr_id 15e72c28. [D 20:00:59.926935] openib_check_cq: completed send wr_id 15e72c28. [D 20:00:59.930716] PINT_client_state_machine_test id 22 [D 20:00:59.940748] PINT_client_state_machine_test id 22 [D 20:00:59.950778] PINT_client_state_machine_test id 22 [D 20:00:59.960810] PINT_client_state_machine_test id 22 [D 20:00:59.984684] PINT_client_state_machine_test id 22 [D 20:00:59.996685] PINT_client_state_machine_test id 22 [D 20:01:00.008685] PINT_client_state_machine_test id 22 -[D 20:01:00.008978] openib_check_cq: completed recv wr_id 15e73998. [D 20:01:00.009025] openib_post_sr_rdmaw: ibv_post_send wr_id 15f07a30 to da13:3346 remote addr 2aaabb051010 rkey c8009800 nic_wr_credit 1. [D 20:01:00.009054] openib_post_rr: da13:3346 bh 50 wr_id 15e73998. [D 20:01:00.010190] openib_check_cq: completed recv wr_id 15e739b0. [D 20:01:00.010217] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaaadec7010 rkey 48003000 nic_wr_credit 2. [D 20:01:00.010245] openib_post_rr: da13:3346 bh 51 wr_id 15e739b0. [D 20:01:00.010490] openib_check_cq: completed recv wr_id 15e739c8. [D 20:01:00.010518] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaaae0c8010 rkey 48003100 nic_wr_credit 3. [D 20:01:00.010544] openib_post_rr: da13:3346 bh 52 wr_id 15e739c8. [D 20:01:00.010789] openib_check_cq: completed recv wr_id 15e739e0. [D 20:01:00.010816] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaaae2c9010 rkey 48003200 nic_wr_credit 4. [D 20:01:00.010844] openib_post_rr: da13:3346 bh 53 wr_id 15e739e0. [D 20:01:00.011084] openib_check_cq: completed recv wr_id 15e739f8. [D 20:01:00.011116] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaaae4ca010 rkey 48003300 nic_wr_credit 5. [D 20:01:00.011143] openib_post_rr: da13:3346 bh 54 wr_id 15e739f8. [D 20:01:00.011379] openib_check_cq: completed recv wr_id 15e73a10. [D 20:01:00.011406] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaaae6cb010 rkey 48003400 nic_wr_credit 6. [D 20:01:00.011433] openib_post_rr: da13:3346 bh 55 wr_id 15e73a10. [D 20:01:00.011479] openib_check_cq: completed rdma wr_id 15f07a30, nic_wr_credit 5. [D 20:01:00.011507] post_sr: da13:3346 bh 8 len 16 wr_id 15e72c40. [D 20:01:00.011745] openib_check_cq: completed recv wr_id 15e73a28. [D 20:01:00.011773] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaaae8cc010 rkey 48003500 nic_wr_credit 6. [D 20:01:00.011799] openib_post_rr: da13:3346 bh 56 wr_id 15e73a28. [D 20:01:00.012042] openib_check_cq: completed recv wr_id 15e73a40. [D 20:01:00.012069] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaaaeacd010 rkey 48003600 nic_wr_credit 7. [D 20:01:00.012096] openib_post_rr: da13:3346 bh 57 wr_id 15e73a40. [D 20:01:00.012351] openib_check_cq: completed recv wr_id 15e73a58. [D 20:01:00.012378] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaaaecce010 rkey 48003700 nic_wr_credit 8. [D 20:01:00.012405] openib_post_rr: da13:3346 bh 58 wr_id 15e73a58. [D 20:01:00.012650] openib_check_cq: completed recv wr_id 15e73a70. [D 20:01:00.012678] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaaaeecf010 rkey c8003800 nic_wr_credit 9. [D 20:01:00.012704] openib_post_rr: da13:3346 bh 59 wr_id 15e73a70. [D 20:01:00.012954] openib_check_cq: completed recv wr_id 15e73a88. [D 20:01:00.012981] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaaaf0d0010 rkey c8003900 nic_wr_credit 10. [D 20:01:00.013009] openib_post_rr: da13:3346 bh 60 wr_id 15e73a88. [D 20:01:00.013270] openib_check_cq: completed recv wr_id 15e73aa0. [D 20:01:00.013300] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaaaf2d1010 rkey c8003a00 nic_wr_credit 11. [D 20:01:00.013328] openib_post_rr: da13:3346 bh 61 wr_id 15e73aa0. [D 20:01:00.013569] openib_check_cq: completed recv wr_id 15e73ab8. [D 20:01:00.013597] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaaaf4d2010 rkey c8003b00 nic_wr_credit 12. [D 20:01:00.013624] openib_post_rr: da13:3346 bh 62 wr_id 15e73ab8. [D 20:01:00.013869] openib_check_cq: completed recv wr_id 15e73ad0. [D 20:01:00.013896] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaaaf6d3010 rkey c8003c00 nic_wr_credit 13. [D 20:01:00.013924] openib_post_rr: da13:3346 bh 63 wr_id 15e73ad0. [D 20:01:00.013950] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 12. [D 20:01:00.013977] post_sr: da13:3346 bh 9 len 16 wr_id 15e72c58. [D 20:01:00.014165] openib_check_cq: completed recv wr_id 15e73ae8. [D 20:01:00.014192] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaaaf8d4010 rkey c8003d00 nic_wr_credit 13. [D 20:01:00.014220] openib_post_rr: da13:3346 bh 64 wr_id 15e73ae8. [D 20:01:00.016389] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 12. [D 20:01:00.016417] post_sr: da13:3346 bh 10 len 16 wr_id 15e72c70. [D 20:01:00.018719] PINT_client_state_machine_test id 22 [D 20:01:00.018839] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 11. [D 20:01:00.018866] post_sr: da13:3346 bh 11 len 16 wr_id 15e72c88. [D 20:01:00.021289] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 10. [D 20:01:00.021318] post_sr: da13:3346 bh 12 len 16 wr_id 15e72ca0. [D 20:01:00.023738] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 9. [D 20:01:00.023766] post_sr: da13:3346 bh 13 len 16 wr_id 15e72cb8. [D 20:01:00.023793] openib_check_cq: completed send wr_id 15e72c40. [D 20:01:00.023823] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:00.023853] ========================================================= [D 20:01:00.023878] PINT_process_request [D 20:01:00.023902] tiling 16 copies [D 20:01:00.023927] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.023954] lvl 0 el 0 blk 0 by 480247808 [D 20:01:00.023979] to 480247808 ta 469762048 fi 503316480 [D 20:01:00.024003] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:00.024028] basic type or contiguous data [D 20:01:00.024053] Request disp [D 20:01:00.024077] contig_offset = 480247808 contig_size = 56623104 lvl_flag = 1 [D 20:01:00.024102] calling distribute [D 20:01:00.024127] PINT_distribute [D 20:01:00.024150] of 480247808 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 480247808 exfl 1 [D 20:01:00.024177] begin iteration loff: 480247808 [D 20:01:00.024203] segment exceeds byte limit [D 20:01:00.024227] file being extended [D 20:01:00.024250] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:00.024275] **********CALL***PROCESS********* [D 20:01:00.024299] segment of 10485760 sz 2097152 [D 20:01:00.024324] PINT_process_request [D 20:01:00.024348] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.024374] lvl 0 el 0 blk 0 by 10485760 [D 20:01:00.024399] to 10485760 ta 10485760 fi 12582912 [D 20:01:00.024424] basic type or contiguous data [D 20:01:00.024448] Request disp [D 20:01:00.024472] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:00.024497] process a segment [D 20:01:00.024520] of 10485760 sz 2097152 [D 20:01:00.024545] add a segment 0 [D 20:01:00.024569] retval = 2097152 [D 20:01:00.024593] exiting distribute returned less than expected [D 20:01:00.024618] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:00.024643] *****RETURN***FROM***PROCESS***** [D 20:01:00.024668] end iteration [D 20:01:00.024692] done with segments or bytes [D 20:01:00.024716] of 482344960 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.024742] next loff: 482344960 next poff: 482344960 [D 20:01:00.024766] return value 2097152 [D 20:01:00.024791] finished [D 20:01:00.024814] retval = 2097152 [D 20:01:00.024839] exiting distribute returned less than expected [D 20:01:00.024863] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 482344960 fo 503316480 eof 0 [D 20:01:00.024889] ========================================================= [D 20:01:00.024914] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:00.024939] element 0: offset: 0xa6e55000, size: 2097152 [D 20:01:00.024968] post_sr: da13:3346 bh 14 len 32 wr_id 15e72cd0. [D 20:01:00.026191] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 8. [D 20:01:00.026219] post_sr: da13:3346 bh 15 len 16 wr_id 15e72ce8. [D 20:01:00.028643] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 7. [D 20:01:00.028671] post_sr: da13:3346 bh 16 len 16 wr_id 15e72d00. [D 20:01:00.028751] PINT_client_state_machine_test id 22 [D 20:01:00.031094] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 6. [D 20:01:00.031123] post_sr: da13:3346 bh 17 len 16 wr_id 15e72d18. [D 20:01:00.033542] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 5. [D 20:01:00.033571] post_sr: da13:3346 bh 18 len 16 wr_id 15e72d30. [D 20:01:00.035992] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 4. [D 20:01:00.036020] post_sr: da13:3346 bh 19 len 16 wr_id 15e72d48. [D 20:01:00.038440] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 3. [D 20:01:00.038469] post_sr: da13:3346 bh 20 len 16 wr_id 15e72d60. [D 20:01:00.038779] PINT_client_state_machine_test id 22 [D 20:01:00.040519] openib_check_cq: completed recv wr_id 15e73b00. [D 20:01:00.040547] openib_post_rr: da13:3346 bh 65 wr_id 15e73b00. [D 20:01:00.040581] sm control op 6 [D 20:01:00.040608] [SM Entering]: (0x15edbf20) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.040634] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.040662] (0x15edbfb8) io_datafile_complete_operations (tag 0) [D 20:01:00.040688] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.040714] - complete_context_send_or_recv called [D 20:01:00.040738] got a recv completion with context index 0 [D 20:01:00.040762] - io_process_context_recv called [D 20:01:00.040787] - io_process_context_recv called [D 20:01:00.040811] PINT_decode [D 20:01:00.040836] lebf_decode_resp [D 20:01:00.040861] PINT_encode_release [D 20:01:00.040885] lebf_encode_rel [D 20:01:00.040910] PINT_decode_release [D 20:01:00.040935] lebf_decode_rel [D 20:01:00.040961] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.040987] io_post_flow: entry [D 20:01:00.041011] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.041037] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.059149] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.059205] bstream_size = 478150656, datafile nr=0, ct=1, file_req_off = 436207616 [D 20:01:00.059236] PINT_new_request_states n=1 [D 20:01:00.059263] PINT_new_request_states n=1 [D 20:01:00.059288] flowproto posting 0x15edcd38 [D 20:01:00.059329] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:00.059355] ========================================================= [D 20:01:00.059380] PINT_process_request [D 20:01:00.059405] tiling 15 copies [D 20:01:00.059429] skipping ahead to target_offset [D 20:01:00.059453] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.059479] lvl 0 el 0 blk 0 by 0 [D 20:01:00.059504] to 0 ta 436207616 fi 469762048 [D 20:01:00.059529] mto 0 mta 0 mfi 33554432 [D 20:01:00.059554] basic type or contiguous data [D 20:01:00.059577] Request disp [D 20:01:00.059601] contig_offset = 0 contig_size = 503316480 lvl_flag = 1 [D 20:01:00.059626] process logical skip [D 20:01:00.059650] retval = 436207616 [D 20:01:00.059674] exiting logical skip because distribute indicates done [D 20:01:00.059699] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.059724] lvl 0 el 0 blk 0 by 436207616 [D 20:01:00.059749] to 436207616 ta 436207616 fi 469762048 [D 20:01:00.059774] mto 0 mta 0 mfi 33554432 [D 20:01:00.059798] basic type or contiguous data [D 20:01:00.059822] Request disp [D 20:01:00.059845] contig_offset = 436207616 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.059870] calling distribute [D 20:01:00.059895] PINT_distribute [D 20:01:00.059919] of 436207616 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:00.059946] begin iteration loff: 436207616 [D 20:01:00.059970] segment exceeds byte limit [D 20:01:00.059995] client lstof 0 diff 0 sgof 0 [D 20:01:00.060019] **********CALL***PROCESS********* [D 20:01:00.060043] segment of 0 sz 2097152 [D 20:01:00.060068] PINT_process_request [D 20:01:00.060091] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.060118] lvl 0 el 0 blk 0 by 0 [D 20:01:00.060143] to 0 ta 0 fi 2097152 [D 20:01:00.060168] basic type or contiguous data [D 20:01:00.060192] Request disp [D 20:01:00.060215] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.060240] process a segment [D 20:01:00.060264] of 0 sz 2097152 [D 20:01:00.060288] add a segment 0 [D 20:01:00.060313] retval = 2097152 [D 20:01:00.060337] exiting distribute returned less than expected [D 20:01:00.060361] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.060387] *****RETURN***FROM***PROCESS***** [D 20:01:00.060412] end iteration [D 20:01:00.060436] done with segments or bytes [D 20:01:00.060460] of 438304768 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.060485] next loff: 438304768 next poff: 438304768 [D 20:01:00.060510] return value 2097152 [D 20:01:00.060535] finished [D 20:01:00.060559] retval = 2097152 [D 20:01:00.060583] exiting distribute returned less than expected [D 20:01:00.060606] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 438304768 fo 469762048 eof 0 [D 20:01:00.060637] ========================================================= [D 20:01:00.060663] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:00.060688] element 0: offset: 0x88437000, size: 2097152 [D 20:01:00.060716] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 2. [D 20:01:00.060744] post_sr: da13:3346 bh 21 len 16 wr_id 15e72d78. [D 20:01:00.060771] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 1. [D 20:01:00.060798] post_sr: da13:3346 bh 22 len 16 wr_id 15e72d90. [D 20:01:00.060824] openib_check_cq: completed send wr_id 15e72c58. [D 20:01:00.060850] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 0. [D 20:01:00.060876] post_sr: da13:3346 bh 23 len 16 wr_id 15e72da8. [D 20:01:00.060902] openib_check_cq: completed send wr_id 15e72c70. [D 20:01:00.060929] openib_check_cq: completed send wr_id 15e72c88. [D 20:01:00.060954] openib_check_cq: completed send wr_id 15e72ca0. [D 20:01:00.060979] openib_check_cq: completed send wr_id 15e72cb8. [D 20:01:00.061004] openib_check_cq: completed send wr_id 15e72cd0. [D 20:01:00.061030] openib_check_cq: completed send wr_id 15e72ce8. [D 20:01:00.061055] openib_check_cq: completed send wr_id 15e72d00. [D 20:01:00.061080] openib_check_cq: completed send wr_id 15e72d18. [D 20:01:00.061105] openib_check_cq: completed send wr_id 15e72d30. [D 20:01:00.061131] openib_check_cq: completed send wr_id 15e72d48. [D 20:01:00.061156] openib_check_cq: completed send wr_id 15e72d60. [D 20:01:00.061184] openib_check_cq: completed recv wr_id 15e73b18. [D 20:01:00.061217] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaabb252010 rkey c8009900 nic_wr_credit 1. [D 20:01:00.061248] openib_post_rr: da13:3346 bh 66 wr_id 15e73b18. [D 20:01:00.061279] openib_check_cq: completed send wr_id 15e72d78. [D 20:01:00.061306] openib_check_cq: completed send wr_id 15e72d90. [D 20:01:00.061332] openib_check_cq: completed send wr_id 15e72da8. [D 20:01:00.061366] flowproto posted 0x15edcd38 [D 20:01:00.061390] Job flows in progress (post time): 16 [D 20:01:00.061420] posted flow for context 0x15edc788 [D 20:01:00.061445] *** all msgpairs complete (all flows posted) [D 20:01:00.061470] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.061495] [SM Exiting]: (0x15edbf20) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.061524] PINT_client_state_machine_test id 22 [D 20:01:00.061560] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:01:00.061587] ========================================================= [D 20:01:00.061611] PINT_process_request [D 20:01:00.061635] tiling 14 copies [D 20:01:00.061660] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.061686] lvl 0 el 0 blk 0 by 421527552 [D 20:01:00.061712] to 421527552 ta 402653184 fi 436207616 [D 20:01:00.061737] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:00.061761] basic type or contiguous data [D 20:01:00.061785] Request disp [D 20:01:00.061810] contig_offset = 421527552 contig_size = 48234496 lvl_flag = 1 [D 20:01:00.061835] calling distribute [D 20:01:00.061859] PINT_distribute [D 20:01:00.061882] of 421527552 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 421527552 exfl 1 [D 20:01:00.061909] begin iteration loff: 421527552 [D 20:01:00.061934] segment exceeds byte limit [D 20:01:00.061959] file being extended [D 20:01:00.061983] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:00.062007] **********CALL***PROCESS********* [D 20:01:00.062031] segment of 18874368 sz 2097152 [D 20:01:00.062056] PINT_process_request [D 20:01:00.062080] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.062106] lvl 0 el 0 blk 0 by 18874368 [D 20:01:00.062130] to 18874368 ta 18874368 fi 20971520 [D 20:01:00.062155] basic type or contiguous data [D 20:01:00.062179] Request disp [D 20:01:00.062204] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:00.062232] process a segment [D 20:01:00.062255] of 18874368 sz 2097152 [D 20:01:00.062280] add a segment 0 [D 20:01:00.062306] retval = 2097152 [D 20:01:00.062331] exiting distribute returned less than expected [D 20:01:00.062355] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:00.062381] *****RETURN***FROM***PROCESS***** [D 20:01:00.062405] end iteration [D 20:01:00.062429] done with segments or bytes [D 20:01:00.062453] of 423624704 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.062479] next loff: 423624704 next poff: 423624704 [D 20:01:00.062503] return value 2097152 [D 20:01:00.062528] finished [D 20:01:00.062552] retval = 2097152 [D 20:01:00.062576] exiting distribute returned less than expected [D 20:01:00.062600] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 423624704 fo 436207616 eof 0 [D 20:01:00.062626] ========================================================= [D 20:01:00.062651] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:01:00.062676] element 0: offset: 0xab659000, size: 2097152 [D 20:01:00.062704] post_sr: da13:3346 bh 24 len 32 wr_id 15e72dc0. [D 20:01:00.062733] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:01:00.062759] ========================================================= [D 20:01:00.062784] PINT_process_request [D 20:01:00.062808] tiling 15 copies [D 20:01:00.062832] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.062858] lvl 0 el 0 blk 0 by 455081984 [D 20:01:00.062883] to 455081984 ta 436207616 fi 469762048 [D 20:01:00.062908] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:00.062933] basic type or contiguous data [D 20:01:00.062957] Request disp [D 20:01:00.062981] contig_offset = 455081984 contig_size = 48234496 lvl_flag = 1 [D 20:01:00.063007] calling distribute [D 20:01:00.063031] PINT_distribute [D 20:01:00.063055] of 455081984 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 455081984 exfl 1 [D 20:01:00.063081] begin iteration loff: 455081984 [D 20:01:00.063106] segment exceeds byte limit [D 20:01:00.063130] file being extended [D 20:01:00.063154] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:00.063181] **********CALL***PROCESS********* [D 20:01:00.063205] segment of 18874368 sz 2097152 [D 20:01:00.063230] PINT_process_request [D 20:01:00.063254] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.063281] lvl 0 el 0 blk 0 by 18874368 [D 20:01:00.063305] to 18874368 ta 18874368 fi 20971520 [D 20:01:00.063330] basic type or contiguous data [D 20:01:00.063354] Request disp [D 20:01:00.063378] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:00.063403] process a segment [D 20:01:00.063427] of 18874368 sz 2097152 [D 20:01:00.063451] add a segment 0 [D 20:01:00.063476] retval = 2097152 [D 20:01:00.063500] exiting distribute returned less than expected [D 20:01:00.063524] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:00.063550] *****RETURN***FROM***PROCESS***** [D 20:01:00.063574] end iteration [D 20:01:00.063598] done with segments or bytes [D 20:01:00.063622] of 457179136 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.063648] next loff: 457179136 next poff: 457179136 [D 20:01:00.063673] return value 2097152 [D 20:01:00.063697] finished [D 20:01:00.063721] retval = 2097152 [D 20:01:00.063745] exiting distribute returned less than expected [D 20:01:00.063769] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 457179136 fo 469762048 eof 0 [D 20:01:00.063796] ========================================================= [D 20:01:00.063819] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:01:00.063845] element 0: offset: 0xa9657000, size: 2097152 [D 20:01:00.063872] post_sr: da13:3346 bh 25 len 32 wr_id 15e72dd8. [D 20:01:00.063900] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:01:00.063929] ========================================================= [D 20:01:00.063954] PINT_process_request [D 20:01:00.063979] tiling 2 copies [D 20:01:00.064002] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.064029] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.064054] to 20971520 ta 0 fi 33554432 [D 20:01:00.064079] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.064104] basic type or contiguous data [D 20:01:00.064128] Request disp [D 20:01:00.064152] contig_offset = 20971520 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.064177] calling distribute [D 20:01:00.064201] PINT_distribute [D 20:01:00.064225] of 20971520 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 20971520 exfl 1 [D 20:01:00.064251] begin iteration loff: 20971520 [D 20:01:00.064276] segment exceeds byte limit [D 20:01:00.064301] file being extended [D 20:01:00.064325] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.064350] **********CALL***PROCESS********* [D 20:01:00.064373] segment of 20971520 sz 2097152 [D 20:01:00.064398] PINT_process_request [D 20:01:00.064422] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.064448] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.064473] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.064497] basic type or contiguous data [D 20:01:00.064521] Request disp [D 20:01:00.064545] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.064570] process a segment [D 20:01:00.064594] of 20971520 sz 2097152 [D 20:01:00.064619] add a segment 0 [D 20:01:00.064642] retval = 2097152 [D 20:01:00.064666] exiting distribute returned less than expected [D 20:01:00.064690] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.064717] *****RETURN***FROM***PROCESS***** [D 20:01:00.064741] end iteration [D 20:01:00.064764] done with segments or bytes [D 20:01:00.064788] of 23068672 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.064814] next loff: 23068672 next poff: 23068672 [D 20:01:00.064839] return value 2097152 [D 20:01:00.064864] finished [D 20:01:00.064888] retval = 2097152 [D 20:01:00.064911] exiting distribute returned less than expected [D 20:01:00.064935] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 23068672 fo 33554432 eof 0 [D 20:01:00.064961] ========================================================= [D 20:01:00.064986] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:01:00.065011] element 0: offset: 0xc3871000, size: 2097152 [D 20:01:00.065037] post_sr: da13:3346 bh 26 len 32 wr_id 15e72df0. [D 20:01:00.065067] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:01:00.065093] ========================================================= [D 20:01:00.065117] PINT_process_request [D 20:01:00.065142] tiling 3 copies [D 20:01:00.065166] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.065196] lvl 0 el 0 blk 0 by 54525952 [D 20:01:00.065225] to 54525952 ta 33554432 fi 67108864 [D 20:01:00.065254] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.065283] basic type or contiguous data [D 20:01:00.065308] Request disp [D 20:01:00.065332] contig_offset = 54525952 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.065357] calling distribute [D 20:01:00.065381] PINT_distribute [D 20:01:00.065404] of 54525952 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 54525952 exfl 1 [D 20:01:00.065431] begin iteration loff: 54525952 [D 20:01:00.065456] segment exceeds byte limit [D 20:01:00.065480] file being extended [D 20:01:00.065504] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.065528] **********CALL***PROCESS********* [D 20:01:00.065552] segment of 20971520 sz 2097152 [D 20:01:00.065577] PINT_process_request [D 20:01:00.065601] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.065627] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.065651] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.065679] basic type or contiguous data [D 20:01:00.065703] Request disp [D 20:01:00.065727] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.065752] process a segment [D 20:01:00.065776] of 20971520 sz 2097152 [D 20:01:00.065800] add a segment 0 [D 20:01:00.065824] retval = 2097152 [D 20:01:00.065848] exiting distribute returned less than expected [D 20:01:00.065873] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.065898] *****RETURN***FROM***PROCESS***** [D 20:01:00.065923] end iteration [D 20:01:00.065947] done with segments or bytes [D 20:01:00.065971] of 56623104 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.065997] next loff: 56623104 next poff: 56623104 [D 20:01:00.066021] return value 2097152 [D 20:01:00.066046] finished [D 20:01:00.066069] retval = 2097152 [D 20:01:00.066094] exiting distribute returned less than expected [D 20:01:00.066118] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 56623104 fo 67108864 eof 0 [D 20:01:00.066143] ========================================================= [D 20:01:00.066168] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:01:00.066194] element 0: offset: 0xc186f000, size: 2097152 [D 20:01:00.066221] post_sr: da13:3346 bh 27 len 32 wr_id 15e72e08. [D 20:01:00.066250] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:01:00.066276] ========================================================= [D 20:01:00.066301] PINT_process_request [D 20:01:00.066325] tiling 4 copies [D 20:01:00.066349] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.066375] lvl 0 el 0 blk 0 by 88080384 [D 20:01:00.066400] to 88080384 ta 67108864 fi 100663296 [D 20:01:00.066426] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.066451] basic type or contiguous data [D 20:01:00.066474] Request disp [D 20:01:00.066498] contig_offset = 88080384 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.066523] calling distribute [D 20:01:00.066548] PINT_distribute [D 20:01:00.066571] of 88080384 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 88080384 exfl 1 [D 20:01:00.066597] begin iteration loff: 88080384 [D 20:01:00.066623] segment exceeds byte limit [D 20:01:00.066647] file being extended [D 20:01:00.066671] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.066696] **********CALL***PROCESS********* [D 20:01:00.066720] segment of 20971520 sz 2097152 [D 20:01:00.066744] PINT_process_request [D 20:01:00.066768] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.066794] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.066819] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.066843] basic type or contiguous data [D 20:01:00.066867] Request disp [D 20:01:00.066892] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.066917] process a segment [D 20:01:00.066941] of 20971520 sz 2097152 [D 20:01:00.066965] add a segment 0 [D 20:01:00.066988] retval = 2097152 [D 20:01:00.067012] exiting distribute returned less than expected [D 20:01:00.067036] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.067063] *****RETURN***FROM***PROCESS***** [D 20:01:00.067087] end iteration [D 20:01:00.067111] done with segments or bytes [D 20:01:00.067134] of 90177536 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.067160] next loff: 90177536 next poff: 90177536 [D 20:01:00.067188] return value 2097152 [D 20:01:00.067213] finished [D 20:01:00.067236] retval = 2097152 [D 20:01:00.067260] exiting distribute returned less than expected [D 20:01:00.067285] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 90177536 fo 100663296 eof 0 [D 20:01:00.067311] ========================================================= [D 20:01:00.067336] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:01:00.067360] element 0: offset: 0xbf86d000, size: 2097152 [D 20:01:00.067390] post_sr: da13:3346 bh 28 len 32 wr_id 15e72e20. [D 20:01:00.067421] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 0. [D 20:01:00.067449] post_sr: da13:3346 bh 29 len 16 wr_id 15e72e38. [D 20:01:00.067475] openib_check_cq: completed send wr_id 15e72dc0. [D 20:01:00.067501] openib_check_cq: completed send wr_id 15e72dd8. [D 20:01:00.067526] openib_check_cq: completed send wr_id 15e72df0. [D 20:01:00.067551] openib_check_cq: completed send wr_id 15e72e08. [D 20:01:00.067577] openib_check_cq: completed send wr_id 15e72e20. [D 20:01:00.067602] openib_check_cq: completed send wr_id 15e72e38. [D 20:01:00.067634] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:01:00.067660] ========================================================= [D 20:01:00.067685] PINT_process_request [D 20:01:00.067708] tiling 5 copies [D 20:01:00.067733] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.067760] lvl 0 el 0 blk 0 by 121634816 [D 20:01:00.067785] to 121634816 ta 100663296 fi 134217728 [D 20:01:00.067810] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.067835] basic type or contiguous data [D 20:01:00.067859] Request disp [D 20:01:00.067883] contig_offset = 121634816 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.067909] calling distribute [D 20:01:00.067933] PINT_distribute [D 20:01:00.067956] of 121634816 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 121634816 exfl 1 [D 20:01:00.067983] begin iteration loff: 121634816 [D 20:01:00.068008] segment exceeds byte limit [D 20:01:00.068033] file being extended [D 20:01:00.068057] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.068081] **********CALL***PROCESS********* [D 20:01:00.068105] segment of 20971520 sz 2097152 [D 20:01:00.068129] PINT_process_request [D 20:01:00.068153] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.068180] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.068204] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.068229] basic type or contiguous data [D 20:01:00.068253] Request disp [D 20:01:00.068277] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.068302] process a segment [D 20:01:00.068325] of 20971520 sz 2097152 [D 20:01:00.068349] add a segment 0 [D 20:01:00.068374] retval = 2097152 [D 20:01:00.068398] exiting distribute returned less than expected [D 20:01:00.068422] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.068449] *****RETURN***FROM***PROCESS***** [D 20:01:00.068473] end iteration [D 20:01:00.068497] done with segments or bytes [D 20:01:00.068521] of 123731968 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.068547] next loff: 123731968 next poff: 123731968 [D 20:01:00.068572] return value 2097152 [D 20:01:00.068595] finished [D 20:01:00.068619] retval = 2097152 [D 20:01:00.068643] exiting distribute returned less than expected [D 20:01:00.068668] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 123731968 fo 134217728 eof 0 [D 20:01:00.068695] ========================================================= [D 20:01:00.068718] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:01:00.068744] element 0: offset: 0xbd86b000, size: 2097152 [D 20:01:00.068771] post_sr: da13:3346 bh 30 len 32 wr_id 15e72e50. [D 20:01:00.068800] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:01:00.068826] ========================================================= [D 20:01:00.068851] PINT_process_request [D 20:01:00.068875] tiling 6 copies [D 20:01:00.068899] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.068925] lvl 0 el 0 blk 0 by 155189248 [D 20:01:00.068950] to 155189248 ta 134217728 fi 167772160 [D 20:01:00.068975] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.069000] basic type or contiguous data [D 20:01:00.069025] Request disp [D 20:01:00.069048] contig_offset = 155189248 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.069076] calling distribute [D 20:01:00.069101] PINT_distribute [D 20:01:00.069125] of 155189248 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 155189248 exfl 1 [D 20:01:00.069151] begin iteration loff: 155189248 [D 20:01:00.069176] segment exceeds byte limit [D 20:01:00.069206] file being extended [D 20:01:00.069233] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.069261] **********CALL***PROCESS********* [D 20:01:00.069289] segment of 20971520 sz 2097152 [D 20:01:00.069314] PINT_process_request [D 20:01:00.069338] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.069364] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.069389] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.069414] basic type or contiguous data [D 20:01:00.069438] Request disp [D 20:01:00.069462] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.069487] process a segment [D 20:01:00.069511] of 20971520 sz 2097152 [D 20:01:00.069536] add a segment 0 [D 20:01:00.069560] retval = 2097152 [D 20:01:00.069583] exiting distribute returned less than expected [D 20:01:00.069607] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.069633] *****RETURN***FROM***PROCESS***** [D 20:01:00.069658] end iteration [D 20:01:00.069682] done with segments or bytes [D 20:01:00.069705] of 157286400 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.069731] next loff: 157286400 next poff: 157286400 [D 20:01:00.069756] return value 2097152 [D 20:01:00.069780] finished [D 20:01:00.069804] retval = 2097152 [D 20:01:00.069828] exiting distribute returned less than expected [D 20:01:00.069851] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 157286400 fo 167772160 eof 0 [D 20:01:00.069878] ========================================================= [D 20:01:00.069902] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:01:00.069928] element 0: offset: 0xbb869000, size: 2097152 [D 20:01:00.069954] post_sr: da13:3346 bh 31 len 32 wr_id 15e72e68. [D 20:01:00.069984] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:01:00.070010] ========================================================= [D 20:01:00.070034] PINT_process_request [D 20:01:00.070058] tiling 7 copies [D 20:01:00.070082] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.070109] lvl 0 el 0 blk 0 by 188743680 [D 20:01:00.070134] to 188743680 ta 167772160 fi 201326592 [D 20:01:00.070158] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.070183] basic type or contiguous data [D 20:01:00.070207] Request disp [D 20:01:00.070232] contig_offset = 188743680 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.070257] calling distribute [D 20:01:00.070281] PINT_distribute [D 20:01:00.070304] of 188743680 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 188743680 exfl 1 [D 20:01:00.070330] begin iteration loff: 188743680 [D 20:01:00.070356] segment exceeds byte limit [D 20:01:00.070380] file being extended [D 20:01:00.070404] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.070428] **********CALL***PROCESS********* [D 20:01:00.070452] segment of 20971520 sz 2097152 [D 20:01:00.070476] PINT_process_request [D 20:01:00.070501] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.070527] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.070551] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.070576] basic type or contiguous data [D 20:01:00.070600] Request disp [D 20:01:00.070624] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.070649] process a segment [D 20:01:00.070672] of 20971520 sz 2097152 [D 20:01:00.070697] add a segment 0 [D 20:01:00.070721] retval = 2097152 [D 20:01:00.070745] exiting distribute returned less than expected [D 20:01:00.070769] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.070796] *****RETURN***FROM***PROCESS***** [D 20:01:00.070819] end iteration [D 20:01:00.070846] done with segments or bytes [D 20:01:00.070870] of 190840832 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.070896] next loff: 190840832 next poff: 190840832 [D 20:01:00.070920] return value 2097152 [D 20:01:00.070945] finished [D 20:01:00.070969] retval = 2097152 [D 20:01:00.070993] exiting distribute returned less than expected [D 20:01:00.071017] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 190840832 fo 201326592 eof 0 [D 20:01:00.071043] ========================================================= [D 20:01:00.071067] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:01:00.071093] element 0: offset: 0xb9867000, size: 2097152 [D 20:01:00.071120] post_sr: da13:3346 bh 32 len 32 wr_id 15e72e80. [D 20:01:00.071148] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:01:00.071175] ========================================================= [D 20:01:00.071202] PINT_process_request [D 20:01:00.071226] tiling 8 copies [D 20:01:00.071250] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.071276] lvl 0 el 0 blk 0 by 222298112 [D 20:01:00.071301] to 222298112 ta 201326592 fi 234881024 [D 20:01:00.071326] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.071352] basic type or contiguous data [D 20:01:00.071375] Request disp [D 20:01:00.071399] contig_offset = 222298112 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.071424] calling distribute [D 20:01:00.071448] PINT_distribute [D 20:01:00.071472] of 222298112 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 222298112 exfl 1 [D 20:01:00.071497] begin iteration loff: 222298112 [D 20:01:00.071522] segment exceeds byte limit [D 20:01:00.071546] file being extended [D 20:01:00.071570] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.071595] **********CALL***PROCESS********* [D 20:01:00.071619] segment of 20971520 sz 2097152 [D 20:01:00.071643] PINT_process_request [D 20:01:00.071667] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.071694] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.071719] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.071743] basic type or contiguous data [D 20:01:00.071767] Request disp [D 20:01:00.071792] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.071817] process a segment [D 20:01:00.071841] of 20971520 sz 2097152 [D 20:01:00.071866] add a segment 0 [D 20:01:00.071889] retval = 2097152 [D 20:01:00.071913] exiting distribute returned less than expected [D 20:01:00.071937] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.071964] *****RETURN***FROM***PROCESS***** [D 20:01:00.071989] end iteration [D 20:01:00.072012] done with segments or bytes [D 20:01:00.072035] of 224395264 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.072061] next loff: 224395264 next poff: 224395264 [D 20:01:00.072086] return value 2097152 [D 20:01:00.072111] finished [D 20:01:00.072134] retval = 2097152 [D 20:01:00.072158] exiting distribute returned less than expected [D 20:01:00.072182] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 224395264 fo 234881024 eof 0 [D 20:01:00.072209] ========================================================= [D 20:01:00.072233] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:01:00.072258] element 0: offset: 0xb7865000, size: 2097152 [D 20:01:00.072285] post_sr: da13:3346 bh 33 len 32 wr_id 15e72e98. [D 20:01:00.072317] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:01:00.072342] ========================================================= [D 20:01:00.072367] PINT_process_request [D 20:01:00.072391] tiling 9 copies [D 20:01:00.072416] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.072442] lvl 0 el 0 blk 0 by 255852544 [D 20:01:00.072466] to 255852544 ta 234881024 fi 268435456 [D 20:01:00.072492] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.072520] basic type or contiguous data [D 20:01:00.072544] Request disp [D 20:01:00.072568] contig_offset = 255852544 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.072593] calling distribute [D 20:01:00.072618] PINT_distribute [D 20:01:00.072641] of 255852544 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 255852544 exfl 1 [D 20:01:00.072668] begin iteration loff: 255852544 [D 20:01:00.072692] segment exceeds byte limit [D 20:01:00.072717] file being extended [D 20:01:00.072741] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.072766] **********CALL***PROCESS********* [D 20:01:00.072790] segment of 20971520 sz 2097152 [D 20:01:00.072814] PINT_process_request [D 20:01:00.072838] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.072865] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.072890] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.072915] basic type or contiguous data [D 20:01:00.072938] Request disp [D 20:01:00.072962] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.072987] process a segment [D 20:01:00.073011] of 20971520 sz 2097152 [D 20:01:00.073036] add a segment 0 [D 20:01:00.073060] retval = 2097152 [D 20:01:00.073083] exiting distribute returned less than expected [D 20:01:00.073108] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.073134] *****RETURN***FROM***PROCESS***** [D 20:01:00.073159] end iteration [D 20:01:00.073185] done with segments or bytes [D 20:01:00.073213] of 257949696 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.073243] next loff: 257949696 next poff: 257949696 [D 20:01:00.073271] return value 2097152 [D 20:01:00.073299] finished [D 20:01:00.073323] retval = 2097152 [D 20:01:00.073347] exiting distribute returned less than expected [D 20:01:00.073371] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 257949696 fo 268435456 eof 0 [D 20:01:00.073397] ========================================================= [D 20:01:00.073422] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:01:00.073448] element 0: offset: 0xb5863000, size: 2097152 [D 20:01:00.073475] post_sr: da13:3346 bh 34 len 32 wr_id 15e72eb0. [D 20:01:00.073505] PINT_client_state_machine_test id 22 [D 20:01:00.073534] openib_check_cq: completed send wr_id 15e72e50. [D 20:01:00.073560] openib_check_cq: completed send wr_id 15e72e68. [D 20:01:00.073586] openib_check_cq: completed send wr_id 15e72e80. [D 20:01:00.073612] openib_check_cq: completed send wr_id 15e72e98. [D 20:01:00.073637] openib_check_cq: completed send wr_id 15e72eb0. [D 20:01:00.073669] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:01:00.073695] ========================================================= [D 20:01:00.073719] PINT_process_request [D 20:01:00.073743] tiling 10 copies [D 20:01:00.073767] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.073794] lvl 0 el 0 blk 0 by 289406976 [D 20:01:00.073819] to 289406976 ta 268435456 fi 301989888 [D 20:01:00.073843] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.073868] basic type or contiguous data [D 20:01:00.073893] Request disp [D 20:01:00.073917] contig_offset = 289406976 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.073942] calling distribute [D 20:01:00.073966] PINT_distribute [D 20:01:00.073990] of 289406976 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 289406976 exfl 1 [D 20:01:00.074017] begin iteration loff: 289406976 [D 20:01:00.074042] segment exceeds byte limit [D 20:01:00.074066] file being extended [D 20:01:00.074090] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.074115] **********CALL***PROCESS********* [D 20:01:00.074139] segment of 20971520 sz 2097152 [D 20:01:00.074164] PINT_process_request [D 20:01:00.074188] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.074214] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.074239] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.074267] basic type or contiguous data [D 20:01:00.074291] Request disp [D 20:01:00.074315] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.074340] process a segment [D 20:01:00.074364] of 20971520 sz 2097152 [D 20:01:00.074389] add a segment 0 [D 20:01:00.074413] retval = 2097152 [D 20:01:00.074437] exiting distribute returned less than expected [D 20:01:00.074460] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.074487] *****RETURN***FROM***PROCESS***** [D 20:01:00.074512] end iteration [D 20:01:00.074536] done with segments or bytes [D 20:01:00.074560] of 291504128 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.074585] next loff: 291504128 next poff: 291504128 [D 20:01:00.074610] return value 2097152 [D 20:01:00.074635] finished [D 20:01:00.074659] retval = 2097152 [D 20:01:00.074683] exiting distribute returned less than expected [D 20:01:00.074707] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 291504128 fo 301989888 eof 0 [D 20:01:00.074732] ========================================================= [D 20:01:00.074757] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:01:00.074783] element 0: offset: 0xb3861000, size: 2097152 [D 20:01:00.074810] post_sr: da13:3346 bh 35 len 32 wr_id 15e72ec8. [D 20:01:00.074838] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:01:00.074865] ========================================================= [D 20:01:00.074889] PINT_process_request [D 20:01:00.074913] tiling 11 copies [D 20:01:00.074937] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.074964] lvl 0 el 0 blk 0 by 322961408 [D 20:01:00.074989] to 322961408 ta 301989888 fi 335544320 [D 20:01:00.075014] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.075038] basic type or contiguous data [D 20:01:00.075062] Request disp [D 20:01:00.075086] contig_offset = 322961408 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.075111] calling distribute [D 20:01:00.075135] PINT_distribute [D 20:01:00.075159] of 322961408 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 322961408 exfl 1 [D 20:01:00.075187] begin iteration loff: 322961408 [D 20:01:00.075213] segment exceeds byte limit [D 20:01:00.075237] file being extended [D 20:01:00.075261] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.075285] **********CALL***PROCESS********* [D 20:01:00.075309] segment of 20971520 sz 2097152 [D 20:01:00.075334] PINT_process_request [D 20:01:00.075358] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.075385] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.075409] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.075434] basic type or contiguous data [D 20:01:00.075458] Request disp [D 20:01:00.075482] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.075508] process a segment [D 20:01:00.075531] of 20971520 sz 2097152 [D 20:01:00.075555] add a segment 0 [D 20:01:00.075579] retval = 2097152 [D 20:01:00.075603] exiting distribute returned less than expected [D 20:01:00.075627] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.075654] *****RETURN***FROM***PROCESS***** [D 20:01:00.075678] end iteration [D 20:01:00.075702] done with segments or bytes [D 20:01:00.075725] of 325058560 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.075752] next loff: 325058560 next poff: 325058560 [D 20:01:00.075777] return value 2097152 [D 20:01:00.075800] finished [D 20:01:00.075824] retval = 2097152 [D 20:01:00.075848] exiting distribute returned less than expected [D 20:01:00.075872] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 325058560 fo 335544320 eof 0 [D 20:01:00.075899] ========================================================= [D 20:01:00.075922] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:01:00.075948] element 0: offset: 0xb185f000, size: 2097152 [D 20:01:00.075982] post_sr: da13:3346 bh 36 len 32 wr_id 15e72ee0. [D 20:01:00.076011] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:01:00.076037] ========================================================= [D 20:01:00.076061] PINT_process_request [D 20:01:00.076086] tiling 12 copies [D 20:01:00.076109] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.076136] lvl 0 el 0 blk 0 by 356515840 [D 20:01:00.076161] to 356515840 ta 335544320 fi 369098752 [D 20:01:00.076186] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.076210] basic type or contiguous data [D 20:01:00.076234] Request disp [D 20:01:00.076258] contig_offset = 356515840 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.076284] calling distribute [D 20:01:00.076308] PINT_distribute [D 20:01:00.076332] of 356515840 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 356515840 exfl 1 [D 20:01:00.076358] begin iteration loff: 356515840 [D 20:01:00.076383] segment exceeds byte limit [D 20:01:00.076407] file being extended [D 20:01:00.076431] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.076455] **********CALL***PROCESS********* [D 20:01:00.076478] segment of 20971520 sz 2097152 [D 20:01:00.076503] PINT_process_request [D 20:01:00.076527] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.076554] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.076579] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.076603] basic type or contiguous data [D 20:01:00.076627] Request disp [D 20:01:00.076650] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.076676] process a segment [D 20:01:00.076700] of 20971520 sz 2097152 [D 20:01:00.076724] add a segment 0 [D 20:01:00.076748] retval = 2097152 [D 20:01:00.076772] exiting distribute returned less than expected [D 20:01:00.076796] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.076822] *****RETURN***FROM***PROCESS***** [D 20:01:00.076847] end iteration [D 20:01:00.076870] done with segments or bytes [D 20:01:00.076894] of 358612992 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.076920] next loff: 358612992 next poff: 358612992 [D 20:01:00.076945] return value 2097152 [D 20:01:00.076969] finished [D 20:01:00.076992] retval = 2097152 [D 20:01:00.077016] exiting distribute returned less than expected [D 20:01:00.077041] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 358612992 fo 369098752 eof 0 [D 20:01:00.077067] ========================================================= [D 20:01:00.077092] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:01:00.077116] element 0: offset: 0xaf85d000, size: 2097152 [D 20:01:00.077144] post_sr: da13:3346 bh 37 len 32 wr_id 15e72ef8. [D 20:01:00.077173] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:01:00.077203] ========================================================= [D 20:01:00.077231] PINT_process_request [D 20:01:00.077259] tiling 13 copies [D 20:01:00.077287] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.077314] lvl 0 el 0 blk 0 by 390070272 [D 20:01:00.077339] to 390070272 ta 369098752 fi 402653184 [D 20:01:00.077363] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.077388] basic type or contiguous data [D 20:01:00.077412] Request disp [D 20:01:00.077437] contig_offset = 390070272 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.077462] calling distribute [D 20:01:00.077486] PINT_distribute [D 20:01:00.077509] of 390070272 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 390070272 exfl 1 [D 20:01:00.077536] begin iteration loff: 390070272 [D 20:01:00.077561] segment exceeds byte limit [D 20:01:00.077585] file being extended [D 20:01:00.077609] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.077633] **********CALL***PROCESS********* [D 20:01:00.077657] segment of 20971520 sz 2097152 [D 20:01:00.077682] PINT_process_request [D 20:01:00.077706] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.077735] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.077760] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.077785] basic type or contiguous data [D 20:01:00.077809] Request disp [D 20:01:00.077834] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.077858] process a segment [D 20:01:00.077882] of 20971520 sz 2097152 [D 20:01:00.077906] add a segment 0 [D 20:01:00.077931] retval = 2097152 [D 20:01:00.077955] exiting distribute returned less than expected [D 20:01:00.077979] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.078005] *****RETURN***FROM***PROCESS***** [D 20:01:00.078029] end iteration [D 20:01:00.078053] done with segments or bytes [D 20:01:00.078077] of 392167424 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.078103] next loff: 392167424 next poff: 392167424 [D 20:01:00.078128] return value 2097152 [D 20:01:00.078152] finished [D 20:01:00.078176] retval = 2097152 [D 20:01:00.078200] exiting distribute returned less than expected [D 20:01:00.078225] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 392167424 fo 402653184 eof 0 [D 20:01:00.078250] ========================================================= [D 20:01:00.078275] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:01:00.078300] element 0: offset: 0xad85b000, size: 2097152 [D 20:01:00.078328] post_sr: da13:3346 bh 38 len 32 wr_id 15e72f10. [D 20:01:00.078356] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:00.078382] ========================================================= [D 20:01:00.078407] PINT_process_request [D 20:01:00.078431] tiling 16 copies [D 20:01:00.078454] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.078481] lvl 0 el 0 blk 0 by 482344960 [D 20:01:00.078506] to 482344960 ta 469762048 fi 503316480 [D 20:01:00.078531] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:00.078556] basic type or contiguous data [D 20:01:00.078580] Request disp [D 20:01:00.078604] contig_offset = 482344960 contig_size = 54525952 lvl_flag = 1 [D 20:01:00.078629] calling distribute [D 20:01:00.078653] PINT_distribute [D 20:01:00.078677] of 482344960 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 482344960 exfl 1 [D 20:01:00.078703] begin iteration loff: 482344960 [D 20:01:00.078729] segment exceeds byte limit [D 20:01:00.078753] file being extended [D 20:01:00.078777] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:00.078802] **********CALL***PROCESS********* [D 20:01:00.078825] segment of 12582912 sz 2097152 [D 20:01:00.078850] PINT_process_request [D 20:01:00.078874] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.078901] lvl 0 el 0 blk 0 by 12582912 [D 20:01:00.078926] to 12582912 ta 12582912 fi 14680064 [D 20:01:00.078949] basic type or contiguous data [D 20:01:00.078974] Request disp [D 20:01:00.078998] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:00.079023] process a segment [D 20:01:00.079047] of 12582912 sz 2097152 [D 20:01:00.079071] add a segment 0 [D 20:01:00.079094] retval = 2097152 [D 20:01:00.079119] exiting distribute returned less than expected [D 20:01:00.079143] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:00.079169] *****RETURN***FROM***PROCESS***** [D 20:01:00.079195] end iteration [D 20:01:00.079219] done with segments or bytes [D 20:01:00.079243] of 484442112 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.079270] next loff: 484442112 next poff: 484442112 [D 20:01:00.079295] return value 2097152 [D 20:01:00.079319] finished [D 20:01:00.079342] retval = 2097152 [D 20:01:00.079367] exiting distribute returned less than expected [D 20:01:00.079391] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 484442112 fo 503316480 eof 0 [D 20:01:00.079417] ========================================================= [D 20:01:00.079444] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:00.079470] element 0: offset: 0xa7055000, size: 2097152 [D 20:01:00.079497] post_sr: da13:3346 bh 39 len 32 wr_id 15e72f28. [D 20:01:00.079528] openib_check_cq: completed send wr_id 15e72ec8. [D 20:01:00.079554] openib_check_cq: completed send wr_id 15e72ee0. [D 20:01:00.079580] openib_check_cq: completed send wr_id 15e72ef8. [D 20:01:00.079605] openib_check_cq: completed send wr_id 15e72f10. [D 20:01:00.079630] openib_check_cq: completed send wr_id 15e72f28. [D 20:01:00.083537] PINT_client_state_machine_test id 22 [D 20:01:00.093567] PINT_client_state_machine_test id 22 [D 20:01:00.103597] PINT_client_state_machine_test id 22 [D 20:01:00.113629] PINT_client_state_machine_test id 22 [D 20:01:00.123661] PINT_client_state_machine_test id 22 -[D 20:01:00.125445] openib_check_cq: completed recv wr_id 15e73b30. [D 20:01:00.125474] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaaafad5010 rkey c8003e00 nic_wr_credit 1. [D 20:01:00.125502] openib_post_rr: da13:3346 bh 67 wr_id 15e73b30. [D 20:01:00.125529] openib_check_cq: completed recv wr_id 15e73b48. [D 20:01:00.125555] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaaafcd6010 rkey c8003f00 nic_wr_credit 2. [D 20:01:00.125581] openib_post_rr: da13:3346 bh 68 wr_id 15e73b48. [D 20:01:00.125691] openib_check_cq: completed recv wr_id 15e73b60. [D 20:01:00.125718] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaaafed7010 rkey 48004000 nic_wr_credit 3. [D 20:01:00.125744] openib_post_rr: da13:3346 bh 69 wr_id 15e73b60. [D 20:01:00.126002] openib_check_cq: completed recv wr_id 15e73b78. [D 20:01:00.126029] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab00d8010 rkey 48004100 nic_wr_credit 4. [D 20:01:00.126055] openib_post_rr: da13:3346 bh 70 wr_id 15e73b78. [D 20:01:00.126450] openib_check_cq: completed recv wr_id 15e73b90. [D 20:01:00.126477] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaab02d9010 rkey 48004200 nic_wr_credit 5. [D 20:01:00.126504] openib_post_rr: da13:3346 bh 71 wr_id 15e73b90. [D 20:01:00.126749] openib_check_cq: completed recv wr_id 15e73ba8. [D 20:01:00.126775] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab04da010 rkey 48004300 nic_wr_credit 6. [D 20:01:00.126803] openib_post_rr: da13:3346 bh 72 wr_id 15e73ba8. [D 20:01:00.127077] openib_check_cq: completed recv wr_id 15e73bc0. [D 20:01:00.127103] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab06db010 rkey 48004400 nic_wr_credit 7. [D 20:01:00.127130] openib_post_rr: da13:3346 bh 73 wr_id 15e73bc0. [D 20:01:00.127538] openib_check_cq: completed recv wr_id 15e73bd8. [D 20:01:00.127567] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab08dc010 rkey 48004500 nic_wr_credit 8. [D 20:01:00.127593] openib_post_rr: da13:3346 bh 74 wr_id 15e73bd8. [D 20:01:00.127838] openib_check_cq: completed recv wr_id 15e73bf0. [D 20:01:00.127864] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab0add010 rkey 48004600 nic_wr_credit 9. [D 20:01:00.127892] openib_post_rr: da13:3346 bh 75 wr_id 15e73bf0. [D 20:01:00.127967] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 8. [D 20:01:00.127994] post_sr: da13:3346 bh 40 len 16 wr_id 15e72f40. [D 20:01:00.128299] openib_check_cq: completed recv wr_id 15e73c08. [D 20:01:00.128327] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaab0cde010 rkey 48004700 nic_wr_credit 9. [D 20:01:00.128354] openib_post_rr: da13:3346 bh 76 wr_id 15e73c08. [D 20:01:00.128693] openib_check_cq: completed recv wr_id 15e73c20. [D 20:01:00.128720] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab0edf010 rkey c8004800 nic_wr_credit 10. [D 20:01:00.128746] openib_post_rr: da13:3346 bh 77 wr_id 15e73c20. [D 20:01:00.129085] openib_check_cq: completed recv wr_id 15e73c38. [D 20:01:00.129117] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab10e0010 rkey c8004900 nic_wr_credit 11. [D 20:01:00.129143] openib_post_rr: da13:3346 bh 78 wr_id 15e73c38. [D 20:01:00.129381] openib_check_cq: completed recv wr_id 15e73c50. [D 20:01:00.129409] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaab12e1010 rkey c8004a00 nic_wr_credit 12. [D 20:01:00.129436] openib_post_rr: da13:3346 bh 79 wr_id 15e73c50. [D 20:01:00.129680] openib_check_cq: completed recv wr_id 15e73c68. [D 20:01:00.129707] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaab1600010 rkey c8004b00 nic_wr_credit 13. [D 20:01:00.129734] openib_post_rr: da13:3346 bh 80 wr_id 15e73c68. [D 20:01:00.130424] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 12. [D 20:01:00.130452] post_sr: da13:3346 bh 41 len 16 wr_id 15e72f58. [D 20:01:00.132874] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 11. [D 20:01:00.132903] post_sr: da13:3346 bh 42 len 16 wr_id 15e72f70. [D 20:01:00.133166] openib_check_cq: completed recv wr_id 15e73c80. [D 20:01:00.133196] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaabb453010 rkey c8009a00 nic_wr_credit 12. [D 20:01:00.133224] openib_post_rr: da13:3346 bh 81 wr_id 15e73c80. [D 20:01:00.133269] openib_check_cq: completed recv wr_id 15e73c98. [D 20:01:00.133296] openib_post_rr: da13:3346 bh 82 wr_id 15e73c98. [D 20:01:00.133330] sm control op 6 [D 20:01:00.133355] [SM Entering]: (0x15eb1140) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.133381] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.133406] (0x15eb11d8) io_datafile_complete_operations (tag 0) [D 20:01:00.133431] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.133457] - complete_context_send_or_recv called [D 20:01:00.133481] got a recv completion with context index 0 [D 20:01:00.133505] - io_process_context_recv called [D 20:01:00.133529] - io_process_context_recv called [D 20:01:00.133554] PINT_decode [D 20:01:00.133579] lebf_decode_resp [D 20:01:00.133603] PINT_encode_release [D 20:01:00.133627] lebf_encode_rel [D 20:01:00.133652] PINT_decode_release [D 20:01:00.133676] lebf_decode_rel [D 20:01:00.133702] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.133728] io_post_flow: entry [D 20:01:00.133752] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.133777] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.155548] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.155605] bstream_size = 440401920, datafile nr=0, ct=1, file_req_off = 0 [D 20:01:00.155636] PINT_new_request_states n=1 [D 20:01:00.155664] PINT_new_request_states n=1 [D 20:01:00.155689] flowproto posting 0x15eb1f38 [D 20:01:00.155728] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:00.155755] ========================================================= [D 20:01:00.155779] PINT_process_request [D 20:01:00.155804] tiling 2 copies [D 20:01:00.155828] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.155855] lvl 0 el 0 blk 0 by 0 [D 20:01:00.155879] to 0 ta 0 fi 33554432 [D 20:01:00.155904] mto 0 mta 0 mfi 33554432 [D 20:01:00.155929] basic type or contiguous data [D 20:01:00.155953] Request disp [D 20:01:00.155977] contig_offset = 0 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.156002] calling distribute [D 20:01:00.156026] PINT_distribute [D 20:01:00.156050] of 0 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:00.156078] begin iteration loff: 0 [D 20:01:00.156104] segment exceeds byte limit [D 20:01:00.156127] client lstof 0 diff 0 sgof 0 [D 20:01:00.156152] **********CALL***PROCESS********* [D 20:01:00.156176] segment of 0 sz 2097152 [D 20:01:00.156201] PINT_process_request [D 20:01:00.156225] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.156255] lvl 0 el 0 blk 0 by 0 [D 20:01:00.156280] to 0 ta 0 fi 2097152 [D 20:01:00.156305] basic type or contiguous data [D 20:01:00.156329] Request disp [D 20:01:00.156352] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.156377] process a segment [D 20:01:00.156401] of 0 sz 2097152 [D 20:01:00.156426] add a segment 0 [D 20:01:00.156450] retval = 2097152 [D 20:01:00.156474] exiting distribute returned less than expected [D 20:01:00.156497] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.156523] *****RETURN***FROM***PROCESS***** [D 20:01:00.156548] end iteration [D 20:01:00.156572] done with segments or bytes [D 20:01:00.156596] of 2097152 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.156622] next loff: 2097152 next poff: 2097152 [D 20:01:00.156646] return value 2097152 [D 20:01:00.156671] finished [D 20:01:00.156695] retval = 2097152 [D 20:01:00.156719] exiting distribute returned less than expected [D 20:01:00.156743] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 33554432 eof 0 [D 20:01:00.156768] ========================================================= [D 20:01:00.156793] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:00.156819] element 0: offset: 0xa2451000, size: 2097152 [D 20:01:00.156847] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 11. [D 20:01:00.156875] post_sr: da13:3346 bh 43 len 16 wr_id 15e72f88. [D 20:01:00.156902] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 10. [D 20:01:00.156929] post_sr: da13:3346 bh 44 len 16 wr_id 15e72fa0. [D 20:01:00.156955] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 9. [D 20:01:00.156982] post_sr: da13:3346 bh 45 len 16 wr_id 15e72fb8. [D 20:01:00.157009] openib_check_cq: completed recv wr_id 15e73cb0. [D 20:01:00.157035] openib_post_rr: da13:3346 bh 83 wr_id 15e73cb0. [D 20:01:00.157062] openib_check_cq: completed recv wr_id 15e73cc8. [D 20:01:00.157089] openib_post_rr: da13:3346 bh 84 wr_id 15e73cc8. [D 20:01:00.157114] openib_check_cq: completed recv wr_id 15e73ce0. [D 20:01:00.157140] openib_post_rr: da13:3346 bh 85 wr_id 15e73ce0. [D 20:01:00.157167] openib_check_cq: completed recv wr_id 15e73cf8. [D 20:01:00.157196] openib_post_rr: da13:3346 bh 86 wr_id 15e73cf8. [D 20:01:00.157222] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 8. [D 20:01:00.157249] post_sr: da13:3346 bh 46 len 16 wr_id 15e72fd0. [D 20:01:00.157275] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 7. [D 20:01:00.157301] post_sr: da13:3346 bh 47 len 16 wr_id 15e72fe8. [D 20:01:00.157327] openib_check_cq: completed recv wr_id 15e73d10. [D 20:01:00.157354] openib_post_rr: da13:3346 bh 87 wr_id 15e73d10. [D 20:01:00.157380] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 6. [D 20:01:00.157406] post_sr: da13:3346 bh 48 len 16 wr_id 15e73000. [D 20:01:00.157433] openib_check_cq: completed send wr_id 15e72f40. [D 20:01:00.157459] openib_check_cq: completed recv wr_id 15e73d28. [D 20:01:00.157485] openib_post_rr: da13:3346 bh 88 wr_id 15e73d28. [D 20:01:00.157510] openib_check_cq: completed recv wr_id 15e73d40. [D 20:01:00.157536] openib_post_rr: da13:3346 bh 89 wr_id 15e73d40. [D 20:01:00.157563] openib_check_cq: completed recv wr_id 15e73d58. [D 20:01:00.157588] openib_post_rr: da13:3346 bh 90 wr_id 15e73d58. [D 20:01:00.157614] openib_check_cq: completed recv wr_id 15e73d70. [D 20:01:00.157641] openib_post_rr: da13:3346 bh 91 wr_id 15e73d70. [D 20:01:00.157668] openib_check_cq: completed recv wr_id 15e73d88. [D 20:01:00.157693] openib_post_rr: da13:3346 bh 92 wr_id 15e73d88. [D 20:01:00.157719] openib_check_cq: completed recv wr_id 15e73da0. [D 20:01:00.157746] openib_post_rr: da13:3346 bh 93 wr_id 15e73da0. [D 20:01:00.157772] openib_check_cq: completed recv wr_id 15e73db8. [D 20:01:00.157798] openib_post_rr: da13:3346 bh 94 wr_id 15e73db8. [D 20:01:00.157824] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 5. [D 20:01:00.157851] post_sr: da13:3346 bh 49 len 16 wr_id 15e73018. [D 20:01:00.157880] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 4. [D 20:01:00.157907] post_sr: da13:3346 bh 50 len 16 wr_id 15e73030. [D 20:01:00.157933] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 3. [D 20:01:00.157959] post_sr: da13:3346 bh 51 len 16 wr_id 15e73048. [D 20:01:00.157995] flowproto posted 0x15eb1f38 [D 20:01:00.158020] Job flows in progress (post time): 17 [D 20:01:00.158048] posted flow for context 0x15eb1988 [D 20:01:00.158074] *** all msgpairs complete (all flows posted) [D 20:01:00.158100] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.158124] [SM Exiting]: (0x15eb1140) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.158153] PINT_client_state_machine_test id 22 [D 20:01:00.158192] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:01:00.158219] ========================================================= [D 20:01:00.158244] PINT_process_request [D 20:01:00.158267] tiling 14 copies [D 20:01:00.158291] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.158318] lvl 0 el 0 blk 0 by 423624704 [D 20:01:00.158343] to 423624704 ta 402653184 fi 436207616 [D 20:01:00.158368] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.158392] basic type or contiguous data [D 20:01:00.158416] Request disp [D 20:01:00.158440] contig_offset = 423624704 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.158465] calling distribute [D 20:01:00.158489] PINT_distribute [D 20:01:00.158513] of 423624704 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 423624704 exfl 1 [D 20:01:00.158538] begin iteration loff: 423624704 [D 20:01:00.158563] segment exceeds byte limit [D 20:01:00.158588] file being extended [D 20:01:00.158611] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.158636] **********CALL***PROCESS********* [D 20:01:00.158659] segment of 20971520 sz 2097152 [D 20:01:00.158684] PINT_process_request [D 20:01:00.158708] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.158734] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.158759] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.158783] basic type or contiguous data [D 20:01:00.158807] Request disp [D 20:01:00.158831] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.158856] process a segment [D 20:01:00.158880] of 20971520 sz 2097152 [D 20:01:00.158905] add a segment 0 [D 20:01:00.158928] retval = 2097152 [D 20:01:00.158952] exiting distribute returned less than expected [D 20:01:00.158976] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.159002] *****RETURN***FROM***PROCESS***** [D 20:01:00.159026] end iteration [D 20:01:00.159049] done with segments or bytes [D 20:01:00.159073] of 425721856 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.159099] next loff: 425721856 next poff: 425721856 [D 20:01:00.159124] return value 2097152 [D 20:01:00.159148] finished [D 20:01:00.159172] retval = 2097152 [D 20:01:00.159200] exiting distribute returned less than expected [D 20:01:00.159227] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 425721856 fo 436207616 eof 0 [D 20:01:00.159257] ========================================================= [D 20:01:00.159286] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:01:00.159312] element 0: offset: 0xab859000, size: 2097152 [D 20:01:00.159339] post_sr: da13:3346 bh 52 len 32 wr_id 15e73060. [D 20:01:00.159378] sm control op 6 [D 20:01:00.159403] [SM Entering]: (0x15eb45f8) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.159429] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.159455] (0x15eb4690) io_datafile_complete_operations (tag 0) [D 20:01:00.159480] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.159505] - complete_context_send_or_recv called [D 20:01:00.159529] got a recv completion with context index 0 [D 20:01:00.159558] - io_process_context_recv called [D 20:01:00.159583] - io_process_context_recv called [D 20:01:00.159607] PINT_decode [D 20:01:00.159632] lebf_decode_resp [D 20:01:00.159658] PINT_encode_release [D 20:01:00.159683] lebf_encode_rel [D 20:01:00.159708] PINT_decode_release [D 20:01:00.159731] lebf_decode_rel [D 20:01:00.159758] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.159783] io_post_flow: entry [D 20:01:00.159808] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.159832] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.182394] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.182450] bstream_size = 440401920, datafile nr=0, ct=1, file_req_off = 33554432 [D 20:01:00.182481] PINT_new_request_states n=1 [D 20:01:00.182508] PINT_new_request_states n=1 [D 20:01:00.182534] flowproto posting 0x15eb5410 [D 20:01:00.182574] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:00.182600] ========================================================= [D 20:01:00.182625] PINT_process_request [D 20:01:00.182650] tiling 3 copies [D 20:01:00.182674] skipping ahead to target_offset [D 20:01:00.182698] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.182724] lvl 0 el 0 blk 0 by 0 [D 20:01:00.182749] to 0 ta 33554432 fi 67108864 [D 20:01:00.182774] mto 0 mta 0 mfi 33554432 [D 20:01:00.182798] basic type or contiguous data [D 20:01:00.182823] Request disp [D 20:01:00.182847] contig_offset = 0 contig_size = 100663296 lvl_flag = 1 [D 20:01:00.182871] process logical skip [D 20:01:00.182895] retval = 33554432 [D 20:01:00.182919] exiting logical skip because distribute indicates done [D 20:01:00.182943] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.182969] lvl 0 el 0 blk 0 by 33554432 [D 20:01:00.182993] to 33554432 ta 33554432 fi 67108864 [D 20:01:00.183018] mto 0 mta 0 mfi 33554432 [D 20:01:00.183043] basic type or contiguous data [D 20:01:00.183067] Request disp [D 20:01:00.183090] contig_offset = 33554432 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.183115] calling distribute [D 20:01:00.183139] PINT_distribute [D 20:01:00.183163] of 33554432 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:00.183195] begin iteration loff: 33554432 [D 20:01:00.183224] segment exceeds byte limit [D 20:01:00.183252] client lstof 0 diff 0 sgof 0 [D 20:01:00.183279] **********CALL***PROCESS********* [D 20:01:00.183306] segment of 0 sz 2097152 [D 20:01:00.183331] PINT_process_request [D 20:01:00.183355] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.183382] lvl 0 el 0 blk 0 by 0 [D 20:01:00.183406] to 0 ta 0 fi 2097152 [D 20:01:00.183431] basic type or contiguous data [D 20:01:00.183455] Request disp [D 20:01:00.183479] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.183505] process a segment [D 20:01:00.183528] of 0 sz 2097152 [D 20:01:00.183552] add a segment 0 [D 20:01:00.183576] retval = 2097152 [D 20:01:00.183601] exiting distribute returned less than expected [D 20:01:00.183625] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.183650] *****RETURN***FROM***PROCESS***** [D 20:01:00.183675] end iteration [D 20:01:00.183699] done with segments or bytes [D 20:01:00.183723] of 35651584 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.183749] next loff: 35651584 next poff: 35651584 [D 20:01:00.183773] return value 2097152 [D 20:01:00.183798] finished [D 20:01:00.183822] retval = 2097152 [D 20:01:00.183846] exiting distribute returned less than expected [D 20:01:00.183870] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 35651584 fo 67108864 eof 0 [D 20:01:00.183896] ========================================================= [D 20:01:00.183920] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:00.183946] element 0: offset: 0xa044f000, size: 2097152 [D 20:01:00.183978] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 2. [D 20:01:00.184007] post_sr: da13:3346 bh 53 len 16 wr_id 15e73078. [D 20:01:00.184034] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 1. [D 20:01:00.184061] post_sr: da13:3346 bh 54 len 16 wr_id 15e73090. [D 20:01:00.184087] openib_check_cq: completed send wr_id 15e72f58. [D 20:01:00.184113] openib_check_cq: completed send wr_id 15e72f70. [D 20:01:00.184139] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 0. [D 20:01:00.184165] post_sr: da13:3346 bh 55 len 16 wr_id 15e730a8. [D 20:01:00.184191] openib_check_cq: completed send wr_id 15e72f88. [D 20:01:00.184216] openib_check_cq: completed send wr_id 15e72fa0. [D 20:01:00.184242] openib_check_cq: completed send wr_id 15e72fb8. [D 20:01:00.184267] openib_check_cq: completed send wr_id 15e72fd0. [D 20:01:00.184292] openib_check_cq: completed send wr_id 15e72fe8. [D 20:01:00.184318] openib_check_cq: completed send wr_id 15e73000. [D 20:01:00.184343] openib_check_cq: completed send wr_id 15e73018. [D 20:01:00.184369] openib_check_cq: completed send wr_id 15e73030. [D 20:01:00.184393] openib_check_cq: completed send wr_id 15e73048. [D 20:01:00.184419] openib_check_cq: completed send wr_id 15e73060. [D 20:01:00.184444] openib_check_cq: completed recv wr_id 15e73dd0. [D 20:01:00.184473] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab1801010 rkey c8004c00 nic_wr_credit 1. [D 20:01:00.184500] openib_post_rr: da13:3346 bh 95 wr_id 15e73dd0. [D 20:01:00.184527] openib_check_cq: completed send wr_id 15e73078. [D 20:01:00.184552] openib_check_cq: completed send wr_id 15e73090. [D 20:01:00.184577] openib_check_cq: completed send wr_id 15e730a8. [D 20:01:00.184615] flowproto posted 0x15eb5410 [D 20:01:00.184639] Job flows in progress (post time): 18 [D 20:01:00.184668] posted flow for context 0x15eb4e60 [D 20:01:00.184693] *** all msgpairs complete (all flows posted) [D 20:01:00.184718] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.184743] [SM Exiting]: (0x15eb45f8) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.184770] sm control op 6 [D 20:01:00.184795] [SM Entering]: (0x15eb7ad0) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.184821] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.184846] (0x15eb7b68) io_datafile_complete_operations (tag 0) [D 20:01:00.184872] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.184898] - complete_context_send_or_recv called [D 20:01:00.184923] got a recv completion with context index 0 [D 20:01:00.184948] - io_process_context_recv called [D 20:01:00.184973] - io_process_context_recv called [D 20:01:00.184997] PINT_decode [D 20:01:00.185024] lebf_decode_resp [D 20:01:00.185049] PINT_encode_release [D 20:01:00.185073] lebf_encode_rel [D 20:01:00.185098] PINT_decode_release [D 20:01:00.185122] lebf_decode_rel [D 20:01:00.185148] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.185173] io_post_flow: entry [D 20:01:00.185201] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.185226] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.207912] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.207963] bstream_size = 440401920, datafile nr=0, ct=1, file_req_off = 67108864 [D 20:01:00.207995] PINT_new_request_states n=1 [D 20:01:00.208022] PINT_new_request_states n=1 [D 20:01:00.208047] flowproto posting 0x15eb88e8 [D 20:01:00.208092] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:00.208120] ========================================================= [D 20:01:00.208145] PINT_process_request [D 20:01:00.208168] tiling 4 copies [D 20:01:00.208192] skipping ahead to target_offset [D 20:01:00.208217] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.208243] lvl 0 el 0 blk 0 by 0 [D 20:01:00.208268] to 0 ta 67108864 fi 100663296 [D 20:01:00.208296] mto 0 mta 0 mfi 33554432 [D 20:01:00.208321] basic type or contiguous data [D 20:01:00.208345] Request disp [D 20:01:00.208369] contig_offset = 0 contig_size = 134217728 lvl_flag = 1 [D 20:01:00.208394] process logical skip [D 20:01:00.208417] retval = 67108864 [D 20:01:00.208442] exiting logical skip because distribute indicates done [D 20:01:00.208466] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.208492] lvl 0 el 0 blk 0 by 67108864 [D 20:01:00.208517] to 67108864 ta 67108864 fi 100663296 [D 20:01:00.208541] mto 0 mta 0 mfi 33554432 [D 20:01:00.208566] basic type or contiguous data [D 20:01:00.208590] Request disp [D 20:01:00.208614] contig_offset = 67108864 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.208639] calling distribute [D 20:01:00.208662] PINT_distribute [D 20:01:00.208687] of 67108864 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:00.208714] begin iteration loff: 67108864 [D 20:01:00.208739] segment exceeds byte limit [D 20:01:00.208764] client lstof 0 diff 0 sgof 0 [D 20:01:00.208788] **********CALL***PROCESS********* [D 20:01:00.208812] segment of 0 sz 2097152 [D 20:01:00.208836] PINT_process_request [D 20:01:00.208860] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.208887] lvl 0 el 0 blk 0 by 0 [D 20:01:00.208910] to 0 ta 0 fi 2097152 [D 20:01:00.208935] basic type or contiguous data [D 20:01:00.208959] Request disp [D 20:01:00.208983] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.209009] process a segment [D 20:01:00.209033] of 0 sz 2097152 [D 20:01:00.209056] add a segment 0 [D 20:01:00.209080] retval = 2097152 [D 20:01:00.209105] exiting distribute returned less than expected [D 20:01:00.209129] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.209156] *****RETURN***FROM***PROCESS***** [D 20:01:00.209182] end iteration [D 20:01:00.209206] done with segments or bytes [D 20:01:00.209231] of 69206016 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.209257] next loff: 69206016 next poff: 69206016 [D 20:01:00.209281] return value 2097152 [D 20:01:00.209305] finished [D 20:01:00.209329] retval = 2097152 [D 20:01:00.209353] exiting distribute returned less than expected [D 20:01:00.209377] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 69206016 fo 100663296 eof 0 [D 20:01:00.209403] ========================================================= [D 20:01:00.209428] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:00.209453] element 0: offset: 0x9e44d000, size: 2097152 [D 20:01:00.209481] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 0. [D 20:01:00.209508] post_sr: da13:3346 bh 56 len 16 wr_id 15e730c0. [D 20:01:00.209544] flowproto posted 0x15eb88e8 [D 20:01:00.209569] Job flows in progress (post time): 19 [D 20:01:00.209598] posted flow for context 0x15eb8338 [D 20:01:00.209623] *** all msgpairs complete (all flows posted) [D 20:01:00.209648] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.209672] [SM Exiting]: (0x15eb7ad0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.209700] sm control op 6 [D 20:01:00.209725] [SM Entering]: (0x15ebafa8) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.209751] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.209776] (0x15ebb040) io_datafile_complete_operations (tag 0) [D 20:01:00.209801] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.209827] - complete_context_send_or_recv called [D 20:01:00.209851] got a recv completion with context index 0 [D 20:01:00.209877] - io_process_context_recv called [D 20:01:00.209901] - io_process_context_recv called [D 20:01:00.209925] PINT_decode [D 20:01:00.209950] lebf_decode_resp [D 20:01:00.209975] PINT_encode_release [D 20:01:00.210000] lebf_encode_rel [D 20:01:00.210024] PINT_decode_release [D 20:01:00.210051] lebf_decode_rel [D 20:01:00.210078] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.210103] io_post_flow: entry [D 20:01:00.210127] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.210152] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.232895] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.232946] bstream_size = 442499072, datafile nr=0, ct=1, file_req_off = 100663296 [D 20:01:00.232976] PINT_new_request_states n=1 [D 20:01:00.233002] PINT_new_request_states n=1 [D 20:01:00.233027] flowproto posting 0x15ebbdc0 [D 20:01:00.233063] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:00.233090] ========================================================= [D 20:01:00.233115] PINT_process_request [D 20:01:00.233139] tiling 5 copies [D 20:01:00.233163] skipping ahead to target_offset [D 20:01:00.233190] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.233217] lvl 0 el 0 blk 0 by 0 [D 20:01:00.233242] to 0 ta 100663296 fi 134217728 [D 20:01:00.233266] mto 0 mta 0 mfi 33554432 [D 20:01:00.233291] basic type or contiguous data [D 20:01:00.233315] Request disp [D 20:01:00.233339] contig_offset = 0 contig_size = 167772160 lvl_flag = 1 [D 20:01:00.233364] process logical skip [D 20:01:00.233387] retval = 100663296 [D 20:01:00.233411] exiting logical skip because distribute indicates done [D 20:01:00.233435] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.233462] lvl 0 el 0 blk 0 by 100663296 [D 20:01:00.233486] to 100663296 ta 100663296 fi 134217728 [D 20:01:00.233510] mto 0 mta 0 mfi 33554432 [D 20:01:00.233535] basic type or contiguous data [D 20:01:00.233559] Request disp [D 20:01:00.233583] contig_offset = 100663296 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.233608] calling distribute [D 20:01:00.233632] PINT_distribute [D 20:01:00.233655] of 100663296 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:00.233682] begin iteration loff: 100663296 [D 20:01:00.233708] segment exceeds byte limit [D 20:01:00.233732] client lstof 0 diff 0 sgof 0 [D 20:01:00.233757] **********CALL***PROCESS********* [D 20:01:00.233780] segment of 0 sz 2097152 [D 20:01:00.233805] PINT_process_request [D 20:01:00.233829] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.233855] lvl 0 el 0 blk 0 by 0 [D 20:01:00.233880] to 0 ta 0 fi 2097152 [D 20:01:00.233904] basic type or contiguous data [D 20:01:00.233928] Request disp [D 20:01:00.233952] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.233977] process a segment [D 20:01:00.234001] of 0 sz 2097152 [D 20:01:00.234025] add a segment 0 [D 20:01:00.234049] retval = 2097152 [D 20:01:00.234073] exiting distribute returned less than expected [D 20:01:00.234097] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.234124] *****RETURN***FROM***PROCESS***** [D 20:01:00.234148] end iteration [D 20:01:00.234172] done with segments or bytes [D 20:01:00.234197] of 102760448 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.234223] next loff: 102760448 next poff: 102760448 [D 20:01:00.234248] return value 2097152 [D 20:01:00.234272] finished [D 20:01:00.234296] retval = 2097152 [D 20:01:00.234320] exiting distribute returned less than expected [D 20:01:00.234344] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 102760448 fo 134217728 eof 0 [D 20:01:00.234370] ========================================================= [D 20:01:00.234395] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:00.234419] element 0: offset: 0x9c44b000, size: 2097152 [D 20:01:00.234447] openib_check_cq: completed send wr_id 15e730c0. [D 20:01:00.234480] flowproto posted 0x15ebbdc0 [D 20:01:00.234505] Job flows in progress (post time): 20 [D 20:01:00.234533] posted flow for context 0x15ebb810 [D 20:01:00.234558] *** all msgpairs complete (all flows posted) [D 20:01:00.234586] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.234611] [SM Exiting]: (0x15ebafa8) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.234638] sm control op 6 [D 20:01:00.234663] [SM Entering]: (0x15ebe480) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.234688] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.234714] (0x15ebe518) io_datafile_complete_operations (tag 0) [D 20:01:00.234739] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.234765] - complete_context_send_or_recv called [D 20:01:00.234788] got a recv completion with context index 0 [D 20:01:00.234814] - io_process_context_recv called [D 20:01:00.234838] - io_process_context_recv called [D 20:01:00.234862] PINT_decode [D 20:01:00.234887] lebf_decode_resp [D 20:01:00.234911] PINT_encode_release [D 20:01:00.234935] lebf_encode_rel [D 20:01:00.234961] PINT_decode_release [D 20:01:00.234985] lebf_decode_rel [D 20:01:00.235011] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.235036] io_post_flow: entry [D 20:01:00.235060] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.235086] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.257806] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.257859] bstream_size = 471859200, datafile nr=0, ct=1, file_req_off = 134217728 [D 20:01:00.257888] PINT_new_request_states n=1 [D 20:01:00.257914] PINT_new_request_states n=1 [D 20:01:00.257940] flowproto posting 0x15ebf298 [D 20:01:00.257977] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:00.258004] ========================================================= [D 20:01:00.258028] PINT_process_request [D 20:01:00.258052] tiling 6 copies [D 20:01:00.258077] skipping ahead to target_offset [D 20:01:00.258101] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.258128] lvl 0 el 0 blk 0 by 0 [D 20:01:00.258152] to 0 ta 134217728 fi 167772160 [D 20:01:00.258177] mto 0 mta 0 mfi 33554432 [D 20:01:00.258202] basic type or contiguous data [D 20:01:00.258226] Request disp [D 20:01:00.258250] contig_offset = 0 contig_size = 201326592 lvl_flag = 1 [D 20:01:00.258275] process logical skip [D 20:01:00.258299] retval = 134217728 [D 20:01:00.258323] exiting logical skip because distribute indicates done [D 20:01:00.258348] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.258374] lvl 0 el 0 blk 0 by 134217728 [D 20:01:00.258399] to 134217728 ta 134217728 fi 167772160 [D 20:01:00.258423] mto 0 mta 0 mfi 33554432 [D 20:01:00.258448] basic type or contiguous data [D 20:01:00.258472] Request disp [D 20:01:00.258496] contig_offset = 134217728 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.258522] calling distribute [D 20:01:00.258545] PINT_distribute [D 20:01:00.258569] of 134217728 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:00.258596] begin iteration loff: 134217728 [D 20:01:00.258622] segment exceeds byte limit [D 20:01:00.258646] client lstof 0 diff 0 sgof 0 [D 20:01:00.258670] **********CALL***PROCESS********* [D 20:01:00.258694] segment of 0 sz 2097152 [D 20:01:00.258719] PINT_process_request [D 20:01:00.258743] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.258769] lvl 0 el 0 blk 0 by 0 [D 20:01:00.258794] to 0 ta 0 fi 2097152 [D 20:01:00.258818] basic type or contiguous data [D 20:01:00.258843] Request disp [D 20:01:00.258867] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.258892] process a segment [D 20:01:00.258915] of 0 sz 2097152 [D 20:01:00.258940] add a segment 0 [D 20:01:00.258964] retval = 2097152 [D 20:01:00.258988] exiting distribute returned less than expected [D 20:01:00.259012] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.259038] *****RETURN***FROM***PROCESS***** [D 20:01:00.259067] end iteration [D 20:01:00.259091] done with segments or bytes [D 20:01:00.259116] of 136314880 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.259141] next loff: 136314880 next poff: 136314880 [D 20:01:00.259166] return value 2097152 [D 20:01:00.259195] finished [D 20:01:00.259222] retval = 2097152 [D 20:01:00.259249] exiting distribute returned less than expected [D 20:01:00.259277] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 136314880 fo 167772160 eof 0 [D 20:01:00.259305] ========================================================= [D 20:01:00.259330] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:00.259356] element 0: offset: 0x9a449000, size: 2097152 [D 20:01:00.259384] openib_check_cq: completed recv wr_id 15e73de8. [D 20:01:00.259411] openib_post_rr: da13:3346 bh 96 wr_id 15e73de8. [D 20:01:00.259447] flowproto posted 0x15ebf298 [D 20:01:00.259471] Job flows in progress (post time): 21 [D 20:01:00.259499] posted flow for context 0x15ebece8 [D 20:01:00.259525] *** all msgpairs complete (all flows posted) [D 20:01:00.259549] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.259574] [SM Exiting]: (0x15ebe480) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.259602] PINT_client_state_machine_test id 22 [D 20:01:00.259645] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:01:00.259672] ========================================================= [D 20:01:00.259696] PINT_process_request [D 20:01:00.259720] tiling 15 copies [D 20:01:00.259744] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.259771] lvl 0 el 0 blk 0 by 457179136 [D 20:01:00.259796] to 457179136 ta 436207616 fi 469762048 [D 20:01:00.259820] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:00.259845] basic type or contiguous data [D 20:01:00.259869] Request disp [D 20:01:00.259894] contig_offset = 457179136 contig_size = 46137344 lvl_flag = 1 [D 20:01:00.259919] calling distribute [D 20:01:00.259943] PINT_distribute [D 20:01:00.259966] of 457179136 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 457179136 exfl 1 [D 20:01:00.259993] begin iteration loff: 457179136 [D 20:01:00.260019] segment exceeds byte limit [D 20:01:00.260043] file being extended [D 20:01:00.260066] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:00.260091] **********CALL***PROCESS********* [D 20:01:00.260115] segment of 20971520 sz 2097152 [D 20:01:00.260140] PINT_process_request [D 20:01:00.260164] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.260190] lvl 0 el 0 blk 0 by 20971520 [D 20:01:00.260214] to 20971520 ta 20971520 fi 23068672 [D 20:01:00.260239] basic type or contiguous data [D 20:01:00.260263] Request disp [D 20:01:00.260287] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:00.260312] process a segment [D 20:01:00.260335] of 20971520 sz 2097152 [D 20:01:00.260360] add a segment 0 [D 20:01:00.260384] retval = 2097152 [D 20:01:00.260408] exiting distribute returned less than expected [D 20:01:00.260432] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:00.260458] *****RETURN***FROM***PROCESS***** [D 20:01:00.260482] end iteration [D 20:01:00.260506] done with segments or bytes [D 20:01:00.260530] of 459276288 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.260556] next loff: 459276288 next poff: 459276288 [D 20:01:00.260581] return value 2097152 [D 20:01:00.260604] finished [D 20:01:00.260628] retval = 2097152 [D 20:01:00.260652] exiting distribute returned less than expected [D 20:01:00.260676] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 459276288 fo 469762048 eof 0 [D 20:01:00.260703] ========================================================= [D 20:01:00.260727] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:01:00.260753] element 0: offset: 0xa9857000, size: 2097152 [D 20:01:00.260784] post_sr: da13:3346 bh 57 len 32 wr_id 15e730d8. [D 20:01:00.260814] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:01:00.260840] ========================================================= [D 20:01:00.260865] PINT_process_request [D 20:01:00.260889] tiling 2 copies [D 20:01:00.260912] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.260939] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.260964] to 23068672 ta 0 fi 33554432 [D 20:01:00.260989] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.261014] basic type or contiguous data [D 20:01:00.261038] Request disp [D 20:01:00.261062] contig_offset = 23068672 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.261087] calling distribute [D 20:01:00.261112] PINT_distribute [D 20:01:00.261136] of 23068672 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 23068672 exfl 1 [D 20:01:00.261161] begin iteration loff: 23068672 [D 20:01:00.261190] segment exceeds byte limit [D 20:01:00.261214] file being extended [D 20:01:00.261238] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.261262] **********CALL***PROCESS********* [D 20:01:00.261287] segment of 23068672 sz 2097152 [D 20:01:00.261311] PINT_process_request [D 20:01:00.261335] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.261362] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.261386] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.261411] basic type or contiguous data [D 20:01:00.261435] Request disp [D 20:01:00.261459] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.261484] process a segment [D 20:01:00.261507] of 23068672 sz 2097152 [D 20:01:00.261531] add a segment 0 [D 20:01:00.261555] retval = 2097152 [D 20:01:00.261580] exiting distribute returned less than expected [D 20:01:00.261604] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.261631] *****RETURN***FROM***PROCESS***** [D 20:01:00.261654] end iteration [D 20:01:00.261678] done with segments or bytes [D 20:01:00.261702] of 25165824 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.261728] next loff: 25165824 next poff: 25165824 [D 20:01:00.261753] return value 2097152 [D 20:01:00.261776] finished [D 20:01:00.261800] retval = 2097152 [D 20:01:00.261825] exiting distribute returned less than expected [D 20:01:00.261850] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 25165824 fo 33554432 eof 0 [D 20:01:00.261876] ========================================================= [D 20:01:00.261899] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:01:00.261925] element 0: offset: 0xc3a71000, size: 2097152 [D 20:01:00.261953] post_sr: da13:3346 bh 58 len 32 wr_id 15e730f0. [D 20:01:00.261981] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:01:00.262007] ========================================================= [D 20:01:00.262032] PINT_process_request [D 20:01:00.262056] tiling 3 copies [D 20:01:00.262081] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.262106] lvl 0 el 0 blk 0 by 56623104 [D 20:01:00.262131] to 56623104 ta 33554432 fi 67108864 [D 20:01:00.262156] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.262182] basic type or contiguous data [D 20:01:00.262206] Request disp [D 20:01:00.262230] contig_offset = 56623104 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.262255] calling distribute [D 20:01:00.262279] PINT_distribute [D 20:01:00.262303] of 56623104 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 56623104 exfl 1 [D 20:01:00.262332] begin iteration loff: 56623104 [D 20:01:00.262357] segment exceeds byte limit [D 20:01:00.262382] file being extended [D 20:01:00.262406] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.262431] **********CALL***PROCESS********* [D 20:01:00.262454] segment of 23068672 sz 2097152 [D 20:01:00.262479] PINT_process_request [D 20:01:00.262503] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.262532] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.262557] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.262581] basic type or contiguous data [D 20:01:00.262606] Request disp [D 20:01:00.262630] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.262655] process a segment [D 20:01:00.262679] of 23068672 sz 2097152 [D 20:01:00.262703] add a segment 0 [D 20:01:00.262727] retval = 2097152 [D 20:01:00.262751] exiting distribute returned less than expected [D 20:01:00.262775] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.262802] *****RETURN***FROM***PROCESS***** [D 20:01:00.262826] end iteration [D 20:01:00.262849] done with segments or bytes [D 20:01:00.262873] of 58720256 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.262899] next loff: 58720256 next poff: 58720256 [D 20:01:00.262924] return value 2097152 [D 20:01:00.262948] finished [D 20:01:00.262972] retval = 2097152 [D 20:01:00.262996] exiting distribute returned less than expected [D 20:01:00.263020] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 58720256 fo 67108864 eof 0 [D 20:01:00.263046] ========================================================= [D 20:01:00.263071] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:01:00.263096] element 0: offset: 0xc1a6f000, size: 2097152 [D 20:01:00.263123] post_sr: da13:3346 bh 59 len 32 wr_id 15e73108. [D 20:01:00.263153] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:01:00.263179] ========================================================= [D 20:01:00.263208] PINT_process_request [D 20:01:00.263236] tiling 4 copies [D 20:01:00.263263] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.263294] lvl 0 el 0 blk 0 by 90177536 [D 20:01:00.263319] to 90177536 ta 67108864 fi 100663296 [D 20:01:00.263344] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.263369] basic type or contiguous data [D 20:01:00.263393] Request disp [D 20:01:00.263417] contig_offset = 90177536 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.263442] calling distribute [D 20:01:00.263465] PINT_distribute [D 20:01:00.263489] of 90177536 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 90177536 exfl 1 [D 20:01:00.263516] begin iteration loff: 90177536 [D 20:01:00.263542] segment exceeds byte limit [D 20:01:00.263566] file being extended [D 20:01:00.263589] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.263614] **********CALL***PROCESS********* [D 20:01:00.263638] segment of 23068672 sz 2097152 [D 20:01:00.263662] PINT_process_request [D 20:01:00.263687] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.263712] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.263737] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.263763] basic type or contiguous data [D 20:01:00.263787] Request disp [D 20:01:00.263811] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.263835] process a segment [D 20:01:00.263859] of 23068672 sz 2097152 [D 20:01:00.263884] add a segment 0 [D 20:01:00.263908] retval = 2097152 [D 20:01:00.263932] exiting distribute returned less than expected [D 20:01:00.263957] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.263982] *****RETURN***FROM***PROCESS***** [D 20:01:00.264007] end iteration [D 20:01:00.264031] done with segments or bytes [D 20:01:00.264055] of 92274688 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.264081] next loff: 92274688 next poff: 92274688 [D 20:01:00.264105] return value 2097152 [D 20:01:00.264129] finished [D 20:01:00.264153] retval = 2097152 [D 20:01:00.264177] exiting distribute returned less than expected [D 20:01:00.264202] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 92274688 fo 100663296 eof 0 [D 20:01:00.264227] ========================================================= [D 20:01:00.264252] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:01:00.264280] element 0: offset: 0xbfa6d000, size: 2097152 [D 20:01:00.264307] post_sr: da13:3346 bh 60 len 32 wr_id 15e73120. [D 20:01:00.264337] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:01:00.264363] ========================================================= [D 20:01:00.264387] PINT_process_request [D 20:01:00.264411] tiling 5 copies [D 20:01:00.264435] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.264461] lvl 0 el 0 blk 0 by 123731968 [D 20:01:00.264486] to 123731968 ta 100663296 fi 134217728 [D 20:01:00.264511] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.264536] basic type or contiguous data [D 20:01:00.264560] Request disp [D 20:01:00.264584] contig_offset = 123731968 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.264609] calling distribute [D 20:01:00.264633] PINT_distribute [D 20:01:00.264657] of 123731968 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 123731968 exfl 1 [D 20:01:00.264683] begin iteration loff: 123731968 [D 20:01:00.264709] segment exceeds byte limit [D 20:01:00.264733] file being extended [D 20:01:00.264757] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.264781] **********CALL***PROCESS********* [D 20:01:00.264805] segment of 23068672 sz 2097152 [D 20:01:00.264830] PINT_process_request [D 20:01:00.264855] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.264881] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.264905] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.264930] basic type or contiguous data [D 20:01:00.264955] Request disp [D 20:01:00.264979] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.265004] process a segment [D 20:01:00.265027] of 23068672 sz 2097152 [D 20:01:00.265051] add a segment 0 [D 20:01:00.265076] retval = 2097152 [D 20:01:00.265100] exiting distribute returned less than expected [D 20:01:00.265124] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.265151] *****RETURN***FROM***PROCESS***** [D 20:01:00.265175] end iteration [D 20:01:00.265202] done with segments or bytes [D 20:01:00.265225] of 125829120 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.265252] next loff: 125829120 next poff: 125829120 [D 20:01:00.265275] return value 2097152 [D 20:01:00.265300] finished [D 20:01:00.265324] retval = 2097152 [D 20:01:00.265348] exiting distribute returned less than expected [D 20:01:00.265372] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 125829120 fo 134217728 eof 0 [D 20:01:00.265398] ========================================================= [D 20:01:00.265423] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:01:00.265448] element 0: offset: 0xbda6b000, size: 2097152 [D 20:01:00.265476] post_sr: da13:3346 bh 61 len 32 wr_id 15e73138. [D 20:01:00.265506] openib_check_cq: completed send wr_id 15e730d8. [D 20:01:00.265532] openib_check_cq: completed recv wr_id 15e73e00. [D 20:01:00.265560] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab1a02010 rkey c8004d00 nic_wr_credit 1. [D 20:01:00.265587] openib_post_rr: da13:3346 bh 97 wr_id 15e73e00. [D 20:01:00.265613] openib_check_cq: completed send wr_id 15e730f0. [D 20:01:00.265638] openib_check_cq: completed recv wr_id 15e73e18. [D 20:01:00.265665] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab1c03010 rkey c8004e00 nic_wr_credit 2. [D 20:01:00.265692] openib_post_rr: da13:3346 bh 98 wr_id 15e73e18. [D 20:01:00.265718] openib_check_cq: completed send wr_id 15e73108. [D 20:01:00.265743] openib_check_cq: completed recv wr_id 15e73e30. [D 20:01:00.265770] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab1e04010 rkey c8004f00 nic_wr_credit 3. [D 20:01:00.265797] openib_post_rr: da13:3346 bh 99 wr_id 15e73e30. [D 20:01:00.265823] openib_check_cq: completed send wr_id 15e73120. [D 20:01:00.265853] openib_check_cq: completed recv wr_id 15e734e8. [D 20:01:00.265881] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab2005010 rkey 48005000 nic_wr_credit 4. [D 20:01:00.265908] openib_post_rr: da13:3346 bh 0 wr_id 15e734e8. [D 20:01:00.265933] openib_check_cq: completed send wr_id 15e73138. [D 20:01:00.265959] openib_check_cq: completed recv wr_id 15e73500. [D 20:01:00.265986] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaab2206010 rkey 48005100 nic_wr_credit 5. [D 20:01:00.266013] openib_post_rr: da13:3346 bh 1 wr_id 15e73500. [D 20:01:00.266046] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:01:00.266072] ========================================================= [D 20:01:00.266097] PINT_process_request [D 20:01:00.266120] tiling 6 copies [D 20:01:00.266144] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.266171] lvl 0 el 0 blk 0 by 157286400 [D 20:01:00.266197] to 157286400 ta 134217728 fi 167772160 [D 20:01:00.266222] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.266246] basic type or contiguous data [D 20:01:00.266270] Request disp [D 20:01:00.266294] contig_offset = 157286400 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.266319] calling distribute [D 20:01:00.266343] PINT_distribute [D 20:01:00.266366] of 157286400 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 157286400 exfl 1 [D 20:01:00.266393] begin iteration loff: 157286400 [D 20:01:00.266419] segment exceeds byte limit [D 20:01:00.266443] file being extended [D 20:01:00.266467] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.266491] **********CALL***PROCESS********* [D 20:01:00.266515] segment of 23068672 sz 2097152 [D 20:01:00.266539] PINT_process_request [D 20:01:00.266563] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.266590] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.266614] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.266639] basic type or contiguous data [D 20:01:00.266663] Request disp [D 20:01:00.266687] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.266712] process a segment [D 20:01:00.266736] of 23068672 sz 2097152 [D 20:01:00.266760] add a segment 0 [D 20:01:00.266784] retval = 2097152 [D 20:01:00.266808] exiting distribute returned less than expected [D 20:01:00.266832] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.266859] *****RETURN***FROM***PROCESS***** [D 20:01:00.266883] end iteration [D 20:01:00.266907] done with segments or bytes [D 20:01:00.266930] of 159383552 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.266956] next loff: 159383552 next poff: 159383552 [D 20:01:00.266981] return value 2097152 [D 20:01:00.267005] finished [D 20:01:00.267028] retval = 2097152 [D 20:01:00.267053] exiting distribute returned less than expected [D 20:01:00.267077] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 159383552 fo 167772160 eof 0 [D 20:01:00.267103] ========================================================= [D 20:01:00.267127] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:01:00.267152] element 0: offset: 0xbba69000, size: 2097152 [D 20:01:00.267181] post_sr: da13:3346 bh 62 len 32 wr_id 15e73150. [D 20:01:00.267215] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:01:00.267245] ========================================================= [D 20:01:00.267272] PINT_process_request [D 20:01:00.267299] tiling 7 copies [D 20:01:00.267324] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.267350] lvl 0 el 0 blk 0 by 190840832 [D 20:01:00.267374] to 190840832 ta 167772160 fi 201326592 [D 20:01:00.267399] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.267424] basic type or contiguous data [D 20:01:00.267449] Request disp [D 20:01:00.267473] contig_offset = 190840832 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.267500] calling distribute [D 20:01:00.267525] PINT_distribute [D 20:01:00.267549] of 190840832 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 190840832 exfl 1 [D 20:01:00.267576] begin iteration loff: 190840832 [D 20:01:00.267601] segment exceeds byte limit [D 20:01:00.267624] file being extended [D 20:01:00.267648] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.267673] **********CALL***PROCESS********* [D 20:01:00.267697] segment of 23068672 sz 2097152 [D 20:01:00.267722] PINT_process_request [D 20:01:00.267745] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.267772] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.267797] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.267822] basic type or contiguous data [D 20:01:00.267846] Request disp [D 20:01:00.267869] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.267894] process a segment [D 20:01:00.267918] of 23068672 sz 2097152 [D 20:01:00.267943] add a segment 0 [D 20:01:00.267967] retval = 2097152 [D 20:01:00.267991] exiting distribute returned less than expected [D 20:01:00.268014] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.268041] *****RETURN***FROM***PROCESS***** [D 20:01:00.268065] end iteration [D 20:01:00.268089] done with segments or bytes [D 20:01:00.268113] of 192937984 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.268138] next loff: 192937984 next poff: 192937984 [D 20:01:00.268163] return value 2097152 [D 20:01:00.268187] finished [D 20:01:00.268211] retval = 2097152 [D 20:01:00.268235] exiting distribute returned less than expected [D 20:01:00.268260] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 192937984 fo 201326592 eof 0 [D 20:01:00.268285] ========================================================= [D 20:01:00.268310] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:01:00.268335] element 0: offset: 0xb9a67000, size: 2097152 [D 20:01:00.268362] post_sr: da13:3346 bh 63 len 32 wr_id 15e73168. [D 20:01:00.268392] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:01:00.268419] ========================================================= [D 20:01:00.268443] PINT_process_request [D 20:01:00.268466] tiling 8 copies [D 20:01:00.268491] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.268518] lvl 0 el 0 blk 0 by 224395264 [D 20:01:00.268543] to 224395264 ta 201326592 fi 234881024 [D 20:01:00.268568] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.268593] basic type or contiguous data [D 20:01:00.268617] Request disp [D 20:01:00.268641] contig_offset = 224395264 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.268666] calling distribute [D 20:01:00.268691] PINT_distribute [D 20:01:00.268714] of 224395264 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 224395264 exfl 1 [D 20:01:00.268740] begin iteration loff: 224395264 [D 20:01:00.268766] segment exceeds byte limit [D 20:01:00.268790] file being extended [D 20:01:00.268814] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.268838] **********CALL***PROCESS********* [D 20:01:00.268862] segment of 23068672 sz 2097152 [D 20:01:00.268887] PINT_process_request [D 20:01:00.268911] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.268937] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.268962] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.268987] basic type or contiguous data [D 20:01:00.269011] Request disp [D 20:01:00.269035] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.269060] process a segment [D 20:01:00.269083] of 23068672 sz 2097152 [D 20:01:00.269107] add a segment 0 [D 20:01:00.269132] retval = 2097152 [D 20:01:00.269156] exiting distribute returned less than expected [D 20:01:00.269182] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.269208] *****RETURN***FROM***PROCESS***** [D 20:01:00.269233] end iteration [D 20:01:00.269260] done with segments or bytes [D 20:01:00.269284] of 226492416 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.269309] next loff: 226492416 next poff: 226492416 [D 20:01:00.269334] return value 2097152 [D 20:01:00.269359] finished [D 20:01:00.269382] retval = 2097152 [D 20:01:00.269407] exiting distribute returned less than expected [D 20:01:00.269431] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 226492416 fo 234881024 eof 0 [D 20:01:00.269457] ========================================================= [D 20:01:00.269481] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:01:00.269507] element 0: offset: 0xb7a65000, size: 2097152 [D 20:01:00.269534] post_sr: da13:3346 bh 64 len 32 wr_id 15e73180. [D 20:01:00.269563] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:01:00.269590] ========================================================= [D 20:01:00.269614] PINT_process_request [D 20:01:00.269638] tiling 9 copies [D 20:01:00.269662] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.269688] lvl 0 el 0 blk 0 by 257949696 [D 20:01:00.269714] to 257949696 ta 234881024 fi 268435456 [D 20:01:00.269739] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.269763] basic type or contiguous data [D 20:01:00.269787] Request disp [D 20:01:00.269811] contig_offset = 257949696 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.269837] calling distribute [D 20:01:00.269861] PINT_distribute [D 20:01:00.269885] of 257949696 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 257949696 exfl 1 [D 20:01:00.269910] begin iteration loff: 257949696 [D 20:01:00.269936] segment exceeds byte limit [D 20:01:00.269960] file being extended [D 20:01:00.269984] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.270009] **********CALL***PROCESS********* [D 20:01:00.270032] segment of 23068672 sz 2097152 [D 20:01:00.270057] PINT_process_request [D 20:01:00.270081] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.270107] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.270132] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.270156] basic type or contiguous data [D 20:01:00.270180] Request disp [D 20:01:00.270204] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.270230] process a segment [D 20:01:00.270253] of 23068672 sz 2097152 [D 20:01:00.270277] add a segment 0 [D 20:01:00.270301] retval = 2097152 [D 20:01:00.270326] exiting distribute returned less than expected [D 20:01:00.270350] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.270376] *****RETURN***FROM***PROCESS***** [D 20:01:00.270401] end iteration [D 20:01:00.270424] done with segments or bytes [D 20:01:00.270448] of 260046848 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.270474] next loff: 260046848 next poff: 260046848 [D 20:01:00.270499] return value 2097152 [D 20:01:00.270524] finished [D 20:01:00.270546] retval = 2097152 [D 20:01:00.270571] exiting distribute returned less than expected [D 20:01:00.270595] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 260046848 fo 268435456 eof 0 [D 20:01:00.270621] ========================================================= [D 20:01:00.270646] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:01:00.270671] element 0: offset: 0xb5a63000, size: 2097152 [D 20:01:00.270698] post_sr: da13:3346 bh 65 len 32 wr_id 15e73198. [D 20:01:00.270728] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:01:00.270753] ========================================================= [D 20:01:00.270778] PINT_process_request [D 20:01:00.270802] tiling 10 copies [D 20:01:00.270826] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.270853] lvl 0 el 0 blk 0 by 291504128 [D 20:01:00.270877] to 291504128 ta 268435456 fi 301989888 [D 20:01:00.270902] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.270929] basic type or contiguous data [D 20:01:00.270954] Request disp [D 20:01:00.270977] contig_offset = 291504128 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.271002] calling distribute [D 20:01:00.271027] PINT_distribute [D 20:01:00.271051] of 291504128 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 291504128 exfl 1 [D 20:01:00.271077] begin iteration loff: 291504128 [D 20:01:00.271102] segment exceeds byte limit [D 20:01:00.271126] file being extended [D 20:01:00.271150] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.271175] **********CALL***PROCESS********* [D 20:01:00.271203] segment of 23068672 sz 2097152 [D 20:01:00.271231] PINT_process_request [D 20:01:00.271259] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.271289] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.271314] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.271339] basic type or contiguous data [D 20:01:00.271363] Request disp [D 20:01:00.271387] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.271412] process a segment [D 20:01:00.271435] of 23068672 sz 2097152 [D 20:01:00.271460] add a segment 0 [D 20:01:00.271484] retval = 2097152 [D 20:01:00.271508] exiting distribute returned less than expected [D 20:01:00.271532] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.271558] *****RETURN***FROM***PROCESS***** [D 20:01:00.271582] end iteration [D 20:01:00.271606] done with segments or bytes [D 20:01:00.271630] of 293601280 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.271656] next loff: 293601280 next poff: 293601280 [D 20:01:00.271679] return value 2097152 [D 20:01:00.271704] finished [D 20:01:00.271727] retval = 2097152 [D 20:01:00.271751] exiting distribute returned less than expected [D 20:01:00.271776] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 293601280 fo 301989888 eof 0 [D 20:01:00.271801] ========================================================= [D 20:01:00.271826] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:01:00.271851] element 0: offset: 0xb3a61000, size: 2097152 [D 20:01:00.271879] post_sr: da13:3346 bh 66 len 32 wr_id 15e731b0. [D 20:01:00.271908] PINT_client_state_machine_test id 22 [D 20:01:00.271937] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 4. [D 20:01:00.271964] post_sr: da13:3346 bh 67 len 16 wr_id 15e731c8. [D 20:01:00.271990] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 3. [D 20:01:00.272017] post_sr: da13:3346 bh 68 len 16 wr_id 15e731e0. [D 20:01:00.272044] openib_check_cq: completed recv wr_id 15e73518. [D 20:01:00.272070] openib_post_rr: da13:3346 bh 2 wr_id 15e73518. [D 20:01:00.272099] post_sr: da13:3346 bh 69 len 48 wr_id 15e731f8. [D 20:01:00.272126] openib_check_cq: completed recv wr_id 15e73530. [D 20:01:00.272153] openib_post_rr: da13:3346 bh 3 wr_id 15e73530. [D 20:01:00.272179] openib_check_cq: completed recv wr_id 15e73548. [D 20:01:00.272207] openib_post_rr: da13:3346 bh 4 wr_id 15e73548. [D 20:01:00.272233] openib_check_cq: completed recv wr_id 15e73560. [D 20:01:00.272260] openib_post_rr: da13:3346 bh 5 wr_id 15e73560. [D 20:01:00.272287] openib_check_cq: completed recv wr_id 15e73578. [D 20:01:00.272315] openib_post_rr: da13:3346 bh 6 wr_id 15e73578. [D 20:01:00.272342] openib_check_cq: completed recv wr_id 15e73590. [D 20:01:00.272370] openib_post_rr: da13:3346 bh 7 wr_id 15e73590. [D 20:01:00.272397] openib_check_cq: completed recv wr_id 15e735a8. [D 20:01:00.272424] openib_post_rr: da13:3346 bh 8 wr_id 15e735a8. [D 20:01:00.272450] openib_check_cq: completed recv wr_id 15e735c0. [D 20:01:00.272478] openib_post_rr: da13:3346 bh 9 wr_id 15e735c0. [D 20:01:00.272512] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:01:00.272538] ========================================================= [D 20:01:00.272562] PINT_process_request [D 20:01:00.272586] tiling 11 copies [D 20:01:00.272611] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.272640] lvl 0 el 0 blk 0 by 325058560 [D 20:01:00.272665] to 325058560 ta 301989888 fi 335544320 [D 20:01:00.272690] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.272715] basic type or contiguous data [D 20:01:00.272739] Request disp [D 20:01:00.272763] contig_offset = 325058560 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.272789] calling distribute [D 20:01:00.272812] PINT_distribute [D 20:01:00.272836] of 325058560 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 325058560 exfl 1 [D 20:01:00.272863] begin iteration loff: 325058560 [D 20:01:00.272888] segment exceeds byte limit [D 20:01:00.272912] file being extended [D 20:01:00.272935] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.272960] **********CALL***PROCESS********* [D 20:01:00.272984] segment of 23068672 sz 2097152 [D 20:01:00.273009] PINT_process_request [D 20:01:00.273033] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.273058] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.273083] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.273108] basic type or contiguous data [D 20:01:00.273132] Request disp [D 20:01:00.273156] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.273182] process a segment [D 20:01:00.273207] of 23068672 sz 2097152 [D 20:01:00.273231] add a segment 0 [D 20:01:00.273255] retval = 2097152 [D 20:01:00.273279] exiting distribute returned less than expected [D 20:01:00.273303] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.273329] *****RETURN***FROM***PROCESS***** [D 20:01:00.273353] end iteration [D 20:01:00.273377] done with segments or bytes [D 20:01:00.273401] of 327155712 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.273426] next loff: 327155712 next poff: 327155712 [D 20:01:00.273450] return value 2097152 [D 20:01:00.273475] finished [D 20:01:00.273498] retval = 2097152 [D 20:01:00.273522] exiting distribute returned less than expected [D 20:01:00.273546] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 327155712 fo 335544320 eof 0 [D 20:01:00.273572] ========================================================= [D 20:01:00.273596] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:01:00.273622] element 0: offset: 0xb1a5f000, size: 2097152 [D 20:01:00.273649] post_sr: da13:3346 bh 70 len 32 wr_id 15e73210. [D 20:01:00.273678] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:01:00.273704] ========================================================= [D 20:01:00.273729] PINT_process_request [D 20:01:00.273753] tiling 12 copies [D 20:01:00.273777] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.273803] lvl 0 el 0 blk 0 by 358612992 [D 20:01:00.273828] to 358612992 ta 335544320 fi 369098752 [D 20:01:00.273853] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.273878] basic type or contiguous data [D 20:01:00.273902] Request disp [D 20:01:00.273926] contig_offset = 358612992 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.273951] calling distribute [D 20:01:00.273976] PINT_distribute [D 20:01:00.274000] of 358612992 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 358612992 exfl 1 [D 20:01:00.274025] begin iteration loff: 358612992 [D 20:01:00.274051] segment exceeds byte limit [D 20:01:00.274075] file being extended [D 20:01:00.274099] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.274124] **********CALL***PROCESS********* [D 20:01:00.274147] segment of 23068672 sz 2097152 [D 20:01:00.274171] PINT_process_request [D 20:01:00.274196] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.274222] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.274247] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.274271] basic type or contiguous data [D 20:01:00.274295] Request disp [D 20:01:00.274319] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.274344] process a segment [D 20:01:00.274371] of 23068672 sz 2097152 [D 20:01:00.274395] add a segment 0 [D 20:01:00.274419] retval = 2097152 [D 20:01:00.274443] exiting distribute returned less than expected [D 20:01:00.274468] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.274494] *****RETURN***FROM***PROCESS***** [D 20:01:00.274518] end iteration [D 20:01:00.274542] done with segments or bytes [D 20:01:00.274565] of 360710144 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.274592] next loff: 360710144 next poff: 360710144 [D 20:01:00.274616] return value 2097152 [D 20:01:00.274641] finished [D 20:01:00.274664] retval = 2097152 [D 20:01:00.274688] exiting distribute returned less than expected [D 20:01:00.274712] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 360710144 fo 369098752 eof 0 [D 20:01:00.274739] ========================================================= [D 20:01:00.274764] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:01:00.274788] element 0: offset: 0xafa5d000, size: 2097152 [D 20:01:00.274816] post_sr: da13:3346 bh 71 len 32 wr_id 15e73228. [D 20:01:00.274846] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:01:00.274871] ========================================================= [D 20:01:00.274895] PINT_process_request [D 20:01:00.274919] tiling 13 copies [D 20:01:00.274944] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.274970] lvl 0 el 0 blk 0 by 392167424 [D 20:01:00.274995] to 392167424 ta 369098752 fi 402653184 [D 20:01:00.275020] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.275045] basic type or contiguous data [D 20:01:00.275069] Request disp [D 20:01:00.275093] contig_offset = 392167424 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.275117] calling distribute [D 20:01:00.275142] PINT_distribute [D 20:01:00.275166] of 392167424 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 392167424 exfl 1 [D 20:01:00.275197] begin iteration loff: 392167424 [D 20:01:00.275225] segment exceeds byte limit [D 20:01:00.275253] file being extended [D 20:01:00.275280] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.275307] **********CALL***PROCESS********* [D 20:01:00.275331] segment of 23068672 sz 2097152 [D 20:01:00.275356] PINT_process_request [D 20:01:00.275380] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.275405] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.275430] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.275455] basic type or contiguous data [D 20:01:00.275479] Request disp [D 20:01:00.275503] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.275527] process a segment [D 20:01:00.275551] of 23068672 sz 2097152 [D 20:01:00.275576] add a segment 0 [D 20:01:00.275600] retval = 2097152 [D 20:01:00.275624] exiting distribute returned less than expected [D 20:01:00.275648] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.275674] *****RETURN***FROM***PROCESS***** [D 20:01:00.275699] end iteration [D 20:01:00.275723] done with segments or bytes [D 20:01:00.275747] of 394264576 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.275773] next loff: 394264576 next poff: 394264576 [D 20:01:00.275796] return value 2097152 [D 20:01:00.275821] finished [D 20:01:00.275845] retval = 2097152 [D 20:01:00.275869] exiting distribute returned less than expected [D 20:01:00.275893] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 394264576 fo 402653184 eof 0 [D 20:01:00.275919] ========================================================= [D 20:01:00.275943] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:01:00.275969] element 0: offset: 0xada5b000, size: 2097152 [D 20:01:00.275996] post_sr: da13:3346 bh 72 len 32 wr_id 15e73240. [D 20:01:00.276025] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:00.276051] ========================================================= [D 20:01:00.276079] PINT_process_request [D 20:01:00.276103] tiling 16 copies [D 20:01:00.276127] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.276154] lvl 0 el 0 blk 0 by 484442112 [D 20:01:00.276179] to 484442112 ta 469762048 fi 503316480 [D 20:01:00.276204] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:00.276228] basic type or contiguous data [D 20:01:00.276252] Request disp [D 20:01:00.276276] contig_offset = 484442112 contig_size = 52428800 lvl_flag = 1 [D 20:01:00.276302] calling distribute [D 20:01:00.276326] PINT_distribute [D 20:01:00.276350] of 484442112 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 484442112 exfl 1 [D 20:01:00.276376] begin iteration loff: 484442112 [D 20:01:00.276402] segment exceeds byte limit [D 20:01:00.276426] file being extended [D 20:01:00.276450] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:00.276474] **********CALL***PROCESS********* [D 20:01:00.276498] segment of 14680064 sz 2097152 [D 20:01:00.276523] PINT_process_request [D 20:01:00.276547] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.276573] lvl 0 el 0 blk 0 by 14680064 [D 20:01:00.276597] to 14680064 ta 14680064 fi 16777216 [D 20:01:00.276622] basic type or contiguous data [D 20:01:00.276646] Request disp [D 20:01:00.276670] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:00.276695] process a segment [D 20:01:00.276719] of 14680064 sz 2097152 [D 20:01:00.276743] add a segment 0 [D 20:01:00.276767] retval = 2097152 [D 20:01:00.276791] exiting distribute returned less than expected [D 20:01:00.276815] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:00.276842] *****RETURN***FROM***PROCESS***** [D 20:01:00.276865] end iteration [D 20:01:00.276889] done with segments or bytes [D 20:01:00.276913] of 486539264 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.276939] next loff: 486539264 next poff: 486539264 [D 20:01:00.276964] return value 2097152 [D 20:01:00.276988] finished [D 20:01:00.277011] retval = 2097152 [D 20:01:00.277035] exiting distribute returned less than expected [D 20:01:00.277060] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 486539264 fo 503316480 eof 0 [D 20:01:00.277086] ========================================================= [D 20:01:00.277111] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:00.277136] element 0: offset: 0xa7255000, size: 2097152 [D 20:01:00.277163] post_sr: da13:3346 bh 73 len 32 wr_id 15e73258. [D 20:01:00.277195] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:01:00.277221] ========================================================= [D 20:01:00.277245] PINT_process_request [D 20:01:00.277270] tiling 14 copies [D 20:01:00.277294] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.277320] lvl 0 el 0 blk 0 by 425721856 [D 20:01:00.277345] to 425721856 ta 402653184 fi 436207616 [D 20:01:00.277370] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.277395] basic type or contiguous data [D 20:01:00.277420] Request disp [D 20:01:00.277443] contig_offset = 425721856 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.277468] calling distribute [D 20:01:00.277493] PINT_distribute [D 20:01:00.277517] of 425721856 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 425721856 exfl 1 [D 20:01:00.277544] begin iteration loff: 425721856 [D 20:01:00.277568] segment exceeds byte limit [D 20:01:00.277592] file being extended [D 20:01:00.277616] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.277641] **********CALL***PROCESS********* [D 20:01:00.277665] segment of 23068672 sz 2097152 [D 20:01:00.277688] PINT_process_request [D 20:01:00.277712] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.277738] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.277764] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.277789] basic type or contiguous data [D 20:01:00.277814] Request disp [D 20:01:00.277839] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.277864] process a segment [D 20:01:00.277888] of 23068672 sz 2097152 [D 20:01:00.277912] add a segment 0 [D 20:01:00.277935] retval = 2097152 [D 20:01:00.277959] exiting distribute returned less than expected [D 20:01:00.277983] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.278010] *****RETURN***FROM***PROCESS***** [D 20:01:00.278034] end iteration [D 20:01:00.278058] done with segments or bytes [D 20:01:00.278081] of 427819008 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.278107] next loff: 427819008 next poff: 427819008 [D 20:01:00.278132] return value 2097152 [D 20:01:00.278156] finished [D 20:01:00.278180] retval = 2097152 [D 20:01:00.278203] exiting distribute returned less than expected [D 20:01:00.278228] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 427819008 fo 436207616 eof 0 [D 20:01:00.278254] ========================================================= [D 20:01:00.278279] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:01:00.278304] element 0: offset: 0xaba59000, size: 2097152 [D 20:01:00.278331] post_sr: da13:3346 bh 74 len 32 wr_id 15e73270. [D 20:01:00.278361] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 2. [D 20:01:00.278388] post_sr: da13:3346 bh 75 len 16 wr_id 15e73288. [D 20:01:00.278415] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 1. [D 20:01:00.278442] post_sr: da13:3346 bh 76 len 16 wr_id 15e732a0. [D 20:01:00.278468] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 0. [D 20:01:00.278494] post_sr: da13:3346 bh 77 len 16 wr_id 15e732b8. [D 20:01:00.278521] openib_check_cq: completed send wr_id 15e73150. [D 20:01:00.278547] openib_check_cq: completed send wr_id 15e73168. [D 20:01:00.278572] openib_check_cq: completed send wr_id 15e73180. [D 20:01:00.278596] openib_check_cq: completed send wr_id 15e73198. [D 20:01:00.278622] openib_check_cq: completed send wr_id 15e731b0. [D 20:01:00.278647] openib_check_cq: completed send wr_id 15e731c8. [D 20:01:00.278673] openib_check_cq: completed send wr_id 15e731e0. [D 20:01:00.278697] openib_check_cq: completed send wr_id 15e731f8. [D 20:01:00.278722] openib_check_cq: completed send wr_id 15e73210. [D 20:01:00.278748] openib_check_cq: completed send wr_id 15e73228. [D 20:01:00.278773] openib_check_cq: completed send wr_id 15e73240. [D 20:01:00.278799] openib_check_cq: completed send wr_id 15e73258. [D 20:01:00.278823] openib_check_cq: completed recv wr_id 15e735d8. [D 20:01:00.278850] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab2407010 rkey 48005200 nic_wr_credit 1. [D 20:01:00.278877] openib_post_rr: da13:3346 bh 10 wr_id 15e735d8. [D 20:01:00.278903] openib_check_cq: completed recv wr_id 15e735f0. [D 20:01:00.278929] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab2608010 rkey 48005300 nic_wr_credit 2. [D 20:01:00.278957] openib_post_rr: da13:3346 bh 11 wr_id 15e735f0. [D 20:01:00.278983] openib_check_cq: completed recv wr_id 15e73608. [D 20:01:00.279009] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab2809010 rkey 48005400 nic_wr_credit 3. [D 20:01:00.279036] openib_post_rr: da13:3346 bh 12 wr_id 15e73608. [D 20:01:00.279062] openib_check_cq: completed recv wr_id 15e73620. [D 20:01:00.279088] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab2a0a010 rkey 48005500 nic_wr_credit 4. [D 20:01:00.279115] openib_post_rr: da13:3346 bh 13 wr_id 15e73620. [D 20:01:00.279141] openib_check_cq: completed recv wr_id 15e73638. [D 20:01:00.279168] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaab2c0b010 rkey 48005600 nic_wr_credit 5. [D 20:01:00.279199] openib_post_rr: da13:3346 bh 14 wr_id 15e73638. [D 20:01:00.279230] openib_check_cq: completed send wr_id 15e73270. [D 20:01:00.279258] openib_check_cq: completed send wr_id 15e73288. [D 20:01:00.279291] openib_check_cq: completed send wr_id 15e732a0. [D 20:01:00.279316] openib_check_cq: completed send wr_id 15e732b8. [D 20:01:00.279349] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:01:00.279375] ========================================================= [D 20:01:00.279398] PINT_process_request [D 20:01:00.279423] tiling 15 copies [D 20:01:00.279447] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.279474] lvl 0 el 0 blk 0 by 459276288 [D 20:01:00.279499] to 459276288 ta 436207616 fi 469762048 [D 20:01:00.279523] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:00.279548] basic type or contiguous data [D 20:01:00.279572] Request disp [D 20:01:00.279596] contig_offset = 459276288 contig_size = 44040192 lvl_flag = 1 [D 20:01:00.279621] calling distribute [D 20:01:00.279645] PINT_distribute [D 20:01:00.279668] of 459276288 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 459276288 exfl 1 [D 20:01:00.279696] begin iteration loff: 459276288 [D 20:01:00.279721] segment exceeds byte limit [D 20:01:00.279745] file being extended [D 20:01:00.279768] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:00.279793] **********CALL***PROCESS********* [D 20:01:00.279817] segment of 23068672 sz 2097152 [D 20:01:00.279842] PINT_process_request [D 20:01:00.279866] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.279891] lvl 0 el 0 blk 0 by 23068672 [D 20:01:00.279916] to 23068672 ta 23068672 fi 25165824 [D 20:01:00.279941] basic type or contiguous data [D 20:01:00.279965] Request disp [D 20:01:00.279989] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:00.280014] process a segment [D 20:01:00.280037] of 23068672 sz 2097152 [D 20:01:00.280061] add a segment 0 [D 20:01:00.280085] retval = 2097152 [D 20:01:00.280109] exiting distribute returned less than expected [D 20:01:00.280133] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:00.280158] *****RETURN***FROM***PROCESS***** [D 20:01:00.280183] end iteration [D 20:01:00.280207] done with segments or bytes [D 20:01:00.280231] of 461373440 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.280256] next loff: 461373440 next poff: 461373440 [D 20:01:00.280281] return value 2097152 [D 20:01:00.280305] finished [D 20:01:00.280328] retval = 2097152 [D 20:01:00.280353] exiting distribute returned less than expected [D 20:01:00.280377] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 461373440 fo 469762048 eof 0 [D 20:01:00.280403] ========================================================= [D 20:01:00.280427] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:01:00.280452] element 0: offset: 0xa9a57000, size: 2097152 [D 20:01:00.280480] post_sr: da13:3346 bh 78 len 32 wr_id 15e732d0. [D 20:01:00.280508] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:01:00.280533] ========================================================= [D 20:01:00.280558] PINT_process_request [D 20:01:00.280582] tiling 2 copies [D 20:01:00.280606] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.280632] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.280657] to 25165824 ta 0 fi 33554432 [D 20:01:00.280682] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.280707] basic type or contiguous data [D 20:01:00.280731] Request disp [D 20:01:00.280755] contig_offset = 25165824 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.280780] calling distribute [D 20:01:00.280804] PINT_distribute [D 20:01:00.280828] of 25165824 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 25165824 exfl 1 [D 20:01:00.280854] begin iteration loff: 25165824 [D 20:01:00.280879] segment exceeds byte limit [D 20:01:00.280903] file being extended [D 20:01:00.280926] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.280951] **********CALL***PROCESS********* [D 20:01:00.280975] segment of 25165824 sz 2097152 [D 20:01:00.281002] PINT_process_request [D 20:01:00.281026] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.281052] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.281077] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.281103] basic type or contiguous data [D 20:01:00.281126] Request disp [D 20:01:00.281150] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.281175] process a segment [D 20:01:00.281202] of 25165824 sz 2097152 [D 20:01:00.281226] add a segment 0 [D 20:01:00.281250] retval = 2097152 [D 20:01:00.281274] exiting distribute returned less than expected [D 20:01:00.281298] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.281325] *****RETURN***FROM***PROCESS***** [D 20:01:00.281349] end iteration [D 20:01:00.281373] done with segments or bytes [D 20:01:00.281396] of 27262976 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.281422] next loff: 27262976 next poff: 27262976 [D 20:01:00.281447] return value 2097152 [D 20:01:00.281471] finished [D 20:01:00.281495] retval = 2097152 [D 20:01:00.281518] exiting distribute returned less than expected [D 20:01:00.281543] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 27262976 fo 33554432 eof 0 [D 20:01:00.281569] ========================================================= [D 20:01:00.281594] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:01:00.281619] element 0: offset: 0xc3c71000, size: 2097152 [D 20:01:00.281646] post_sr: da13:3346 bh 79 len 32 wr_id 15e732e8. [D 20:01:00.281674] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:01:00.281700] ========================================================= [D 20:01:00.281724] PINT_process_request [D 20:01:00.281748] tiling 3 copies [D 20:01:00.281773] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.281799] lvl 0 el 0 blk 0 by 58720256 [D 20:01:00.281824] to 58720256 ta 33554432 fi 67108864 [D 20:01:00.281848] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.281873] basic type or contiguous data [D 20:01:00.281897] Request disp [D 20:01:00.281921] contig_offset = 58720256 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.281947] calling distribute [D 20:01:00.281970] PINT_distribute [D 20:01:00.281994] of 58720256 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 58720256 exfl 1 [D 20:01:00.282020] begin iteration loff: 58720256 [D 20:01:00.282046] segment exceeds byte limit [D 20:01:00.282070] file being extended [D 20:01:00.282094] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.282118] **********CALL***PROCESS********* [D 20:01:00.282142] segment of 25165824 sz 2097152 [D 20:01:00.282167] PINT_process_request [D 20:01:00.282192] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.282217] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.282242] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.282267] basic type or contiguous data [D 20:01:00.282291] Request disp [D 20:01:00.282317] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.282342] process a segment [D 20:01:00.282366] of 25165824 sz 2097152 [D 20:01:00.282390] add a segment 0 [D 20:01:00.282414] retval = 2097152 [D 20:01:00.282438] exiting distribute returned less than expected [D 20:01:00.282462] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.282488] *****RETURN***FROM***PROCESS***** [D 20:01:00.282512] end iteration [D 20:01:00.282536] done with segments or bytes [D 20:01:00.282560] of 60817408 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.282586] next loff: 60817408 next poff: 60817408 [D 20:01:00.282610] return value 2097152 [D 20:01:00.282634] finished [D 20:01:00.282658] retval = 2097152 [D 20:01:00.282682] exiting distribute returned less than expected [D 20:01:00.282707] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 60817408 fo 67108864 eof 0 [D 20:01:00.282735] ========================================================= [D 20:01:00.282760] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:01:00.282785] element 0: offset: 0xc1c6f000, size: 2097152 [D 20:01:00.282813] post_sr: da13:3346 bh 80 len 32 wr_id 15e73300. [D 20:01:00.282841] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:01:00.282867] ========================================================= [D 20:01:00.282891] PINT_process_request [D 20:01:00.282916] tiling 4 copies [D 20:01:00.282939] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.282965] lvl 0 el 0 blk 0 by 92274688 [D 20:01:00.282990] to 92274688 ta 67108864 fi 100663296 [D 20:01:00.283016] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.283041] basic type or contiguous data [D 20:01:00.283064] Request disp [D 20:01:00.283088] contig_offset = 92274688 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.283113] calling distribute [D 20:01:00.283137] PINT_distribute [D 20:01:00.283161] of 92274688 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 92274688 exfl 1 [D 20:01:00.283190] begin iteration loff: 92274688 [D 20:01:00.283219] segment exceeds byte limit [D 20:01:00.283246] file being extended [D 20:01:00.283274] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.283302] **********CALL***PROCESS********* [D 20:01:00.283326] segment of 25165824 sz 2097152 [D 20:01:00.283350] PINT_process_request [D 20:01:00.283374] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.283400] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.283425] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.283450] basic type or contiguous data [D 20:01:00.283474] Request disp [D 20:01:00.283497] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.283523] process a segment [D 20:01:00.283546] of 25165824 sz 2097152 [D 20:01:00.283571] add a segment 0 [D 20:01:00.283595] retval = 2097152 [D 20:01:00.283618] exiting distribute returned less than expected [D 20:01:00.283642] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.283669] *****RETURN***FROM***PROCESS***** [D 20:01:00.283693] end iteration [D 20:01:00.283717] done with segments or bytes [D 20:01:00.283740] of 94371840 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.283766] next loff: 94371840 next poff: 94371840 [D 20:01:00.283791] return value 2097152 [D 20:01:00.283816] finished [D 20:01:00.283840] retval = 2097152 [D 20:01:00.283864] exiting distribute returned less than expected [D 20:01:00.283887] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 94371840 fo 100663296 eof 0 [D 20:01:00.283914] ========================================================= [D 20:01:00.283938] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:01:00.283964] element 0: offset: 0xbfc6d000, size: 2097152 [D 20:01:00.283990] post_sr: da13:3346 bh 81 len 32 wr_id 15e73318. [D 20:01:00.284019] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:01:00.284045] ========================================================= [D 20:01:00.284070] PINT_process_request [D 20:01:00.284093] tiling 5 copies [D 20:01:00.284118] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.284144] lvl 0 el 0 blk 0 by 125829120 [D 20:01:00.284169] to 125829120 ta 100663296 fi 134217728 [D 20:01:00.284193] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.284218] basic type or contiguous data [D 20:01:00.284242] Request disp [D 20:01:00.284266] contig_offset = 125829120 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.284292] calling distribute [D 20:01:00.284316] PINT_distribute [D 20:01:00.284339] of 125829120 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 125829120 exfl 1 [D 20:01:00.284365] begin iteration loff: 125829120 [D 20:01:00.284391] segment exceeds byte limit [D 20:01:00.284415] file being extended [D 20:01:00.284440] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.284466] **********CALL***PROCESS********* [D 20:01:00.284490] segment of 25165824 sz 2097152 [D 20:01:00.284514] PINT_process_request [D 20:01:00.284539] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.284564] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.284589] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.284614] basic type or contiguous data [D 20:01:00.284638] Request disp [D 20:01:00.284662] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.284687] process a segment [D 20:01:00.284710] of 25165824 sz 2097152 [D 20:01:00.284735] add a segment 0 [D 20:01:00.284759] retval = 2097152 [D 20:01:00.284783] exiting distribute returned less than expected [D 20:01:00.284807] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.284833] *****RETURN***FROM***PROCESS***** [D 20:01:00.284857] end iteration [D 20:01:00.284881] done with segments or bytes [D 20:01:00.284905] of 127926272 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.284931] next loff: 127926272 next poff: 127926272 [D 20:01:00.284955] return value 2097152 [D 20:01:00.284979] finished [D 20:01:00.285003] retval = 2097152 [D 20:01:00.285027] exiting distribute returned less than expected [D 20:01:00.285052] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 127926272 fo 134217728 eof 0 [D 20:01:00.285078] ========================================================= [D 20:01:00.285101] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:01:00.285127] element 0: offset: 0xbdc6b000, size: 2097152 [D 20:01:00.285154] post_sr: da13:3346 bh 82 len 32 wr_id 15e73330. [D 20:01:00.285186] PINT_client_state_machine_test id 22 [D 20:01:00.285215] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 4. [D 20:01:00.285243] post_sr: da13:3346 bh 83 len 16 wr_id 15e73348. [D 20:01:00.285268] openib_check_cq: completed recv wr_id 15e73650. [D 20:01:00.285295] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab2e0c010 rkey 48005700 nic_wr_credit 5. [D 20:01:00.285323] openib_post_rr: da13:3346 bh 15 wr_id 15e73650. [D 20:01:00.285348] openib_check_cq: completed recv wr_id 15e73668. [D 20:01:00.285375] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab300e010 rkey c8005800 nic_wr_credit 6. [D 20:01:00.285402] openib_post_rr: da13:3346 bh 16 wr_id 15e73668. [D 20:01:00.285428] openib_check_cq: completed recv wr_id 15e73680. [D 20:01:00.285454] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaab320f010 rkey c8005900 nic_wr_credit 7. [D 20:01:00.285481] openib_post_rr: da13:3346 bh 17 wr_id 15e73680. [D 20:01:00.285508] openib_check_cq: completed recv wr_id 15e73698. [D 20:01:00.285534] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaabc85d010 rkey 4800a400 nic_wr_credit 8. [D 20:01:00.285561] openib_post_rr: da13:3346 bh 18 wr_id 15e73698. [D 20:01:00.285587] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 7. [D 20:01:00.285614] post_sr: da13:3346 bh 84 len 16 wr_id 15e73360. [D 20:01:00.285657] sm control op 6 [D 20:01:00.285683] [SM Entering]: (0x15ec1880) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.285709] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.285735] (0x15ec1918) io_datafile_complete_operations (tag 0) [D 20:01:00.285760] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.285785] - complete_context_send_or_recv called [D 20:01:00.285810] got a recv completion with context index 0 [D 20:01:00.285835] - io_process_context_recv called [D 20:01:00.285860] - io_process_context_recv called [D 20:01:00.285884] PINT_decode [D 20:01:00.285909] lebf_decode_resp [D 20:01:00.285934] PINT_encode_release [D 20:01:00.285959] lebf_encode_rel [D 20:01:00.285984] PINT_decode_release [D 20:01:00.286007] lebf_decode_rel [D 20:01:00.286033] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.286062] io_post_flow: entry [D 20:01:00.286087] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.286111] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.308709] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.308760] bstream_size = 471859200, datafile nr=0, ct=1, file_req_off = 167772160 [D 20:01:00.308792] PINT_new_request_states n=1 [D 20:01:00.308817] PINT_new_request_states n=1 [D 20:01:00.308842] flowproto posting 0x15ec2678 [D 20:01:00.308884] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:00.308912] ========================================================= [D 20:01:00.308937] PINT_process_request [D 20:01:00.308961] tiling 7 copies [D 20:01:00.308985] skipping ahead to target_offset [D 20:01:00.309010] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.309036] lvl 0 el 0 blk 0 by 0 [D 20:01:00.309061] to 0 ta 167772160 fi 201326592 [D 20:01:00.309085] mto 0 mta 0 mfi 33554432 [D 20:01:00.309110] basic type or contiguous data [D 20:01:00.309134] Request disp [D 20:01:00.309158] contig_offset = 0 contig_size = 234881024 lvl_flag = 1 [D 20:01:00.309186] process logical skip [D 20:01:00.309210] retval = 167772160 [D 20:01:00.309235] exiting logical skip because distribute indicates done [D 20:01:00.309259] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.309285] lvl 0 el 0 blk 0 by 167772160 [D 20:01:00.309309] to 167772160 ta 167772160 fi 201326592 [D 20:01:00.309334] mto 0 mta 0 mfi 33554432 [D 20:01:00.309359] basic type or contiguous data [D 20:01:00.309383] Request disp [D 20:01:00.309407] contig_offset = 167772160 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.309432] calling distribute [D 20:01:00.309456] PINT_distribute [D 20:01:00.309480] of 167772160 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:00.309507] begin iteration loff: 167772160 [D 20:01:00.309532] segment exceeds byte limit [D 20:01:00.309556] client lstof 0 diff 0 sgof 0 [D 20:01:00.309580] **********CALL***PROCESS********* [D 20:01:00.309604] segment of 0 sz 2097152 [D 20:01:00.309629] PINT_process_request [D 20:01:00.309653] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.309679] lvl 0 el 0 blk 0 by 0 [D 20:01:00.309704] to 0 ta 0 fi 2097152 [D 20:01:00.309728] basic type or contiguous data [D 20:01:00.309753] Request disp [D 20:01:00.309777] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.309802] process a segment [D 20:01:00.309825] of 0 sz 2097152 [D 20:01:00.309849] add a segment 0 [D 20:01:00.309874] retval = 2097152 [D 20:01:00.309898] exiting distribute returned less than expected [D 20:01:00.309923] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.309948] *****RETURN***FROM***PROCESS***** [D 20:01:00.309973] end iteration [D 20:01:00.309997] done with segments or bytes [D 20:01:00.310021] of 169869312 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.310047] next loff: 169869312 next poff: 169869312 [D 20:01:00.310071] return value 2097152 [D 20:01:00.310096] finished [D 20:01:00.310120] retval = 2097152 [D 20:01:00.310144] exiting distribute returned less than expected [D 20:01:00.310168] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 169869312 fo 201326592 eof 0 [D 20:01:00.310194] ========================================================= [D 20:01:00.310219] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:00.310244] element 0: offset: 0x98447000, size: 2097152 [D 20:01:00.310272] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 6. [D 20:01:00.310299] post_sr: da13:3346 bh 85 len 16 wr_id 15e73378. [D 20:01:00.310326] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 5. [D 20:01:00.310353] post_sr: da13:3346 bh 86 len 16 wr_id 15e73390. [D 20:01:00.310379] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 4. [D 20:01:00.310410] post_sr: da13:3346 bh 87 len 16 wr_id 15e733a8. [D 20:01:00.310437] openib_check_cq: completed send wr_id 15e732d0. [D 20:01:00.310462] openib_check_cq: completed send wr_id 15e732e8. [D 20:01:00.310487] openib_check_cq: completed send wr_id 15e73300. [D 20:01:00.310513] openib_check_cq: completed send wr_id 15e73318. [D 20:01:00.310539] openib_check_cq: completed send wr_id 15e73330. [D 20:01:00.310564] openib_check_cq: completed send wr_id 15e73348. [D 20:01:00.310589] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 3. [D 20:01:00.310616] post_sr: da13:3346 bh 88 len 16 wr_id 15e733c0. [D 20:01:00.310642] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 2. [D 20:01:00.310668] post_sr: da13:3346 bh 89 len 16 wr_id 15e733d8. [D 20:01:00.310695] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 1. [D 20:01:00.310722] post_sr: da13:3346 bh 90 len 16 wr_id 15e733f0. [D 20:01:00.310749] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 0. [D 20:01:00.310775] post_sr: da13:3346 bh 91 len 16 wr_id 15e73408. [D 20:01:00.310801] openib_check_cq: completed send wr_id 15e73360. [D 20:01:00.310827] openib_check_cq: completed send wr_id 15e73378. [D 20:01:00.310852] openib_check_cq: completed send wr_id 15e73390. [D 20:01:00.310877] openib_check_cq: completed send wr_id 15e733a8. [D 20:01:00.310902] openib_check_cq: completed send wr_id 15e733c0. [D 20:01:00.310928] openib_check_cq: completed send wr_id 15e733d8. [D 20:01:00.310953] openib_check_cq: completed send wr_id 15e733f0. [D 20:01:00.310977] openib_check_cq: completed send wr_id 15e73408. [D 20:01:00.311011] flowproto posted 0x15ec2678 [D 20:01:00.311037] Job flows in progress (post time): 22 [D 20:01:00.311064] posted flow for context 0x15ec20c8 [D 20:01:00.311089] *** all msgpairs complete (all flows posted) [D 20:01:00.311113] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.311139] [SM Exiting]: (0x15ec1880) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.311165] sm control op 6 [D 20:01:00.311194] [SM Entering]: (0x15ec4d38) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.311224] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.311253] (0x15ec4dd0) io_datafile_complete_operations (tag 0) [D 20:01:00.311283] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.311309] - complete_context_send_or_recv called [D 20:01:00.311334] got a recv completion with context index 0 [D 20:01:00.311360] - io_process_context_recv called [D 20:01:00.311385] - io_process_context_recv called [D 20:01:00.311408] PINT_decode [D 20:01:00.311433] lebf_decode_resp [D 20:01:00.311458] PINT_encode_release [D 20:01:00.311483] lebf_encode_rel [D 20:01:00.311508] PINT_decode_release [D 20:01:00.311532] lebf_decode_rel [D 20:01:00.311558] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.311584] io_post_flow: entry [D 20:01:00.311608] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.311634] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.334280] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.334330] bstream_size = 471859200, datafile nr=0, ct=1, file_req_off = 201326592 [D 20:01:00.334360] PINT_new_request_states n=1 [D 20:01:00.334385] PINT_new_request_states n=1 [D 20:01:00.334411] flowproto posting 0x15ec5b50 [D 20:01:00.334449] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:00.334476] ========================================================= [D 20:01:00.334501] PINT_process_request [D 20:01:00.334526] tiling 8 copies [D 20:01:00.334549] skipping ahead to target_offset [D 20:01:00.334574] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.334601] lvl 0 el 0 blk 0 by 0 [D 20:01:00.334626] to 0 ta 201326592 fi 234881024 [D 20:01:00.334650] mto 0 mta 0 mfi 33554432 [D 20:01:00.334674] basic type or contiguous data [D 20:01:00.334705] Request disp [D 20:01:00.334730] contig_offset = 0 contig_size = 268435456 lvl_flag = 1 [D 20:01:00.334754] process logical skip [D 20:01:00.334778] retval = 201326592 [D 20:01:00.334802] exiting logical skip because distribute indicates done [D 20:01:00.334826] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.334853] lvl 0 el 0 blk 0 by 201326592 [D 20:01:00.334877] to 201326592 ta 201326592 fi 234881024 [D 20:01:00.334901] mto 0 mta 0 mfi 33554432 [D 20:01:00.334926] basic type or contiguous data [D 20:01:00.334950] Request disp [D 20:01:00.334974] contig_offset = 201326592 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.334999] calling distribute [D 20:01:00.335022] PINT_distribute [D 20:01:00.335046] of 201326592 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:00.335074] begin iteration loff: 201326592 [D 20:01:00.335099] segment exceeds byte limit [D 20:01:00.335123] client lstof 0 diff 0 sgof 0 [D 20:01:00.335147] **********CALL***PROCESS********* [D 20:01:00.335171] segment of 0 sz 2097152 [D 20:01:00.335201] PINT_process_request [D 20:01:00.335228] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.335258] lvl 0 el 0 blk 0 by 0 [D 20:01:00.335288] to 0 ta 0 fi 2097152 [D 20:01:00.335312] basic type or contiguous data [D 20:01:00.335336] Request disp [D 20:01:00.335360] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.335385] process a segment [D 20:01:00.335409] of 0 sz 2097152 [D 20:01:00.335433] add a segment 0 [D 20:01:00.335457] retval = 2097152 [D 20:01:00.335481] exiting distribute returned less than expected [D 20:01:00.335505] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.335531] *****RETURN***FROM***PROCESS***** [D 20:01:00.335556] end iteration [D 20:01:00.335579] done with segments or bytes [D 20:01:00.335603] of 203423744 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.335629] next loff: 203423744 next poff: 203423744 [D 20:01:00.335654] return value 2097152 [D 20:01:00.335679] finished [D 20:01:00.335702] retval = 2097152 [D 20:01:00.335726] exiting distribute returned less than expected [D 20:01:00.335750] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 203423744 fo 234881024 eof 0 [D 20:01:00.335776] ========================================================= [D 20:01:00.335801] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:00.335825] element 0: offset: 0x96445000, size: 2097152 [D 20:01:00.335861] flowproto posted 0x15ec5b50 [D 20:01:00.335886] Job flows in progress (post time): 23 [D 20:01:00.335914] posted flow for context 0x15ec55a0 [D 20:01:00.335939] *** all msgpairs complete (all flows posted) [D 20:01:00.335964] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.335988] [SM Exiting]: (0x15ec4d38) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.336015] sm control op 6 [D 20:01:00.336039] [SM Entering]: (0x15ec8210) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.336065] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.336091] (0x15ec82a8) io_datafile_complete_operations (tag 0) [D 20:01:00.336115] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.336140] - complete_context_send_or_recv called [D 20:01:00.336165] got a recv completion with context index 0 [D 20:01:00.336190] - io_process_context_recv called [D 20:01:00.336215] - io_process_context_recv called [D 20:01:00.336238] PINT_decode [D 20:01:00.336263] lebf_decode_resp [D 20:01:00.336288] PINT_encode_release [D 20:01:00.336312] lebf_encode_rel [D 20:01:00.336338] PINT_decode_release [D 20:01:00.336361] lebf_decode_rel [D 20:01:00.336387] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.336412] io_post_flow: entry [D 20:01:00.336437] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.336464] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.359135] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.359199] bstream_size = 471859200, datafile nr=0, ct=1, file_req_off = 234881024 [D 20:01:00.359234] PINT_new_request_states n=1 [D 20:01:00.359264] PINT_new_request_states n=1 [D 20:01:00.359292] flowproto posting 0x15ec9028 [D 20:01:00.359333] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:00.359359] ========================================================= [D 20:01:00.359384] PINT_process_request [D 20:01:00.359408] tiling 9 copies [D 20:01:00.359433] skipping ahead to target_offset [D 20:01:00.359457] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.359483] lvl 0 el 0 blk 0 by 0 [D 20:01:00.359508] to 0 ta 234881024 fi 268435456 [D 20:01:00.359532] mto 0 mta 0 mfi 33554432 [D 20:01:00.359557] basic type or contiguous data [D 20:01:00.359580] Request disp [D 20:01:00.359605] contig_offset = 0 contig_size = 301989888 lvl_flag = 1 [D 20:01:00.359630] process logical skip [D 20:01:00.359654] retval = 234881024 [D 20:01:00.359678] exiting logical skip because distribute indicates done [D 20:01:00.359703] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.359728] lvl 0 el 0 blk 0 by 234881024 [D 20:01:00.359753] to 234881024 ta 234881024 fi 268435456 [D 20:01:00.359778] mto 0 mta 0 mfi 33554432 [D 20:01:00.359803] basic type or contiguous data [D 20:01:00.359827] Request disp [D 20:01:00.359850] contig_offset = 234881024 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.359875] calling distribute [D 20:01:00.359899] PINT_distribute [D 20:01:00.359923] of 234881024 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:00.359950] begin iteration loff: 234881024 [D 20:01:00.359975] segment exceeds byte limit [D 20:01:00.359999] client lstof 0 diff 0 sgof 0 [D 20:01:00.360024] **********CALL***PROCESS********* [D 20:01:00.360048] segment of 0 sz 2097152 [D 20:01:00.360073] PINT_process_request [D 20:01:00.360096] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.360122] lvl 0 el 0 blk 0 by 0 [D 20:01:00.360147] to 0 ta 0 fi 2097152 [D 20:01:00.360172] basic type or contiguous data [D 20:01:00.360196] Request disp [D 20:01:00.360219] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.360245] process a segment [D 20:01:00.360269] of 0 sz 2097152 [D 20:01:00.360293] add a segment 0 [D 20:01:00.360317] retval = 2097152 [D 20:01:00.360342] exiting distribute returned less than expected [D 20:01:00.360365] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.360392] *****RETURN***FROM***PROCESS***** [D 20:01:00.360417] end iteration [D 20:01:00.360441] done with segments or bytes [D 20:01:00.360465] of 236978176 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.360490] next loff: 236978176 next poff: 236978176 [D 20:01:00.360515] return value 2097152 [D 20:01:00.360539] finished [D 20:01:00.360563] retval = 2097152 [D 20:01:00.360588] exiting distribute returned less than expected [D 20:01:00.360611] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 236978176 fo 268435456 eof 0 [D 20:01:00.360637] ========================================================= [D 20:01:00.360662] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:00.360688] element 0: offset: 0x94443000, size: 2097152 [D 20:01:00.360715] openib_check_cq: completed recv wr_id 15e736b0. [D 20:01:00.360747] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaab3410010 rkey c8005a00 nic_wr_credit 1. [D 20:01:00.360775] openib_post_rr: da13:3346 bh 19 wr_id 15e736b0. [D 20:01:00.360802] openib_check_cq: completed recv wr_id 15e736c8. [D 20:01:00.360833] openib_post_rr: da13:3346 bh 20 wr_id 15e736c8. [D 20:01:00.360859] openib_check_cq: completed recv wr_id 15e736e0. [D 20:01:00.360887] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab3611010 rkey c8005b00 nic_wr_credit 2. [D 20:01:00.360918] openib_post_rr: da13:3346 bh 21 wr_id 15e736e0. [D 20:01:00.360944] openib_check_cq: completed recv wr_id 15e736f8. [D 20:01:00.360972] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab3812010 rkey c8005c00 nic_wr_credit 3. [D 20:01:00.361000] openib_post_rr: da13:3346 bh 22 wr_id 15e736f8. [D 20:01:00.361025] openib_check_cq: completed recv wr_id 15e73710. [D 20:01:00.361053] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab3a13010 rkey c8005d00 nic_wr_credit 4. [D 20:01:00.361080] openib_post_rr: da13:3346 bh 23 wr_id 15e73710. [D 20:01:00.361105] openib_check_cq: completed recv wr_id 15e73728. [D 20:01:00.361133] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab3c14010 rkey c8005e00 nic_wr_credit 5. [D 20:01:00.361160] openib_post_rr: da13:3346 bh 24 wr_id 15e73728. [D 20:01:00.361189] openib_check_cq: completed recv wr_id 15e73740. [D 20:01:00.361216] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaab3e15010 rkey c8005f00 nic_wr_credit 6. [D 20:01:00.361244] openib_post_rr: da13:3346 bh 25 wr_id 15e73740. [D 20:01:00.361269] openib_check_cq: completed recv wr_id 15e73758. [D 20:01:00.361297] openib_post_rr: da13:3346 bh 26 wr_id 15e73758. [D 20:01:00.361326] post_sr: da13:3346 bh 92 len 48 wr_id 15e73420. [D 20:01:00.361352] openib_check_cq: completed recv wr_id 15e73770. [D 20:01:00.361382] openib_post_rr: da13:3346 bh 27 wr_id 15e73770. [D 20:01:00.361408] openib_check_cq: completed recv wr_id 15e73788. [D 20:01:00.361435] openib_post_rr: da13:3346 bh 28 wr_id 15e73788. [D 20:01:00.361462] openib_check_cq: completed recv wr_id 15e737a0. [D 20:01:00.361489] openib_post_rr: da13:3346 bh 29 wr_id 15e737a0. [D 20:01:00.361515] openib_check_cq: completed recv wr_id 15e737b8. [D 20:01:00.361543] openib_post_rr: da13:3346 bh 30 wr_id 15e737b8. [D 20:01:00.361577] flowproto posted 0x15ec9028 [D 20:01:00.361602] Job flows in progress (post time): 24 [D 20:01:00.361630] posted flow for context 0x15ec8a78 [D 20:01:00.361655] *** all msgpairs complete (all flows posted) [D 20:01:00.361680] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.361705] [SM Exiting]: (0x15ec8210) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.361732] sm control op 6 [D 20:01:00.361758] [SM Entering]: (0x15ecb6e8) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.361783] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.361809] (0x15ecb780) io_datafile_complete_operations (tag 0) [D 20:01:00.361834] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.361860] - complete_context_send_or_recv called [D 20:01:00.361884] got a recv completion with context index 0 [D 20:01:00.361909] - io_process_context_recv called [D 20:01:00.361933] - io_process_context_recv called [D 20:01:00.361958] PINT_decode [D 20:01:00.361984] lebf_decode_resp [D 20:01:00.362009] PINT_encode_release [D 20:01:00.362033] lebf_encode_rel [D 20:01:00.362058] PINT_decode_release [D 20:01:00.362082] lebf_decode_rel [D 20:01:00.362108] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.362133] io_post_flow: entry [D 20:01:00.362158] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.362183] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.384778] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.384829] bstream_size = 476053504, datafile nr=0, ct=1, file_req_off = 268435456 [D 20:01:00.384860] PINT_new_request_states n=1 [D 20:01:00.384886] PINT_new_request_states n=1 [D 20:01:00.384911] flowproto posting 0x15ecc500 [D 20:01:00.384950] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:00.384977] ========================================================= [D 20:01:00.385002] PINT_process_request [D 20:01:00.385030] tiling 10 copies [D 20:01:00.385054] skipping ahead to target_offset [D 20:01:00.385079] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.385105] lvl 0 el 0 blk 0 by 0 [D 20:01:00.385129] to 0 ta 268435456 fi 301989888 [D 20:01:00.385154] mto 0 mta 0 mfi 33554432 [D 20:01:00.385182] basic type or contiguous data [D 20:01:00.385207] Request disp [D 20:01:00.385231] contig_offset = 0 contig_size = 335544320 lvl_flag = 1 [D 20:01:00.385255] process logical skip [D 20:01:00.385279] retval = 268435456 [D 20:01:00.385303] exiting logical skip because distribute indicates done [D 20:01:00.385328] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.385354] lvl 0 el 0 blk 0 by 268435456 [D 20:01:00.385377] to 268435456 ta 268435456 fi 301989888 [D 20:01:00.385402] mto 0 mta 0 mfi 33554432 [D 20:01:00.385427] basic type or contiguous data [D 20:01:00.385451] Request disp [D 20:01:00.385474] contig_offset = 268435456 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.385499] calling distribute [D 20:01:00.385522] PINT_distribute [D 20:01:00.385546] of 268435456 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:00.385573] begin iteration loff: 268435456 [D 20:01:00.385599] segment exceeds byte limit [D 20:01:00.385622] client lstof 0 diff 0 sgof 0 [D 20:01:00.385647] **********CALL***PROCESS********* [D 20:01:00.385671] segment of 0 sz 2097152 [D 20:01:00.385695] PINT_process_request [D 20:01:00.385719] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.385746] lvl 0 el 0 blk 0 by 0 [D 20:01:00.385769] to 0 ta 0 fi 2097152 [D 20:01:00.385794] basic type or contiguous data [D 20:01:00.385818] Request disp [D 20:01:00.385842] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.385867] process a segment [D 20:01:00.385890] of 0 sz 2097152 [D 20:01:00.385915] add a segment 0 [D 20:01:00.385939] retval = 2097152 [D 20:01:00.385963] exiting distribute returned less than expected [D 20:01:00.385987] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.386013] *****RETURN***FROM***PROCESS***** [D 20:01:00.386037] end iteration [D 20:01:00.386061] done with segments or bytes [D 20:01:00.386085] of 270532608 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.386111] next loff: 270532608 next poff: 270532608 [D 20:01:00.386136] return value 2097152 [D 20:01:00.386159] finished [D 20:01:00.386183] retval = 2097152 [D 20:01:00.386207] exiting distribute returned less than expected [D 20:01:00.386232] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 270532608 fo 301989888 eof 0 [D 20:01:00.386258] ========================================================= [D 20:01:00.386282] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:00.386308] element 0: offset: 0x92441000, size: 2097152 [D 20:01:00.386336] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 5. [D 20:01:00.386363] post_sr: da13:3346 bh 93 len 16 wr_id 15e73438. [D 20:01:00.386391] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 4. [D 20:01:00.386418] post_sr: da13:3346 bh 94 len 16 wr_id 15e73450. [D 20:01:00.386445] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 3. [D 20:01:00.386471] post_sr: da13:3346 bh 95 len 16 wr_id 15e73468. [D 20:01:00.386497] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 2. [D 20:01:00.386525] post_sr: da13:3346 bh 96 len 16 wr_id 15e73480. [D 20:01:00.386550] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 1. [D 20:01:00.386577] post_sr: da13:3346 bh 97 len 16 wr_id 15e73498. [D 20:01:00.386604] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 0. [D 20:01:00.386630] post_sr: da13:3346 bh 98 len 16 wr_id 15e734b0. [D 20:01:00.386656] openib_check_cq: completed send wr_id 15e73420. [D 20:01:00.386682] openib_check_cq: completed send wr_id 15e73438. [D 20:01:00.386707] openib_check_cq: completed send wr_id 15e73450. [D 20:01:00.386734] openib_check_cq: completed send wr_id 15e73468. [D 20:01:00.386760] openib_check_cq: completed send wr_id 15e73480. [D 20:01:00.386786] openib_check_cq: completed send wr_id 15e73498. [D 20:01:00.386811] openib_check_cq: completed send wr_id 15e734b0. [D 20:01:00.386849] flowproto posted 0x15ecc500 [D 20:01:00.386874] Job flows in progress (post time): 25 [D 20:01:00.386902] posted flow for context 0x15ecbf50 [D 20:01:00.386927] *** all msgpairs complete (all flows posted) [D 20:01:00.386952] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.386978] [SM Exiting]: (0x15ecb6e8) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.387004] sm control op 6 [D 20:01:00.387029] [SM Entering]: (0x15ecebc0) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.387055] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.387081] (0x15ecec58) io_datafile_complete_operations (tag 0) [D 20:01:00.387105] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.387131] - complete_context_send_or_recv called [D 20:01:00.387155] got a recv completion with context index 0 [D 20:01:00.387183] - io_process_context_recv called [D 20:01:00.387212] - io_process_context_recv called [D 20:01:00.387240] PINT_decode [D 20:01:00.387269] lebf_decode_resp [D 20:01:00.387297] PINT_encode_release [D 20:01:00.387322] lebf_encode_rel [D 20:01:00.387348] PINT_decode_release [D 20:01:00.387372] lebf_decode_rel [D 20:01:00.387397] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.387423] io_post_flow: entry [D 20:01:00.387447] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.387473] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.410246] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.410297] bstream_size = 476053504, datafile nr=0, ct=1, file_req_off = 301989888 [D 20:01:00.410327] PINT_new_request_states n=1 [D 20:01:00.410354] PINT_new_request_states n=1 [D 20:01:00.410379] flowproto posting 0x15ecf9d8 [D 20:01:00.410416] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:00.410443] ========================================================= [D 20:01:00.410468] PINT_process_request [D 20:01:00.410491] tiling 11 copies [D 20:01:00.410516] skipping ahead to target_offset [D 20:01:00.410540] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.410566] lvl 0 el 0 blk 0 by 0 [D 20:01:00.410591] to 0 ta 301989888 fi 335544320 [D 20:01:00.410615] mto 0 mta 0 mfi 33554432 [D 20:01:00.410640] basic type or contiguous data [D 20:01:00.410664] Request disp [D 20:01:00.410688] contig_offset = 0 contig_size = 369098752 lvl_flag = 1 [D 20:01:00.410713] process logical skip [D 20:01:00.410737] retval = 301989888 [D 20:01:00.410760] exiting logical skip because distribute indicates done [D 20:01:00.410784] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.410810] lvl 0 el 0 blk 0 by 301989888 [D 20:01:00.410835] to 301989888 ta 301989888 fi 335544320 [D 20:01:00.410860] mto 0 mta 0 mfi 33554432 [D 20:01:00.410884] basic type or contiguous data [D 20:01:00.410908] Request disp [D 20:01:00.410932] contig_offset = 301989888 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.410957] calling distribute [D 20:01:00.410982] PINT_distribute [D 20:01:00.411005] of 301989888 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:00.411032] begin iteration loff: 301989888 [D 20:01:00.411057] segment exceeds byte limit [D 20:01:00.411081] client lstof 0 diff 0 sgof 0 [D 20:01:00.411106] **********CALL***PROCESS********* [D 20:01:00.411129] segment of 0 sz 2097152 [D 20:01:00.411154] PINT_process_request [D 20:01:00.411178] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.411211] lvl 0 el 0 blk 0 by 0 [D 20:01:00.411239] to 0 ta 0 fi 2097152 [D 20:01:00.411268] basic type or contiguous data [D 20:01:00.411299] Request disp [D 20:01:00.411324] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.411349] process a segment [D 20:01:00.411373] of 0 sz 2097152 [D 20:01:00.411397] add a segment 0 [D 20:01:00.411421] retval = 2097152 [D 20:01:00.411446] exiting distribute returned less than expected [D 20:01:00.411470] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.411497] *****RETURN***FROM***PROCESS***** [D 20:01:00.411521] end iteration [D 20:01:00.411545] done with segments or bytes [D 20:01:00.411569] of 304087040 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.411595] next loff: 304087040 next poff: 304087040 [D 20:01:00.411620] return value 2097152 [D 20:01:00.411644] finished [D 20:01:00.411668] retval = 2097152 [D 20:01:00.411692] exiting distribute returned less than expected [D 20:01:00.411717] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 304087040 fo 335544320 eof 0 [D 20:01:00.411743] ========================================================= [D 20:01:00.411767] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:00.411792] element 0: offset: 0x9043f000, size: 2097152 [D 20:01:00.411828] flowproto posted 0x15ecf9d8 [D 20:01:00.411853] Job flows in progress (post time): 26 [D 20:01:00.411881] posted flow for context 0x15ecf428 [D 20:01:00.411906] *** all msgpairs complete (all flows posted) [D 20:01:00.411931] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.411955] [SM Exiting]: (0x15ecebc0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.411983] PINT_client_state_machine_test id 22 [D 20:01:00.412014] sm control op 300 [D 20:01:00.412038] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:01:00.412065] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:01:00.412094] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:01:00.412120] PINT_client_state_machine_test id 22 [D 20:01:00.412163] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:01:00.412189] ========================================================= [D 20:01:00.412214] PINT_process_request [D 20:01:00.412238] tiling 6 copies [D 20:01:00.412262] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.412288] lvl 0 el 0 blk 0 by 159383552 [D 20:01:00.412313] to 159383552 ta 134217728 fi 167772160 [D 20:01:00.412341] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.412365] basic type or contiguous data [D 20:01:00.412390] Request disp [D 20:01:00.412414] contig_offset = 159383552 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.412439] calling distribute [D 20:01:00.412463] PINT_distribute [D 20:01:00.412487] of 159383552 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 159383552 exfl 1 [D 20:01:00.412513] begin iteration loff: 159383552 [D 20:01:00.412538] segment exceeds byte limit [D 20:01:00.412562] file being extended [D 20:01:00.412586] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.412611] **********CALL***PROCESS********* [D 20:01:00.412634] segment of 25165824 sz 2097152 [D 20:01:00.412659] PINT_process_request [D 20:01:00.412683] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.412709] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.412734] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.412759] basic type or contiguous data [D 20:01:00.412783] Request disp [D 20:01:00.412806] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.412832] process a segment [D 20:01:00.412856] of 25165824 sz 2097152 [D 20:01:00.412879] add a segment 0 [D 20:01:00.412903] retval = 2097152 [D 20:01:00.412927] exiting distribute returned less than expected [D 20:01:00.412952] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.412978] *****RETURN***FROM***PROCESS***** [D 20:01:00.413002] end iteration [D 20:01:00.413028] done with segments or bytes [D 20:01:00.413053] of 161480704 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.413079] next loff: 161480704 next poff: 161480704 [D 20:01:00.413104] return value 2097152 [D 20:01:00.413127] finished [D 20:01:00.413151] retval = 2097152 [D 20:01:00.413175] exiting distribute returned less than expected [D 20:01:00.413202] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 161480704 fo 167772160 eof 0 [D 20:01:00.413228] ========================================================= [D 20:01:00.413252] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:01:00.413278] element 0: offset: 0xbbc69000, size: 2097152 [D 20:01:00.413307] post_sr: da13:3346 bh 99 len 32 wr_id 15e734c8. [D 20:01:00.413336] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:01:00.413362] ========================================================= [D 20:01:00.413387] PINT_process_request [D 20:01:00.413411] tiling 7 copies [D 20:01:00.413434] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.413460] lvl 0 el 0 blk 0 by 192937984 [D 20:01:00.413486] to 192937984 ta 167772160 fi 201326592 [D 20:01:00.413511] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.413536] basic type or contiguous data [D 20:01:00.413559] Request disp [D 20:01:00.413583] contig_offset = 192937984 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.413608] calling distribute [D 20:01:00.413633] PINT_distribute [D 20:01:00.413657] of 192937984 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 192937984 exfl 1 [D 20:01:00.413683] begin iteration loff: 192937984 [D 20:01:00.413708] segment exceeds byte limit [D 20:01:00.413732] file being extended [D 20:01:00.413756] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.413781] **********CALL***PROCESS********* [D 20:01:00.413805] segment of 25165824 sz 2097152 [D 20:01:00.413829] PINT_process_request [D 20:01:00.413853] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.413879] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.413905] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.413929] basic type or contiguous data [D 20:01:00.413953] Request disp [D 20:01:00.413977] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.414002] process a segment [D 20:01:00.414025] of 25165824 sz 2097152 [D 20:01:00.414050] add a segment 0 [D 20:01:00.414073] retval = 2097152 [D 20:01:00.414097] exiting distribute returned less than expected [D 20:01:00.414121] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.414147] *****RETURN***FROM***PROCESS***** [D 20:01:00.414172] end iteration [D 20:01:00.414196] done with segments or bytes [D 20:01:00.414220] of 195035136 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.414246] next loff: 195035136 next poff: 195035136 [D 20:01:00.414271] return value 2097152 [D 20:01:00.414295] finished [D 20:01:00.414319] retval = 2097152 [D 20:01:00.414342] exiting distribute returned less than expected [D 20:01:00.414366] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 195035136 fo 201326592 eof 0 [D 20:01:00.414393] ========================================================= [D 20:01:00.414418] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:01:00.414443] element 0: offset: 0xb9c67000, size: 2097152 [D 20:01:00.414470] post_sr: da13:3346 bh 0 len 32 wr_id 15e72b80. [D 20:01:00.414499] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:01:00.414525] ========================================================= [D 20:01:00.414549] PINT_process_request [D 20:01:00.414573] tiling 8 copies [D 20:01:00.414598] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.414624] lvl 0 el 0 blk 0 by 226492416 [D 20:01:00.414649] to 226492416 ta 201326592 fi 234881024 [D 20:01:00.414674] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.414702] basic type or contiguous data [D 20:01:00.414726] Request disp [D 20:01:00.414751] contig_offset = 226492416 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.414775] calling distribute [D 20:01:00.414799] PINT_distribute [D 20:01:00.414823] of 226492416 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 226492416 exfl 1 [D 20:01:00.414850] begin iteration loff: 226492416 [D 20:01:00.414875] segment exceeds byte limit [D 20:01:00.414899] file being extended [D 20:01:00.414923] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.414948] **********CALL***PROCESS********* [D 20:01:00.414972] segment of 25165824 sz 2097152 [D 20:01:00.414997] PINT_process_request [D 20:01:00.415020] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.415046] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.415071] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.415096] basic type or contiguous data [D 20:01:00.415120] Request disp [D 20:01:00.415143] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.415168] process a segment [D 20:01:00.415195] of 25165824 sz 2097152 [D 20:01:00.415224] add a segment 0 [D 20:01:00.415250] retval = 2097152 [D 20:01:00.415278] exiting distribute returned less than expected [D 20:01:00.415305] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.415331] *****RETURN***FROM***PROCESS***** [D 20:01:00.415355] end iteration [D 20:01:00.415379] done with segments or bytes [D 20:01:00.415403] of 228589568 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.415429] next loff: 228589568 next poff: 228589568 [D 20:01:00.415453] return value 2097152 [D 20:01:00.415478] finished [D 20:01:00.415502] retval = 2097152 [D 20:01:00.415526] exiting distribute returned less than expected [D 20:01:00.415550] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 228589568 fo 234881024 eof 0 [D 20:01:00.415576] ========================================================= [D 20:01:00.415600] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:01:00.415625] element 0: offset: 0xb7c65000, size: 2097152 [D 20:01:00.415653] post_sr: da13:3346 bh 1 len 32 wr_id 15e72b98. [D 20:01:00.415682] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:01:00.415708] ========================================================= [D 20:01:00.415733] PINT_process_request [D 20:01:00.415757] tiling 9 copies [D 20:01:00.415781] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.415807] lvl 0 el 0 blk 0 by 260046848 [D 20:01:00.415832] to 260046848 ta 234881024 fi 268435456 [D 20:01:00.415858] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.415882] basic type or contiguous data [D 20:01:00.415906] Request disp [D 20:01:00.415930] contig_offset = 260046848 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.415955] calling distribute [D 20:01:00.415979] PINT_distribute [D 20:01:00.416003] of 260046848 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 260046848 exfl 1 [D 20:01:00.416028] begin iteration loff: 260046848 [D 20:01:00.416053] segment exceeds byte limit [D 20:01:00.416077] file being extended [D 20:01:00.416101] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.416126] **********CALL***PROCESS********* [D 20:01:00.416150] segment of 25165824 sz 2097152 [D 20:01:00.416174] PINT_process_request [D 20:01:00.416198] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.416224] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.416250] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.416274] basic type or contiguous data [D 20:01:00.416298] Request disp [D 20:01:00.416322] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.416347] process a segment [D 20:01:00.416371] of 25165824 sz 2097152 [D 20:01:00.416396] add a segment 0 [D 20:01:00.416419] retval = 2097152 [D 20:01:00.416443] exiting distribute returned less than expected [D 20:01:00.416467] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.416496] *****RETURN***FROM***PROCESS***** [D 20:01:00.416521] end iteration [D 20:01:00.416544] done with segments or bytes [D 20:01:00.416568] of 262144000 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.416594] next loff: 262144000 next poff: 262144000 [D 20:01:00.416619] return value 2097152 [D 20:01:00.416644] finished [D 20:01:00.416666] retval = 2097152 [D 20:01:00.416691] exiting distribute returned less than expected [D 20:01:00.416715] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 262144000 fo 268435456 eof 0 [D 20:01:00.416742] ========================================================= [D 20:01:00.416766] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:01:00.416790] element 0: offset: 0xb5c63000, size: 2097152 [D 20:01:00.416818] post_sr: da13:3346 bh 2 len 32 wr_id 15e72bb0. [D 20:01:00.416848] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:01:00.416873] ========================================================= [D 20:01:00.416897] PINT_process_request [D 20:01:00.416922] tiling 10 copies [D 20:01:00.416946] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.416972] lvl 0 el 0 blk 0 by 293601280 [D 20:01:00.416996] to 293601280 ta 268435456 fi 301989888 [D 20:01:00.417021] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.417047] basic type or contiguous data [D 20:01:00.417071] Request disp [D 20:01:00.417095] contig_offset = 293601280 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.417119] calling distribute [D 20:01:00.417143] PINT_distribute [D 20:01:00.417167] of 293601280 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 293601280 exfl 1 [D 20:01:00.417197] begin iteration loff: 293601280 [D 20:01:00.417222] segment exceeds byte limit [D 20:01:00.417245] file being extended [D 20:01:00.417269] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.417294] **********CALL***PROCESS********* [D 20:01:00.417318] segment of 25165824 sz 2097152 [D 20:01:00.417343] PINT_process_request [D 20:01:00.417366] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.417392] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.417417] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.417442] basic type or contiguous data [D 20:01:00.417466] Request disp [D 20:01:00.417490] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.417514] process a segment [D 20:01:00.417538] of 25165824 sz 2097152 [D 20:01:00.417563] add a segment 0 [D 20:01:00.417587] retval = 2097152 [D 20:01:00.417611] exiting distribute returned less than expected [D 20:01:00.417634] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.417661] *****RETURN***FROM***PROCESS***** [D 20:01:00.417685] end iteration [D 20:01:00.417709] done with segments or bytes [D 20:01:00.417733] of 295698432 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.417758] next loff: 295698432 next poff: 295698432 [D 20:01:00.417783] return value 2097152 [D 20:01:00.417807] finished [D 20:01:00.417831] retval = 2097152 [D 20:01:00.417855] exiting distribute returned less than expected [D 20:01:00.417879] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 295698432 fo 301989888 eof 0 [D 20:01:00.417905] ========================================================= [D 20:01:00.417929] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:01:00.417955] element 0: offset: 0xb3c61000, size: 2097152 [D 20:01:00.417983] post_sr: da13:3346 bh 3 len 32 wr_id 15e72bc8. [D 20:01:00.418013] openib_check_cq: completed send wr_id 15e734c8. [D 20:01:00.418039] openib_check_cq: completed send wr_id 15e72b80. [D 20:01:00.418065] openib_check_cq: completed send wr_id 15e72b98. [D 20:01:00.418089] openib_check_cq: completed send wr_id 15e72bb0. [D 20:01:00.418115] openib_check_cq: completed send wr_id 15e72bc8. [D 20:01:00.418151] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:01:00.418178] ========================================================= [D 20:01:00.418202] PINT_process_request [D 20:01:00.418227] tiling 11 copies [D 20:01:00.418251] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.418277] lvl 0 el 0 blk 0 by 327155712 [D 20:01:00.418302] to 327155712 ta 301989888 fi 335544320 [D 20:01:00.418327] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.418352] basic type or contiguous data [D 20:01:00.418376] Request disp [D 20:01:00.418400] contig_offset = 327155712 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.418425] calling distribute [D 20:01:00.418449] PINT_distribute [D 20:01:00.418473] of 327155712 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 327155712 exfl 1 [D 20:01:00.418499] begin iteration loff: 327155712 [D 20:01:00.418524] segment exceeds byte limit [D 20:01:00.418548] file being extended [D 20:01:00.418572] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.418597] **********CALL***PROCESS********* [D 20:01:00.418621] segment of 25165824 sz 2097152 [D 20:01:00.418645] PINT_process_request [D 20:01:00.418669] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.418695] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.418720] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.418745] basic type or contiguous data [D 20:01:00.418768] Request disp [D 20:01:00.418792] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.418817] process a segment [D 20:01:00.418841] of 25165824 sz 2097152 [D 20:01:00.418866] add a segment 0 [D 20:01:00.418889] retval = 2097152 [D 20:01:00.418913] exiting distribute returned less than expected [D 20:01:00.418937] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.418963] *****RETURN***FROM***PROCESS***** [D 20:01:00.418988] end iteration [D 20:01:00.419012] done with segments or bytes [D 20:01:00.419035] of 329252864 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.419061] next loff: 329252864 next poff: 329252864 [D 20:01:00.419086] return value 2097152 [D 20:01:00.419110] finished [D 20:01:00.419134] retval = 2097152 [D 20:01:00.419157] exiting distribute returned less than expected [D 20:01:00.419184] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 329252864 fo 335544320 eof 0 [D 20:01:00.419215] ========================================================= [D 20:01:00.419243] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:01:00.419272] element 0: offset: 0xb1c5f000, size: 2097152 [D 20:01:00.419302] post_sr: da13:3346 bh 4 len 32 wr_id 15e72be0. [D 20:01:00.419332] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:01:00.419358] ========================================================= [D 20:01:00.419382] PINT_process_request [D 20:01:00.419406] tiling 12 copies [D 20:01:00.419430] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.419456] lvl 0 el 0 blk 0 by 360710144 [D 20:01:00.419481] to 360710144 ta 335544320 fi 369098752 [D 20:01:00.419506] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.419530] basic type or contiguous data [D 20:01:00.419554] Request disp [D 20:01:00.419578] contig_offset = 360710144 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.419604] calling distribute [D 20:01:00.419628] PINT_distribute [D 20:01:00.419651] of 360710144 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 360710144 exfl 1 [D 20:01:00.419678] begin iteration loff: 360710144 [D 20:01:00.419703] segment exceeds byte limit [D 20:01:00.419727] file being extended [D 20:01:00.419751] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.419775] **********CALL***PROCESS********* [D 20:01:00.419799] segment of 25165824 sz 2097152 [D 20:01:00.419823] PINT_process_request [D 20:01:00.419848] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.419877] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.419901] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.419926] basic type or contiguous data [D 20:01:00.419951] Request disp [D 20:01:00.419974] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.420000] process a segment [D 20:01:00.420022] of 25165824 sz 2097152 [D 20:01:00.420047] add a segment 0 [D 20:01:00.420071] retval = 2097152 [D 20:01:00.420095] exiting distribute returned less than expected [D 20:01:00.420119] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.420145] *****RETURN***FROM***PROCESS***** [D 20:01:00.420169] end iteration [D 20:01:00.420193] done with segments or bytes [D 20:01:00.420217] of 362807296 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.420243] next loff: 362807296 next poff: 362807296 [D 20:01:00.420268] return value 2097152 [D 20:01:00.420291] finished [D 20:01:00.420315] retval = 2097152 [D 20:01:00.420339] exiting distribute returned less than expected [D 20:01:00.420363] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 362807296 fo 369098752 eof 0 [D 20:01:00.420390] ========================================================= [D 20:01:00.420413] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:01:00.420439] element 0: offset: 0xafc5d000, size: 2097152 [D 20:01:00.420466] post_sr: da13:3346 bh 5 len 32 wr_id 15e72bf8. [D 20:01:00.420495] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:01:00.420521] ========================================================= [D 20:01:00.420545] PINT_process_request [D 20:01:00.420569] tiling 13 copies [D 20:01:00.420594] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.420620] lvl 0 el 0 blk 0 by 394264576 [D 20:01:00.420645] to 394264576 ta 369098752 fi 402653184 [D 20:01:00.420670] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.420695] basic type or contiguous data [D 20:01:00.420719] Request disp [D 20:01:00.420743] contig_offset = 394264576 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.420768] calling distribute [D 20:01:00.420792] PINT_distribute [D 20:01:00.420816] of 394264576 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 394264576 exfl 1 [D 20:01:00.420843] begin iteration loff: 394264576 [D 20:01:00.420867] segment exceeds byte limit [D 20:01:00.420891] file being extended [D 20:01:00.420915] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.420940] **********CALL***PROCESS********* [D 20:01:00.420964] segment of 25165824 sz 2097152 [D 20:01:00.420988] PINT_process_request [D 20:01:00.421011] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.421037] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.421063] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.421088] basic type or contiguous data [D 20:01:00.421112] Request disp [D 20:01:00.421135] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.421160] process a segment [D 20:01:00.421188] of 25165824 sz 2097152 [D 20:01:00.421213] add a segment 0 [D 20:01:00.421236] retval = 2097152 [D 20:01:00.421260] exiting distribute returned less than expected [D 20:01:00.421284] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.421311] *****RETURN***FROM***PROCESS***** [D 20:01:00.421335] end iteration [D 20:01:00.421359] done with segments or bytes [D 20:01:00.421382] of 396361728 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.421408] next loff: 396361728 next poff: 396361728 [D 20:01:00.421433] return value 2097152 [D 20:01:00.421457] finished [D 20:01:00.421481] retval = 2097152 [D 20:01:00.421504] exiting distribute returned less than expected [D 20:01:00.421528] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 396361728 fo 402653184 eof 0 [D 20:01:00.421555] ========================================================= [D 20:01:00.421580] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:01:00.421607] element 0: offset: 0xadc5b000, size: 2097152 [D 20:01:00.421635] post_sr: da13:3346 bh 6 len 32 wr_id 15e72c10. [D 20:01:00.421664] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:00.421689] ========================================================= [D 20:01:00.421714] PINT_process_request [D 20:01:00.421739] tiling 16 copies [D 20:01:00.421763] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.421789] lvl 0 el 0 blk 0 by 486539264 [D 20:01:00.421813] to 486539264 ta 469762048 fi 503316480 [D 20:01:00.421838] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:00.421863] basic type or contiguous data [D 20:01:00.421888] Request disp [D 20:01:00.421912] contig_offset = 486539264 contig_size = 50331648 lvl_flag = 1 [D 20:01:00.421936] calling distribute [D 20:01:00.421960] PINT_distribute [D 20:01:00.421984] of 486539264 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 486539264 exfl 1 [D 20:01:00.422011] begin iteration loff: 486539264 [D 20:01:00.422036] segment exceeds byte limit [D 20:01:00.422060] file being extended [D 20:01:00.422082] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:00.422107] **********CALL***PROCESS********* [D 20:01:00.422131] segment of 16777216 sz 2097152 [D 20:01:00.422156] PINT_process_request [D 20:01:00.422180] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.422205] lvl 0 el 0 blk 0 by 16777216 [D 20:01:00.422230] to 16777216 ta 16777216 fi 18874368 [D 20:01:00.422255] basic type or contiguous data [D 20:01:00.422279] Request disp [D 20:01:00.422303] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:00.422330] process a segment [D 20:01:00.422354] of 16777216 sz 2097152 [D 20:01:00.422378] add a segment 0 [D 20:01:00.422402] retval = 2097152 [D 20:01:00.422427] exiting distribute returned less than expected [D 20:01:00.422451] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:00.422476] *****RETURN***FROM***PROCESS***** [D 20:01:00.422501] end iteration [D 20:01:00.422524] done with segments or bytes [D 20:01:00.422548] of 488636416 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.422574] next loff: 488636416 next poff: 488636416 [D 20:01:00.422598] return value 2097152 [D 20:01:00.422622] finished [D 20:01:00.422646] retval = 2097152 [D 20:01:00.422670] exiting distribute returned less than expected [D 20:01:00.422694] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 488636416 fo 503316480 eof 0 [D 20:01:00.422720] ========================================================= [D 20:01:00.422745] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:00.422770] element 0: offset: 0xa7455000, size: 2097152 [D 20:01:00.422798] post_sr: da13:3346 bh 7 len 32 wr_id 15e72c28. [D 20:01:00.422827] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:01:00.422853] ========================================================= [D 20:01:00.422877] PINT_process_request [D 20:01:00.422902] tiling 14 copies [D 20:01:00.422925] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.422951] lvl 0 el 0 blk 0 by 427819008 [D 20:01:00.422976] to 427819008 ta 402653184 fi 436207616 [D 20:01:00.423001] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.423026] basic type or contiguous data [D 20:01:00.423050] Request disp [D 20:01:00.423074] contig_offset = 427819008 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.423099] calling distribute [D 20:01:00.423123] PINT_distribute [D 20:01:00.423146] of 427819008 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 427819008 exfl 1 [D 20:01:00.423172] begin iteration loff: 427819008 [D 20:01:00.423202] segment exceeds byte limit [D 20:01:00.423230] file being extended [D 20:01:00.423256] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.423286] **********CALL***PROCESS********* [D 20:01:00.423310] segment of 25165824 sz 2097152 [D 20:01:00.423337] PINT_process_request [D 20:01:00.423362] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.423388] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.423413] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.423438] basic type or contiguous data [D 20:01:00.423461] Request disp [D 20:01:00.423485] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.423510] process a segment [D 20:01:00.423534] of 25165824 sz 2097152 [D 20:01:00.423558] add a segment 0 [D 20:01:00.423583] retval = 2097152 [D 20:01:00.423606] exiting distribute returned less than expected [D 20:01:00.423630] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.423657] *****RETURN***FROM***PROCESS***** [D 20:01:00.423681] end iteration [D 20:01:00.423705] done with segments or bytes [D 20:01:00.423728] of 429916160 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.423754] next loff: 429916160 next poff: 429916160 [D 20:01:00.423779] return value 2097152 [D 20:01:00.423803] finished [D 20:01:00.423827] retval = 2097152 [D 20:01:00.423851] exiting distribute returned less than expected [D 20:01:00.423874] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 429916160 fo 436207616 eof 0 [D 20:01:00.423901] ========================================================= [D 20:01:00.423926] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:01:00.423951] element 0: offset: 0xabc59000, size: 2097152 [D 20:01:00.423978] post_sr: da13:3346 bh 8 len 32 wr_id 15e72c40. [D 20:01:00.424008] PINT_client_state_machine_test id 22 [D 20:01:00.424036] openib_check_cq: completed send wr_id 15e72be0. [D 20:01:00.424062] openib_check_cq: completed send wr_id 15e72bf8. [D 20:01:00.424088] openib_check_cq: completed send wr_id 15e72c10. [D 20:01:00.424113] openib_check_cq: completed send wr_id 15e72c28. [D 20:01:00.424138] openib_check_cq: completed send wr_id 15e72c40. [D 20:01:00.424170] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:01:00.424197] ========================================================= [D 20:01:00.424221] PINT_process_request [D 20:01:00.424245] tiling 15 copies [D 20:01:00.424269] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.424295] lvl 0 el 0 blk 0 by 461373440 [D 20:01:00.424320] to 461373440 ta 436207616 fi 469762048 [D 20:01:00.424345] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:00.424369] basic type or contiguous data [D 20:01:00.424393] Request disp [D 20:01:00.424417] contig_offset = 461373440 contig_size = 41943040 lvl_flag = 1 [D 20:01:00.424442] calling distribute [D 20:01:00.424466] PINT_distribute [D 20:01:00.424489] of 461373440 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 461373440 exfl 1 [D 20:01:00.424516] begin iteration loff: 461373440 [D 20:01:00.424541] segment exceeds byte limit [D 20:01:00.424566] file being extended [D 20:01:00.424589] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:00.424613] **********CALL***PROCESS********* [D 20:01:00.424637] segment of 25165824 sz 2097152 [D 20:01:00.424662] PINT_process_request [D 20:01:00.424686] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.424713] lvl 0 el 0 blk 0 by 25165824 [D 20:01:00.424737] to 25165824 ta 25165824 fi 27262976 [D 20:01:00.424762] basic type or contiguous data [D 20:01:00.424786] Request disp [D 20:01:00.424810] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:00.424835] process a segment [D 20:01:00.424859] of 25165824 sz 2097152 [D 20:01:00.424882] add a segment 0 [D 20:01:00.424906] retval = 2097152 [D 20:01:00.424930] exiting distribute returned less than expected [D 20:01:00.424954] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:00.424981] *****RETURN***FROM***PROCESS***** [D 20:01:00.425004] end iteration [D 20:01:00.425028] done with segments or bytes [D 20:01:00.425059] of 463470592 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.425086] next loff: 463470592 next poff: 463470592 [D 20:01:00.425110] return value 2097152 [D 20:01:00.425134] finished [D 20:01:00.425158] retval = 2097152 [D 20:01:00.425185] exiting distribute returned less than expected [D 20:01:00.425209] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 463470592 fo 469762048 eof 0 [D 20:01:00.425235] ========================================================= [D 20:01:00.425260] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:01:00.425285] element 0: offset: 0xa9c57000, size: 2097152 [D 20:01:00.425312] post_sr: da13:3346 bh 9 len 32 wr_id 15e72c58. [D 20:01:00.425342] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:01:00.425368] ========================================================= [D 20:01:00.425393] PINT_process_request [D 20:01:00.425416] tiling 2 copies [D 20:01:00.425440] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.425466] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.425491] to 27262976 ta 0 fi 33554432 [D 20:01:00.425516] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.425540] basic type or contiguous data [D 20:01:00.425564] Request disp [D 20:01:00.425589] contig_offset = 27262976 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.425614] calling distribute [D 20:01:00.425638] PINT_distribute [D 20:01:00.425662] of 27262976 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 27262976 exfl 1 [D 20:01:00.425688] begin iteration loff: 27262976 [D 20:01:00.425713] segment exceeds byte limit [D 20:01:00.425737] file being extended [D 20:01:00.425761] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.425786] **********CALL***PROCESS********* [D 20:01:00.425809] segment of 27262976 sz 2097152 [D 20:01:00.425833] PINT_process_request [D 20:01:00.425857] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.425884] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.425909] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.425932] basic type or contiguous data [D 20:01:00.425956] Request disp [D 20:01:00.425980] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.426005] process a segment [D 20:01:00.426029] of 27262976 sz 2097152 [D 20:01:00.426054] add a segment 0 [D 20:01:00.426076] retval = 2097152 [D 20:01:00.426100] exiting distribute returned less than expected [D 20:01:00.426124] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.426151] *****RETURN***FROM***PROCESS***** [D 20:01:00.426175] end iteration [D 20:01:00.426198] done with segments or bytes [D 20:01:00.426222] of 29360128 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.426248] next loff: 29360128 next poff: 29360128 [D 20:01:00.426272] return value 2097152 [D 20:01:00.426297] finished [D 20:01:00.426321] retval = 2097152 [D 20:01:00.426344] exiting distribute returned less than expected [D 20:01:00.426368] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 29360128 fo 33554432 eof 0 [D 20:01:00.426395] ========================================================= [D 20:01:00.426419] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:01:00.426444] element 0: offset: 0xc3e71000, size: 2097152 [D 20:01:00.426471] post_sr: da13:3346 bh 10 len 32 wr_id 15e72c70. [D 20:01:00.426501] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:01:00.426527] ========================================================= [D 20:01:00.426550] PINT_process_request [D 20:01:00.426574] tiling 3 copies [D 20:01:00.426599] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.426625] lvl 0 el 0 blk 0 by 60817408 [D 20:01:00.426651] to 60817408 ta 33554432 fi 67108864 [D 20:01:00.426675] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.426700] basic type or contiguous data [D 20:01:00.426724] Request disp [D 20:01:00.426752] contig_offset = 60817408 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.426777] calling distribute [D 20:01:00.426802] PINT_distribute [D 20:01:00.426826] of 60817408 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 60817408 exfl 1 [D 20:01:00.426853] begin iteration loff: 60817408 [D 20:01:00.426877] segment exceeds byte limit [D 20:01:00.426901] file being extended [D 20:01:00.426925] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.426950] **********CALL***PROCESS********* [D 20:01:00.426974] segment of 27262976 sz 2097152 [D 20:01:00.426999] PINT_process_request [D 20:01:00.427022] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.427048] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.427074] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.427099] basic type or contiguous data [D 20:01:00.427123] Request disp [D 20:01:00.427146] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.427171] process a segment [D 20:01:00.427199] of 27262976 sz 2097152 [D 20:01:00.427227] add a segment 0 [D 20:01:00.427254] retval = 2097152 [D 20:01:00.427282] exiting distribute returned less than expected [D 20:01:00.427308] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.427334] *****RETURN***FROM***PROCESS***** [D 20:01:00.427359] end iteration [D 20:01:00.427383] done with segments or bytes [D 20:01:00.427407] of 62914560 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.427432] next loff: 62914560 next poff: 62914560 [D 20:01:00.427456] return value 2097152 [D 20:01:00.427481] finished [D 20:01:00.427505] retval = 2097152 [D 20:01:00.427529] exiting distribute returned less than expected [D 20:01:00.427553] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 62914560 fo 67108864 eof 0 [D 20:01:00.427578] ========================================================= [D 20:01:00.427603] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:01:00.427628] element 0: offset: 0xc1e6f000, size: 2097152 [D 20:01:00.427656] post_sr: da13:3346 bh 11 len 32 wr_id 15e72c88. [D 20:01:00.427685] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:01:00.427711] ========================================================= [D 20:01:00.427735] PINT_process_request [D 20:01:00.427759] tiling 4 copies [D 20:01:00.427783] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.427809] lvl 0 el 0 blk 0 by 94371840 [D 20:01:00.427834] to 94371840 ta 67108864 fi 100663296 [D 20:01:00.427859] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.427884] basic type or contiguous data [D 20:01:00.427908] Request disp [D 20:01:00.427932] contig_offset = 94371840 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.427957] calling distribute [D 20:01:00.427981] PINT_distribute [D 20:01:00.428005] of 94371840 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 94371840 exfl 1 [D 20:01:00.428031] begin iteration loff: 94371840 [D 20:01:00.428056] segment exceeds byte limit [D 20:01:00.428080] file being extended [D 20:01:00.428104] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.428129] **********CALL***PROCESS********* [D 20:01:00.428153] segment of 27262976 sz 2097152 [D 20:01:00.428176] PINT_process_request [D 20:01:00.428200] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.428227] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.428252] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.428277] basic type or contiguous data [D 20:01:00.428300] Request disp [D 20:01:00.428324] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.428349] process a segment [D 20:01:00.428373] of 27262976 sz 2097152 [D 20:01:00.428397] add a segment 0 [D 20:01:00.428421] retval = 2097152 [D 20:01:00.428445] exiting distribute returned less than expected [D 20:01:00.428469] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.428496] *****RETURN***FROM***PROCESS***** [D 20:01:00.428523] end iteration [D 20:01:00.428546] done with segments or bytes [D 20:01:00.428570] of 96468992 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.428596] next loff: 96468992 next poff: 96468992 [D 20:01:00.428621] return value 2097152 [D 20:01:00.428646] finished [D 20:01:00.428669] retval = 2097152 [D 20:01:00.428693] exiting distribute returned less than expected [D 20:01:00.428717] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 96468992 fo 100663296 eof 0 [D 20:01:00.428744] ========================================================= [D 20:01:00.428768] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:01:00.428793] element 0: offset: 0xbfe6d000, size: 2097152 [D 20:01:00.428820] post_sr: da13:3346 bh 12 len 32 wr_id 15e72ca0. [D 20:01:00.428850] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:01:00.428875] ========================================================= [D 20:01:00.428900] PINT_process_request [D 20:01:00.428924] tiling 5 copies [D 20:01:00.428948] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.428975] lvl 0 el 0 blk 0 by 127926272 [D 20:01:00.428999] to 127926272 ta 100663296 fi 134217728 [D 20:01:00.429024] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.429049] basic type or contiguous data [D 20:01:00.429073] Request disp [D 20:01:00.429097] contig_offset = 127926272 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.429122] calling distribute [D 20:01:00.429146] PINT_distribute [D 20:01:00.429170] of 127926272 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 127926272 exfl 1 [D 20:01:00.429199] begin iteration loff: 127926272 [D 20:01:00.429224] segment exceeds byte limit [D 20:01:00.429248] file being extended [D 20:01:00.429272] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.429296] **********CALL***PROCESS********* [D 20:01:00.429320] segment of 27262976 sz 2097152 [D 20:01:00.429345] PINT_process_request [D 20:01:00.429368] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.429394] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.429419] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.429444] basic type or contiguous data [D 20:01:00.429469] Request disp [D 20:01:00.429491] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.429517] process a segment [D 20:01:00.429540] of 27262976 sz 2097152 [D 20:01:00.429565] add a segment 0 [D 20:01:00.429589] retval = 2097152 [D 20:01:00.429613] exiting distribute returned less than expected [D 20:01:00.429636] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.429663] *****RETURN***FROM***PROCESS***** [D 20:01:00.429687] end iteration [D 20:01:00.429711] done with segments or bytes [D 20:01:00.429735] of 130023424 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.429760] next loff: 130023424 next poff: 130023424 [D 20:01:00.429785] return value 2097152 [D 20:01:00.429809] finished [D 20:01:00.429833] retval = 2097152 [D 20:01:00.429858] exiting distribute returned less than expected [D 20:01:00.429882] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 130023424 fo 134217728 eof 0 [D 20:01:00.429908] ========================================================= [D 20:01:00.429932] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:01:00.429958] element 0: offset: 0xbde6b000, size: 2097152 [D 20:01:00.429986] post_sr: da13:3346 bh 13 len 32 wr_id 15e72cb8. [D 20:01:00.430016] openib_check_cq: completed send wr_id 15e72c58. [D 20:01:00.430042] openib_check_cq: completed send wr_id 15e72c70. [D 20:01:00.430068] openib_check_cq: completed recv wr_id 15e737d0. [D 20:01:00.430094] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab4016010 rkey 48006000 nic_wr_credit 1. [D 20:01:00.430121] openib_post_rr: da13:3346 bh 31 wr_id 15e737d0. [D 20:01:00.430148] openib_check_cq: completed recv wr_id 15e737e8. [D 20:01:00.430176] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab4217010 rkey 48006100 nic_wr_credit 2. [D 20:01:00.430204] openib_post_rr: da13:3346 bh 32 wr_id 15e737e8. [D 20:01:00.430230] openib_check_cq: completed recv wr_id 15e73800. [D 20:01:00.430256] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab4418010 rkey 48006200 nic_wr_credit 3. [D 20:01:00.430283] openib_post_rr: da13:3346 bh 33 wr_id 15e73800. [D 20:01:00.430309] openib_check_cq: completed recv wr_id 15e73818. [D 20:01:00.430335] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab4619010 rkey 48006300 nic_wr_credit 4. [D 20:01:00.430361] openib_post_rr: da13:3346 bh 34 wr_id 15e73818. [D 20:01:00.430388] openib_check_cq: completed recv wr_id 15e73830. [D 20:01:00.430414] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaab481a010 rkey 48006400 nic_wr_credit 5. [D 20:01:00.430441] openib_post_rr: da13:3346 bh 35 wr_id 15e73830. [D 20:01:00.430467] openib_check_cq: completed recv wr_id 15e73848. [D 20:01:00.430493] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab4a1b010 rkey 48006500 nic_wr_credit 6. [D 20:01:00.430520] openib_post_rr: da13:3346 bh 36 wr_id 15e73848. [D 20:01:00.430546] openib_check_cq: completed recv wr_id 15e73860. [D 20:01:00.430573] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab4c1c010 rkey 48006600 nic_wr_credit 7. [D 20:01:00.430600] openib_post_rr: da13:3346 bh 37 wr_id 15e73860. [D 20:01:00.430625] openib_check_cq: completed recv wr_id 15e73878. [D 20:01:00.430652] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaab4e1d010 rkey 48006700 nic_wr_credit 8. [D 20:01:00.430679] openib_post_rr: da13:3346 bh 38 wr_id 15e73878. [D 20:01:00.430705] openib_check_cq: completed recv wr_id 15e73890. [D 20:01:00.430731] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaab521f010 rkey c8006900 nic_wr_credit 9. [D 20:01:00.430758] openib_post_rr: da13:3346 bh 39 wr_id 15e73890. [D 20:01:00.430784] openib_check_cq: completed send wr_id 15e72c88. [D 20:01:00.430809] openib_check_cq: completed send wr_id 15e72ca0. [D 20:01:00.430834] openib_check_cq: completed send wr_id 15e72cb8. [D 20:01:00.430871] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:00.430897] ========================================================= [D 20:01:00.430921] PINT_process_request [D 20:01:00.430946] tiling 15 copies [D 20:01:00.430970] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.430995] lvl 0 el 0 blk 0 by 438304768 [D 20:01:00.431020] to 438304768 ta 436207616 fi 469762048 [D 20:01:00.431045] mto 2097152 mta 0 mfi 2097152 [D 20:01:00.431070] basic type or contiguous data [D 20:01:00.431094] Request disp [D 20:01:00.431118] contig_offset = 438304768 contig_size = 65011712 lvl_flag = 1 [D 20:01:00.431143] calling distribute [D 20:01:00.431167] PINT_distribute [D 20:01:00.431195] of 438304768 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:00.431225] begin iteration loff: 438304768 [D 20:01:00.431254] segment exceeds byte limit [D 20:01:00.431281] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:00.431308] **********CALL***PROCESS********* [D 20:01:00.431332] segment of 2097152 sz 2097152 [D 20:01:00.431357] PINT_process_request [D 20:01:00.431381] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.431406] lvl 0 el 0 blk 0 by 2097152 [D 20:01:00.431431] to 2097152 ta 2097152 fi 4194304 [D 20:01:00.431456] basic type or contiguous data [D 20:01:00.431480] Request disp [D 20:01:00.431504] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:00.431529] process a segment [D 20:01:00.431552] of 2097152 sz 2097152 [D 20:01:00.431576] add a segment 0 [D 20:01:00.431600] retval = 2097152 [D 20:01:00.431624] exiting distribute returned less than expected [D 20:01:00.431652] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:00.431677] *****RETURN***FROM***PROCESS***** [D 20:01:00.431702] end iteration [D 20:01:00.431726] done with segments or bytes [D 20:01:00.431750] of 440401920 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.431776] next loff: 440401920 next poff: 440401920 [D 20:01:00.431800] return value 2097152 [D 20:01:00.431824] finished [D 20:01:00.431848] retval = 2097152 [D 20:01:00.431872] exiting distribute returned less than expected [D 20:01:00.431897] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 440401920 fo 469762048 eof 0 [D 20:01:00.431922] ========================================================= [D 20:01:00.431947] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:00.431973] element 0: offset: 0x88637000, size: 2097152 [D 20:01:00.432003] post_sr: da13:3346 bh 14 len 48 wr_id 15e72cd0. [D 20:01:00.432038] sm control op 6 [D 20:01:00.432063] [SM Entering]: (0x15ed2098) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.432088] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.432114] (0x15ed2130) io_datafile_complete_operations (tag 0) [D 20:01:00.432139] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.432164] - complete_context_send_or_recv called [D 20:01:00.432188] got a recv completion with context index 0 [D 20:01:00.432213] - io_process_context_recv called [D 20:01:00.432237] - io_process_context_recv called [D 20:01:00.432262] PINT_decode [D 20:01:00.432287] lebf_decode_resp [D 20:01:00.432311] PINT_encode_release [D 20:01:00.432337] lebf_encode_rel [D 20:01:00.432362] PINT_decode_release [D 20:01:00.432387] lebf_decode_rel [D 20:01:00.432413] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.432437] io_post_flow: entry [D 20:01:00.432462] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.432487] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.455373] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.455428] bstream_size = 476053504, datafile nr=0, ct=1, file_req_off = 335544320 [D 20:01:00.455457] PINT_new_request_states n=1 [D 20:01:00.455484] PINT_new_request_states n=1 [D 20:01:00.455509] flowproto posting 0x15ed2eb0 [D 20:01:00.455548] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:00.455574] ========================================================= [D 20:01:00.455599] PINT_process_request [D 20:01:00.455623] tiling 12 copies [D 20:01:00.455647] skipping ahead to target_offset [D 20:01:00.455672] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.455698] lvl 0 el 0 blk 0 by 0 [D 20:01:00.455722] to 0 ta 335544320 fi 369098752 [D 20:01:00.455747] mto 0 mta 0 mfi 33554432 [D 20:01:00.455772] basic type or contiguous data [D 20:01:00.455797] Request disp [D 20:01:00.455821] contig_offset = 0 contig_size = 402653184 lvl_flag = 1 [D 20:01:00.455845] process logical skip [D 20:01:00.455869] retval = 335544320 [D 20:01:00.455893] exiting logical skip because distribute indicates done [D 20:01:00.455917] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.455944] lvl 0 el 0 blk 0 by 335544320 [D 20:01:00.455967] to 335544320 ta 335544320 fi 369098752 [D 20:01:00.455992] mto 0 mta 0 mfi 33554432 [D 20:01:00.456017] basic type or contiguous data [D 20:01:00.456041] Request disp [D 20:01:00.456065] contig_offset = 335544320 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.456090] calling distribute [D 20:01:00.456114] PINT_distribute [D 20:01:00.456138] of 335544320 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:00.456165] begin iteration loff: 335544320 [D 20:01:00.456191] segment exceeds byte limit [D 20:01:00.456214] client lstof 0 diff 0 sgof 0 [D 20:01:00.456239] **********CALL***PROCESS********* [D 20:01:00.456263] segment of 0 sz 2097152 [D 20:01:00.456292] PINT_process_request [D 20:01:00.456315] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.456341] lvl 0 el 0 blk 0 by 0 [D 20:01:00.456367] to 0 ta 0 fi 2097152 [D 20:01:00.456391] basic type or contiguous data [D 20:01:00.456415] Request disp [D 20:01:00.456439] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.456464] process a segment [D 20:01:00.456488] of 0 sz 2097152 [D 20:01:00.456512] add a segment 0 [D 20:01:00.456536] retval = 2097152 [D 20:01:00.456561] exiting distribute returned less than expected [D 20:01:00.456584] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.456610] *****RETURN***FROM***PROCESS***** [D 20:01:00.456635] end iteration [D 20:01:00.456659] done with segments or bytes [D 20:01:00.456683] of 337641472 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.456707] next loff: 337641472 next poff: 337641472 [D 20:01:00.456732] return value 2097152 [D 20:01:00.456757] finished [D 20:01:00.456781] retval = 2097152 [D 20:01:00.456805] exiting distribute returned less than expected [D 20:01:00.456829] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 337641472 fo 369098752 eof 0 [D 20:01:00.456855] ========================================================= [D 20:01:00.456879] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:00.456905] element 0: offset: 0x8e43d000, size: 2097152 [D 20:01:00.456932] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 8. [D 20:01:00.456960] post_sr: da13:3346 bh 15 len 16 wr_id 15e72ce8. [D 20:01:00.456987] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 7. [D 20:01:00.457014] post_sr: da13:3346 bh 16 len 16 wr_id 15e72d00. [D 20:01:00.457040] openib_check_cq: completed recv wr_id 15e738a8. [D 20:01:00.457069] openib_post_rr: da13:3346 bh 40 wr_id 15e738a8. [D 20:01:00.457095] openib_check_cq: completed recv wr_id 15e738c0. [D 20:01:00.457123] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaab501e010 rkey c8006800 nic_wr_credit 8. [D 20:01:00.457150] openib_post_rr: da13:3346 bh 41 wr_id 15e738c0. [D 20:01:00.457177] openib_check_cq: completed recv wr_id 15e738d8. [D 20:01:00.457207] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab5420010 rkey c8006a00 nic_wr_credit 9. [D 20:01:00.457234] openib_post_rr: da13:3346 bh 42 wr_id 15e738d8. [D 20:01:00.457260] openib_check_cq: completed recv wr_id 15e738f0. [D 20:01:00.457286] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab5621010 rkey c8006b00 nic_wr_credit 10. [D 20:01:00.457314] openib_post_rr: da13:3346 bh 43 wr_id 15e738f0. [D 20:01:00.457340] openib_check_cq: completed recv wr_id 15e73908. [D 20:01:00.457367] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab5822010 rkey c8006c00 nic_wr_credit 11. [D 20:01:00.457394] openib_post_rr: da13:3346 bh 44 wr_id 15e73908. [D 20:01:00.457420] openib_check_cq: completed recv wr_id 15e73920. [D 20:01:00.457447] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab5a23010 rkey c8006d00 nic_wr_credit 12. [D 20:01:00.457474] openib_post_rr: da13:3346 bh 45 wr_id 15e73920. [D 20:01:00.457500] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 11. [D 20:01:00.457527] post_sr: da13:3346 bh 17 len 16 wr_id 15e72d18. [D 20:01:00.457553] openib_check_cq: completed recv wr_id 15e73938. [D 20:01:00.457580] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaab5e26010 rkey c8006f00 nic_wr_credit 12. [D 20:01:00.457607] openib_post_rr: da13:3346 bh 46 wr_id 15e73938. [D 20:01:00.457633] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 11. [D 20:01:00.457660] post_sr: da13:3346 bh 18 len 16 wr_id 15e72d30. [D 20:01:00.457686] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 10. [D 20:01:00.457713] post_sr: da13:3346 bh 19 len 16 wr_id 15e72d48. [D 20:01:00.457742] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 9. [D 20:01:00.457769] post_sr: da13:3346 bh 20 len 16 wr_id 15e72d60. [D 20:01:00.457795] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 8. [D 20:01:00.457821] post_sr: da13:3346 bh 21 len 16 wr_id 15e72d78. [D 20:01:00.457848] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 7. [D 20:01:00.457875] post_sr: da13:3346 bh 22 len 16 wr_id 15e72d90. [D 20:01:00.457901] openib_check_cq: completed recv wr_id 15e73950. [D 20:01:00.457931] openib_post_rr: da13:3346 bh 47 wr_id 15e73950. [D 20:01:00.457957] openib_check_cq: completed recv wr_id 15e73968. [D 20:01:00.457985] openib_post_rr: da13:3346 bh 48 wr_id 15e73968. [D 20:01:00.458011] openib_check_cq: completed recv wr_id 15e73980. [D 20:01:00.458039] openib_post_rr: da13:3346 bh 49 wr_id 15e73980. [D 20:01:00.458065] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 6. [D 20:01:00.458092] post_sr: da13:3346 bh 23 len 16 wr_id 15e72da8. [D 20:01:00.458118] openib_check_cq: completed send wr_id 15e72cd0. [D 20:01:00.458144] openib_check_cq: completed send wr_id 15e72ce8. [D 20:01:00.458168] openib_check_cq: completed send wr_id 15e72d00. [D 20:01:00.458195] openib_check_cq: completed recv wr_id 15e73998. [D 20:01:00.458222] openib_post_rr: da13:3346 bh 50 wr_id 15e73998. [D 20:01:00.458248] openib_check_cq: completed recv wr_id 15e739b0. [D 20:01:00.458275] openib_post_rr: da13:3346 bh 51 wr_id 15e739b0. [D 20:01:00.458307] flowproto posted 0x15ed2eb0 [D 20:01:00.458332] Job flows in progress (post time): 27 [D 20:01:00.458359] posted flow for context 0x15ed2900 [D 20:01:00.458385] *** all msgpairs complete (all flows posted) [D 20:01:00.458409] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.458435] [SM Exiting]: (0x15ed2098) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.458461] sm control op 6 [D 20:01:00.458486] [SM Entering]: (0x15ed5570) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.458512] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.458537] (0x15ed5608) io_datafile_complete_operations (tag 0) [D 20:01:00.458561] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.458587] - complete_context_send_or_recv called [D 20:01:00.458611] got a recv completion with context index 0 [D 20:01:00.458637] - io_process_context_recv called [D 20:01:00.458662] - io_process_context_recv called [D 20:01:00.458685] PINT_decode [D 20:01:00.458710] lebf_decode_resp [D 20:01:00.458735] PINT_encode_release [D 20:01:00.458759] lebf_encode_rel [D 20:01:00.458785] PINT_decode_release [D 20:01:00.458808] lebf_decode_rel [D 20:01:00.458834] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.458860] io_post_flow: entry [D 20:01:00.458884] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.458909] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.481510] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.481558] bstream_size = 478150656, datafile nr=0, ct=1, file_req_off = 369098752 [D 20:01:00.481589] PINT_new_request_states n=1 [D 20:01:00.481615] PINT_new_request_states n=1 [D 20:01:00.481639] flowproto posting 0x15ed6388 [D 20:01:00.481677] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:00.481703] ========================================================= [D 20:01:00.481728] PINT_process_request [D 20:01:00.481753] tiling 13 copies [D 20:01:00.481777] skipping ahead to target_offset [D 20:01:00.481801] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.481827] lvl 0 el 0 blk 0 by 0 [D 20:01:00.481852] to 0 ta 369098752 fi 402653184 [D 20:01:00.481877] mto 0 mta 0 mfi 33554432 [D 20:01:00.481902] basic type or contiguous data [D 20:01:00.481925] Request disp [D 20:01:00.481949] contig_offset = 0 contig_size = 436207616 lvl_flag = 1 [D 20:01:00.481974] process logical skip [D 20:01:00.481998] retval = 369098752 [D 20:01:00.482027] exiting logical skip because distribute indicates done [D 20:01:00.482051] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.482077] lvl 0 el 0 blk 0 by 369098752 [D 20:01:00.482102] to 369098752 ta 369098752 fi 402653184 [D 20:01:00.482127] mto 0 mta 0 mfi 33554432 [D 20:01:00.482151] basic type or contiguous data [D 20:01:00.482175] Request disp [D 20:01:00.482200] contig_offset = 369098752 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.482225] calling distribute [D 20:01:00.482249] PINT_distribute [D 20:01:00.482273] of 369098752 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:00.482299] begin iteration loff: 369098752 [D 20:01:00.482328] segment exceeds byte limit [D 20:01:00.482352] client lstof 0 diff 0 sgof 0 [D 20:01:00.482377] **********CALL***PROCESS********* [D 20:01:00.482400] segment of 0 sz 2097152 [D 20:01:00.482425] PINT_process_request [D 20:01:00.482449] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.482475] lvl 0 el 0 blk 0 by 0 [D 20:01:00.482500] to 0 ta 0 fi 2097152 [D 20:01:00.482524] basic type or contiguous data [D 20:01:00.482548] Request disp [D 20:01:00.482572] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.482597] process a segment [D 20:01:00.482621] of 0 sz 2097152 [D 20:01:00.482645] add a segment 0 [D 20:01:00.482668] retval = 2097152 [D 20:01:00.482693] exiting distribute returned less than expected [D 20:01:00.482717] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.482743] *****RETURN***FROM***PROCESS***** [D 20:01:00.482768] end iteration [D 20:01:00.482791] done with segments or bytes [D 20:01:00.482815] of 371195904 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.482840] next loff: 371195904 next poff: 371195904 [D 20:01:00.482865] return value 2097152 [D 20:01:00.482890] finished [D 20:01:00.482913] retval = 2097152 [D 20:01:00.482936] exiting distribute returned less than expected [D 20:01:00.482961] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 371195904 fo 402653184 eof 0 [D 20:01:00.482987] ========================================================= [D 20:01:00.483012] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:00.483036] element 0: offset: 0x8c43b000, size: 2097152 [D 20:01:00.483064] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 5. [D 20:01:00.483092] post_sr: da13:3346 bh 24 len 16 wr_id 15e72dc0. [D 20:01:00.483119] openib_check_cq: completed recv wr_id 15e739c8. [D 20:01:00.483147] openib_post_rr: da13:3346 bh 52 wr_id 15e739c8. [D 20:01:00.483174] post_sr: da13:3346 bh 25 len 48 wr_id 15e72dd8. [D 20:01:00.483205] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 4. [D 20:01:00.483236] post_sr: da13:3346 bh 26 len 16 wr_id 15e72df0. [D 20:01:00.483266] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 3. [D 20:01:00.483297] post_sr: da13:3346 bh 27 len 16 wr_id 15e72e08. [D 20:01:00.483324] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 2. [D 20:01:00.483350] post_sr: da13:3346 bh 28 len 16 wr_id 15e72e20. [D 20:01:00.483376] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 1. [D 20:01:00.483403] post_sr: da13:3346 bh 29 len 16 wr_id 15e72e38. [D 20:01:00.483429] openib_check_cq: completed send wr_id 15e72d18. [D 20:01:00.483455] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 0. [D 20:01:00.483481] post_sr: da13:3346 bh 30 len 16 wr_id 15e72e50. [D 20:01:00.483508] openib_check_cq: completed send wr_id 15e72d30. [D 20:01:00.483532] openib_check_cq: completed send wr_id 15e72d48. [D 20:01:00.483558] openib_check_cq: completed send wr_id 15e72d60. [D 20:01:00.483583] openib_check_cq: completed send wr_id 15e72d78. [D 20:01:00.483609] openib_check_cq: completed send wr_id 15e72d90. [D 20:01:00.483633] openib_check_cq: completed send wr_id 15e72da8. [D 20:01:00.483658] openib_check_cq: completed send wr_id 15e72dc0. [D 20:01:00.483687] openib_check_cq: completed send wr_id 15e72dd8. [D 20:01:00.483712] openib_check_cq: completed send wr_id 15e72df0. [D 20:01:00.483737] openib_check_cq: completed send wr_id 15e72e08. [D 20:01:00.483762] openib_check_cq: completed send wr_id 15e72e20. [D 20:01:00.483787] openib_check_cq: completed send wr_id 15e72e38. [D 20:01:00.483813] openib_check_cq: completed send wr_id 15e72e50. [D 20:01:00.483845] flowproto posted 0x15ed6388 [D 20:01:00.483870] Job flows in progress (post time): 28 [D 20:01:00.483898] posted flow for context 0x15ed5dd8 [D 20:01:00.483922] *** all msgpairs complete (all flows posted) [D 20:01:00.483947] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.483972] [SM Exiting]: (0x15ed5570) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.483999] sm control op 6 [D 20:01:00.484024] [SM Entering]: (0x15ed8a48) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.484049] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.484075] (0x15ed8ae0) io_datafile_complete_operations (tag 0) [D 20:01:00.484100] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.484126] - complete_context_send_or_recv called [D 20:01:00.484149] got a recv completion with context index 0 [D 20:01:00.484174] - io_process_context_recv called [D 20:01:00.484199] - io_process_context_recv called [D 20:01:00.484223] PINT_decode [D 20:01:00.484248] lebf_decode_resp [D 20:01:00.484272] PINT_encode_release [D 20:01:00.484296] lebf_encode_rel [D 20:01:00.484321] PINT_decode_release [D 20:01:00.484346] lebf_decode_rel [D 20:01:00.484371] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.484396] io_post_flow: entry [D 20:01:00.484421] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.484446] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.507578] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.507633] bstream_size = 478150656, datafile nr=0, ct=1, file_req_off = 402653184 [D 20:01:00.507664] PINT_new_request_states n=1 [D 20:01:00.507691] PINT_new_request_states n=1 [D 20:01:00.507716] flowproto posting 0x15ed9860 [D 20:01:00.507757] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:00.507784] ========================================================= [D 20:01:00.507809] PINT_process_request [D 20:01:00.507834] tiling 14 copies [D 20:01:00.507858] skipping ahead to target_offset [D 20:01:00.507883] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.507908] lvl 0 el 0 blk 0 by 0 [D 20:01:00.507933] to 0 ta 402653184 fi 436207616 [D 20:01:00.507958] mto 0 mta 0 mfi 33554432 [D 20:01:00.507983] basic type or contiguous data [D 20:01:00.508007] Request disp [D 20:01:00.508031] contig_offset = 0 contig_size = 469762048 lvl_flag = 1 [D 20:01:00.508056] process logical skip [D 20:01:00.508080] retval = 402653184 [D 20:01:00.508105] exiting logical skip because distribute indicates done [D 20:01:00.508129] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.508154] lvl 0 el 0 blk 0 by 402653184 [D 20:01:00.508179] to 402653184 ta 402653184 fi 436207616 [D 20:01:00.508204] mto 0 mta 0 mfi 33554432 [D 20:01:00.508228] basic type or contiguous data [D 20:01:00.508252] Request disp [D 20:01:00.508275] contig_offset = 402653184 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.508301] calling distribute [D 20:01:00.508325] PINT_distribute [D 20:01:00.508349] of 402653184 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:00.508376] begin iteration loff: 402653184 [D 20:01:00.508401] segment exceeds byte limit [D 20:01:00.508425] client lstof 0 diff 0 sgof 0 [D 20:01:00.508450] **********CALL***PROCESS********* [D 20:01:00.508474] segment of 0 sz 2097152 [D 20:01:00.508499] PINT_process_request [D 20:01:00.508522] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.508553] lvl 0 el 0 blk 0 by 0 [D 20:01:00.508578] to 0 ta 0 fi 2097152 [D 20:01:00.508603] basic type or contiguous data [D 20:01:00.508626] Request disp [D 20:01:00.508650] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.508676] process a segment [D 20:01:00.508700] of 0 sz 2097152 [D 20:01:00.508724] add a segment 0 [D 20:01:00.508748] retval = 2097152 [D 20:01:00.508772] exiting distribute returned less than expected [D 20:01:00.508796] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.508822] *****RETURN***FROM***PROCESS***** [D 20:01:00.508847] end iteration [D 20:01:00.508871] done with segments or bytes [D 20:01:00.508894] of 404750336 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.508920] next loff: 404750336 next poff: 404750336 [D 20:01:00.508945] return value 2097152 [D 20:01:00.508970] finished [D 20:01:00.508994] retval = 2097152 [D 20:01:00.509017] exiting distribute returned less than expected [D 20:01:00.509041] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 404750336 fo 436207616 eof 0 [D 20:01:00.509067] ========================================================= [D 20:01:00.509093] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:00.509118] element 0: offset: 0x8a439000, size: 2097152 [D 20:01:00.509160] flowproto posted 0x15ed9860 [D 20:01:00.509188] Job flows in progress (post time): 29 [D 20:01:00.509218] posted flow for context 0x15ed92b0 [D 20:01:00.509243] *** all msgpairs complete (all flows posted) [D 20:01:00.509267] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.509293] [SM Exiting]: (0x15ed8a48) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.509340] sm control op 6 [D 20:01:00.509365] [SM Entering]: (0x15edf3f8) client_io_sm:io_datafile_complete_operations (status: 0) [D 20:01:00.509390] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.509416] (0x15edf490) io_datafile_complete_operations (tag 0) [D 20:01:00.509442] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.509467] - complete_context_send_or_recv called [D 20:01:00.509491] got a recv completion with context index 0 [D 20:01:00.509516] - io_process_context_recv called [D 20:01:00.509541] - io_process_context_recv called [D 20:01:00.509565] PINT_decode [D 20:01:00.509591] lebf_decode_resp [D 20:01:00.509615] PINT_encode_release [D 20:01:00.509640] lebf_encode_rel [D 20:01:00.509666] PINT_decode_release [D 20:01:00.509690] lebf_decode_rel [D 20:01:00.509716] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:00.509742] io_post_flow: entry [D 20:01:00.509766] [BMI CONTROL]: BMI_set_info: set_info: 367403208 option: 14 [D 20:01:00.509792] [BMI CONTROL]: BMI_set_info: searching for ref 367403208 [D 20:01:00.532382] * mem req size is 33554432, file_req size is 33554432 (bytes) [D 20:01:00.532431] bstream_size = 480247808, datafile nr=0, ct=1, file_req_off = 469762048 [D 20:01:00.532463] PINT_new_request_states n=1 [D 20:01:00.532495] PINT_new_request_states n=1 [D 20:01:00.532521] flowproto posting 0x15ee0210 [D 20:01:00.532554] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ee0210. [D 20:01:00.532581] ========================================================= [D 20:01:00.532606] PINT_process_request [D 20:01:00.532629] tiling 16 copies [D 20:01:00.532654] skipping ahead to target_offset [D 20:01:00.532678] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.532705] lvl 0 el 0 blk 0 by 0 [D 20:01:00.532729] to 0 ta 469762048 fi 503316480 [D 20:01:00.532753] mto 0 mta 0 mfi 33554432 [D 20:01:00.532778] basic type or contiguous data [D 20:01:00.532803] Request disp [D 20:01:00.532827] contig_offset = 0 contig_size = 536870912 lvl_flag = 1 [D 20:01:00.532852] process logical skip [D 20:01:00.532876] retval = 469762048 [D 20:01:00.532899] exiting logical skip because distribute indicates done [D 20:01:00.532924] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.532955] lvl 0 el 0 blk 0 by 469762048 [D 20:01:00.532979] to 469762048 ta 469762048 fi 503316480 [D 20:01:00.533004] mto 0 mta 0 mfi 33554432 [D 20:01:00.533029] basic type or contiguous data [D 20:01:00.533053] Request disp [D 20:01:00.533077] contig_offset = 469762048 contig_size = 67108864 lvl_flag = 1 [D 20:01:00.533102] calling distribute [D 20:01:00.533125] PINT_distribute [D 20:01:00.533149] of 469762048 sz 33554432 ix 0 sm 64 by 0 bm 2097152 fsz 480247808 exfl 0 [D 20:01:00.533177] begin iteration loff: 469762048 [D 20:01:00.533205] segment exceeds byte limit [D 20:01:00.533229] client lstof 0 diff 0 sgof 0 [D 20:01:00.533254] **********CALL***PROCESS********* [D 20:01:00.533278] segment of 0 sz 2097152 [D 20:01:00.533303] PINT_process_request [D 20:01:00.533327] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.533352] lvl 0 el 0 blk 0 by 0 [D 20:01:00.533377] to 0 ta 0 fi 2097152 [D 20:01:00.533402] basic type or contiguous data [D 20:01:00.533426] Request disp [D 20:01:00.533451] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:00.533475] process a segment [D 20:01:00.533499] of 0 sz 2097152 [D 20:01:00.533523] add a segment 0 [D 20:01:00.533548] retval = 2097152 [D 20:01:00.533572] exiting distribute returned less than expected [D 20:01:00.533596] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 2097152 fo 2097152 eof 0 [D 20:01:00.533622] *****RETURN***FROM***PROCESS***** [D 20:01:00.533646] end iteration [D 20:01:00.533670] done with segments or bytes [D 20:01:00.533694] of 471859200 sz 31457280 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.533720] next loff: 471859200 next poff: 471859200 [D 20:01:00.533744] return value 2097152 [D 20:01:00.533768] finished [D 20:01:00.533792] retval = 2097152 [D 20:01:00.533817] exiting distribute returned less than expected [D 20:01:00.533841] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 471859200 fo 503316480 eof 0 [D 20:01:00.533866] ========================================================= [D 20:01:00.533891] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 37 [D 20:01:00.533917] element 0: offset: 0x86435000, size: 2097152 [D 20:01:00.533952] flowproto posted 0x15ee0210 [D 20:01:00.533978] Job flows in progress (post time): 30 [D 20:01:00.534005] posted flow for context 0x15edfc60 [D 20:01:00.534030] *** all msgpairs complete (all flows posted) [D 20:01:00.534054] 1 flows pending, 0 write acks pending, 0 msgpair [D 20:01:00.534079] [SM Exiting]: (0x15edf3f8) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:00.534107] PINT_client_state_machine_test id 22 [D 20:01:00.534152] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:01:00.534178] ========================================================= [D 20:01:00.534203] PINT_process_request [D 20:01:00.534227] tiling 6 copies [D 20:01:00.534252] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.534277] lvl 0 el 0 blk 0 by 161480704 [D 20:01:00.534302] to 161480704 ta 134217728 fi 167772160 [D 20:01:00.534327] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.534352] basic type or contiguous data [D 20:01:00.534376] Request disp [D 20:01:00.534399] contig_offset = 161480704 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.534425] calling distribute [D 20:01:00.534449] PINT_distribute [D 20:01:00.534473] of 161480704 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 161480704 exfl 1 [D 20:01:00.534500] begin iteration loff: 161480704 [D 20:01:00.534524] segment exceeds byte limit [D 20:01:00.534548] file being extended [D 20:01:00.534572] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.534597] **********CALL***PROCESS********* [D 20:01:00.534621] segment of 27262976 sz 2097152 [D 20:01:00.534645] PINT_process_request [D 20:01:00.534669] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.534698] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.534724] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.534749] basic type or contiguous data [D 20:01:00.534772] Request disp [D 20:01:00.534796] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.534822] process a segment [D 20:01:00.534846] of 27262976 sz 2097152 [D 20:01:00.534870] add a segment 0 [D 20:01:00.534893] retval = 2097152 [D 20:01:00.534917] exiting distribute returned less than expected [D 20:01:00.534942] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.534968] *****RETURN***FROM***PROCESS***** [D 20:01:00.534992] end iteration [D 20:01:00.535016] done with segments or bytes [D 20:01:00.535039] of 163577856 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.535065] next loff: 163577856 next poff: 163577856 [D 20:01:00.535090] return value 2097152 [D 20:01:00.535114] finished [D 20:01:00.535138] retval = 2097152 [D 20:01:00.535161] exiting distribute returned less than expected [D 20:01:00.535188] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 163577856 fo 167772160 eof 0 [D 20:01:00.535219] ========================================================= [D 20:01:00.535247] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:01:00.535276] element 0: offset: 0xbbe69000, size: 2097152 [D 20:01:00.535308] post_sr: da13:3346 bh 31 len 32 wr_id 15e72e68. [D 20:01:00.535339] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:01:00.535365] ========================================================= [D 20:01:00.535389] PINT_process_request [D 20:01:00.535413] tiling 7 copies [D 20:01:00.535437] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.535464] lvl 0 el 0 blk 0 by 195035136 [D 20:01:00.535489] to 195035136 ta 167772160 fi 201326592 [D 20:01:00.535513] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.535538] basic type or contiguous data [D 20:01:00.535562] Request disp [D 20:01:00.535587] contig_offset = 195035136 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.535612] calling distribute [D 20:01:00.535635] PINT_distribute [D 20:01:00.535659] of 195035136 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 195035136 exfl 1 [D 20:01:00.535686] begin iteration loff: 195035136 [D 20:01:00.535711] segment exceeds byte limit [D 20:01:00.535735] file being extended [D 20:01:00.535758] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.535783] **********CALL***PROCESS********* [D 20:01:00.535807] segment of 27262976 sz 2097152 [D 20:01:00.535832] PINT_process_request [D 20:01:00.535856] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.535882] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.535907] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.535932] basic type or contiguous data [D 20:01:00.535956] Request disp [D 20:01:00.535979] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.536005] process a segment [D 20:01:00.536027] of 27262976 sz 2097152 [D 20:01:00.536052] add a segment 0 [D 20:01:00.536076] retval = 2097152 [D 20:01:00.536100] exiting distribute returned less than expected [D 20:01:00.536125] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.536150] *****RETURN***FROM***PROCESS***** [D 20:01:00.536175] end iteration [D 20:01:00.536199] done with segments or bytes [D 20:01:00.536223] of 197132288 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.536249] next loff: 197132288 next poff: 197132288 [D 20:01:00.536273] return value 2097152 [D 20:01:00.536297] finished [D 20:01:00.536321] retval = 2097152 [D 20:01:00.536345] exiting distribute returned less than expected [D 20:01:00.536369] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 197132288 fo 201326592 eof 0 [D 20:01:00.536395] ========================================================= [D 20:01:00.536419] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:01:00.536447] element 0: offset: 0xb9e67000, size: 2097152 [D 20:01:00.536475] post_sr: da13:3346 bh 32 len 32 wr_id 15e72e80. [D 20:01:00.536504] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:01:00.536530] ========================================================= [D 20:01:00.536555] PINT_process_request [D 20:01:00.536579] tiling 8 copies [D 20:01:00.536603] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.536629] lvl 0 el 0 blk 0 by 228589568 [D 20:01:00.536654] to 228589568 ta 201326592 fi 234881024 [D 20:01:00.536680] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.536705] basic type or contiguous data [D 20:01:00.536728] Request disp [D 20:01:00.536752] contig_offset = 228589568 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.536777] calling distribute [D 20:01:00.536802] PINT_distribute [D 20:01:00.536825] of 228589568 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 228589568 exfl 1 [D 20:01:00.536851] begin iteration loff: 228589568 [D 20:01:00.536876] segment exceeds byte limit [D 20:01:00.536900] file being extended [D 20:01:00.536924] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.536949] **********CALL***PROCESS********* [D 20:01:00.536972] segment of 27262976 sz 2097152 [D 20:01:00.536997] PINT_process_request [D 20:01:00.537021] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.537047] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.537072] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.537096] basic type or contiguous data [D 20:01:00.537120] Request disp [D 20:01:00.537144] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.537170] process a segment [D 20:01:00.537196] of 27262976 sz 2097152 [D 20:01:00.537220] add a segment 0 [D 20:01:00.537244] retval = 2097152 [D 20:01:00.537268] exiting distribute returned less than expected [D 20:01:00.537292] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.537319] *****RETURN***FROM***PROCESS***** [D 20:01:00.537343] end iteration [D 20:01:00.537366] done with segments or bytes [D 20:01:00.537390] of 230686720 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.537416] next loff: 230686720 next poff: 230686720 [D 20:01:00.537441] return value 2097152 [D 20:01:00.537465] finished [D 20:01:00.537488] retval = 2097152 [D 20:01:00.537512] exiting distribute returned less than expected [D 20:01:00.537536] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 230686720 fo 234881024 eof 0 [D 20:01:00.537563] ========================================================= [D 20:01:00.537587] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:01:00.537612] element 0: offset: 0xb7e65000, size: 2097152 [D 20:01:00.537640] post_sr: da13:3346 bh 33 len 32 wr_id 15e72e98. [D 20:01:00.537669] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:01:00.537695] ========================================================= [D 20:01:00.537719] PINT_process_request [D 20:01:00.537743] tiling 9 copies [D 20:01:00.537768] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.537794] lvl 0 el 0 blk 0 by 262144000 [D 20:01:00.537818] to 262144000 ta 234881024 fi 268435456 [D 20:01:00.537843] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.537868] basic type or contiguous data [D 20:01:00.537892] Request disp [D 20:01:00.537916] contig_offset = 262144000 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.537940] calling distribute [D 20:01:00.537965] PINT_distribute [D 20:01:00.537989] of 262144000 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 262144000 exfl 1 [D 20:01:00.538015] begin iteration loff: 262144000 [D 20:01:00.538041] segment exceeds byte limit [D 20:01:00.538064] file being extended [D 20:01:00.538088] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.538113] **********CALL***PROCESS********* [D 20:01:00.538139] segment of 27262976 sz 2097152 [D 20:01:00.538165] PINT_process_request [D 20:01:00.538188] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.538214] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.538239] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.538264] basic type or contiguous data [D 20:01:00.538289] Request disp [D 20:01:00.538312] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.538337] process a segment [D 20:01:00.538361] of 27262976 sz 2097152 [D 20:01:00.538385] add a segment 0 [D 20:01:00.538410] retval = 2097152 [D 20:01:00.538434] exiting distribute returned less than expected [D 20:01:00.538457] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.538484] *****RETURN***FROM***PROCESS***** [D 20:01:00.538509] end iteration [D 20:01:00.538533] done with segments or bytes [D 20:01:00.538556] of 264241152 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.538581] next loff: 264241152 next poff: 264241152 [D 20:01:00.538606] return value 2097152 [D 20:01:00.538631] finished [D 20:01:00.538655] retval = 2097152 [D 20:01:00.538679] exiting distribute returned less than expected [D 20:01:00.538702] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 264241152 fo 268435456 eof 0 [D 20:01:00.538728] ========================================================= [D 20:01:00.538753] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:01:00.538779] element 0: offset: 0xb5e63000, size: 2097152 [D 20:01:00.538805] post_sr: da13:3346 bh 34 len 32 wr_id 15e72eb0. [D 20:01:00.538834] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:01:00.538861] ========================================================= [D 20:01:00.538885] PINT_process_request [D 20:01:00.538908] tiling 10 copies [D 20:01:00.538933] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.538959] lvl 0 el 0 blk 0 by 295698432 [D 20:01:00.538984] to 295698432 ta 268435456 fi 301989888 [D 20:01:00.539009] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.539033] basic type or contiguous data [D 20:01:00.539057] Request disp [D 20:01:00.539081] contig_offset = 295698432 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.539106] calling distribute [D 20:01:00.539130] PINT_distribute [D 20:01:00.539154] of 295698432 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 295698432 exfl 1 [D 20:01:00.539181] begin iteration loff: 295698432 [D 20:01:00.539212] segment exceeds byte limit [D 20:01:00.539238] file being extended [D 20:01:00.539266] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.539294] **********CALL***PROCESS********* [D 20:01:00.539319] segment of 27262976 sz 2097152 [D 20:01:00.539342] PINT_process_request [D 20:01:00.539367] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.539393] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.539419] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.539443] basic type or contiguous data [D 20:01:00.539467] Request disp [D 20:01:00.539490] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.539516] process a segment [D 20:01:00.539539] of 27262976 sz 2097152 [D 20:01:00.539564] add a segment 0 [D 20:01:00.539587] retval = 2097152 [D 20:01:00.539611] exiting distribute returned less than expected [D 20:01:00.539636] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.539662] *****RETURN***FROM***PROCESS***** [D 20:01:00.539687] end iteration [D 20:01:00.539710] done with segments or bytes [D 20:01:00.539734] of 297795584 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.539760] next loff: 297795584 next poff: 297795584 [D 20:01:00.539785] return value 2097152 [D 20:01:00.539809] finished [D 20:01:00.539833] retval = 2097152 [D 20:01:00.539856] exiting distribute returned less than expected [D 20:01:00.539880] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 297795584 fo 301989888 eof 0 [D 20:01:00.539909] ========================================================= [D 20:01:00.539934] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:01:00.539959] element 0: offset: 0xb3e61000, size: 2097152 [D 20:01:00.539986] post_sr: da13:3346 bh 35 len 32 wr_id 15e72ec8. [D 20:01:00.540018] openib_check_cq: completed send wr_id 15e72e68. [D 20:01:00.540044] openib_check_cq: completed send wr_id 15e72e80. [D 20:01:00.540069] openib_check_cq: completed send wr_id 15e72e98. [D 20:01:00.540094] openib_check_cq: completed send wr_id 15e72eb0. [D 20:01:00.540120] openib_check_cq: completed send wr_id 15e72ec8. [D 20:01:00.540153] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:01:00.540179] ========================================================= [D 20:01:00.540204] PINT_process_request [D 20:01:00.540227] tiling 11 copies [D 20:01:00.540251] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.540278] lvl 0 el 0 blk 0 by 329252864 [D 20:01:00.540303] to 329252864 ta 301989888 fi 335544320 [D 20:01:00.540328] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.540352] basic type or contiguous data [D 20:01:00.540376] Request disp [D 20:01:00.540400] contig_offset = 329252864 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.540426] calling distribute [D 20:01:00.540449] PINT_distribute [D 20:01:00.540472] of 329252864 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 329252864 exfl 1 [D 20:01:00.540499] begin iteration loff: 329252864 [D 20:01:00.540524] segment exceeds byte limit [D 20:01:00.540549] file being extended [D 20:01:00.540572] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.540597] **********CALL***PROCESS********* [D 20:01:00.540621] segment of 27262976 sz 2097152 [D 20:01:00.540645] PINT_process_request [D 20:01:00.540669] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.540696] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.540720] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.540745] basic type or contiguous data [D 20:01:00.540769] Request disp [D 20:01:00.540793] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.540818] process a segment [D 20:01:00.540842] of 27262976 sz 2097152 [D 20:01:00.540866] add a segment 0 [D 20:01:00.540890] retval = 2097152 [D 20:01:00.540914] exiting distribute returned less than expected [D 20:01:00.540938] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.540964] *****RETURN***FROM***PROCESS***** [D 20:01:00.540988] end iteration [D 20:01:00.541012] done with segments or bytes [D 20:01:00.541036] of 331350016 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.541062] next loff: 331350016 next poff: 331350016 [D 20:01:00.541087] return value 2097152 [D 20:01:00.541110] finished [D 20:01:00.541134] retval = 2097152 [D 20:01:00.541158] exiting distribute returned less than expected [D 20:01:00.541184] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 331350016 fo 335544320 eof 0 [D 20:01:00.541211] ========================================================= [D 20:01:00.541235] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:01:00.541260] element 0: offset: 0xb1e5f000, size: 2097152 [D 20:01:00.541288] post_sr: da13:3346 bh 36 len 32 wr_id 15e72ee0. [D 20:01:00.541316] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:01:00.541342] ========================================================= [D 20:01:00.541366] PINT_process_request [D 20:01:00.541391] tiling 12 copies [D 20:01:00.541415] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.541440] lvl 0 el 0 blk 0 by 362807296 [D 20:01:00.541465] to 362807296 ta 335544320 fi 369098752 [D 20:01:00.541490] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.541515] basic type or contiguous data [D 20:01:00.541539] Request disp [D 20:01:00.541567] contig_offset = 362807296 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.541593] calling distribute [D 20:01:00.541617] PINT_distribute [D 20:01:00.541641] of 362807296 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 362807296 exfl 1 [D 20:01:00.541667] begin iteration loff: 362807296 [D 20:01:00.541692] segment exceeds byte limit [D 20:01:00.541716] file being extended [D 20:01:00.541740] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.541765] **********CALL***PROCESS********* [D 20:01:00.541789] segment of 27262976 sz 2097152 [D 20:01:00.541813] PINT_process_request [D 20:01:00.541837] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.541863] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.541888] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.541913] basic type or contiguous data [D 20:01:00.541936] Request disp [D 20:01:00.541960] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.541986] process a segment [D 20:01:00.542009] of 27262976 sz 2097152 [D 20:01:00.542034] add a segment 0 [D 20:01:00.542057] retval = 2097152 [D 20:01:00.542081] exiting distribute returned less than expected [D 20:01:00.542105] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.542132] *****RETURN***FROM***PROCESS***** [D 20:01:00.542156] end iteration [D 20:01:00.542180] done with segments or bytes [D 20:01:00.542203] of 364904448 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.542229] next loff: 364904448 next poff: 364904448 [D 20:01:00.542254] return value 2097152 [D 20:01:00.542278] finished [D 20:01:00.542302] retval = 2097152 [D 20:01:00.542325] exiting distribute returned less than expected [D 20:01:00.542352] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 364904448 fo 369098752 eof 0 [D 20:01:00.542378] ========================================================= [D 20:01:00.542403] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:01:00.542428] element 0: offset: 0xafe5d000, size: 2097152 [D 20:01:00.542455] post_sr: da13:3346 bh 37 len 32 wr_id 15e72ef8. [D 20:01:00.542485] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:01:00.542510] ========================================================= [D 20:01:00.542535] PINT_process_request [D 20:01:00.542559] tiling 13 copies [D 20:01:00.542583] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.542610] lvl 0 el 0 blk 0 by 396361728 [D 20:01:00.542634] to 396361728 ta 369098752 fi 402653184 [D 20:01:00.542659] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.542684] basic type or contiguous data [D 20:01:00.542708] Request disp [D 20:01:00.542732] contig_offset = 396361728 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.542757] calling distribute [D 20:01:00.542781] PINT_distribute [D 20:01:00.542805] of 396361728 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 396361728 exfl 1 [D 20:01:00.542832] begin iteration loff: 396361728 [D 20:01:00.542857] segment exceeds byte limit [D 20:01:00.542881] file being extended [D 20:01:00.542904] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.542929] **********CALL***PROCESS********* [D 20:01:00.542953] segment of 27262976 sz 2097152 [D 20:01:00.542977] PINT_process_request [D 20:01:00.543002] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.543027] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.543052] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.543077] basic type or contiguous data [D 20:01:00.543101] Request disp [D 20:01:00.543125] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.543149] process a segment [D 20:01:00.543173] of 27262976 sz 2097152 [D 20:01:00.543202] add a segment 0 [D 20:01:00.543229] retval = 2097152 [D 20:01:00.543256] exiting distribute returned less than expected [D 20:01:00.543284] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.543314] *****RETURN***FROM***PROCESS***** [D 20:01:00.543339] end iteration [D 20:01:00.543364] done with segments or bytes [D 20:01:00.543387] of 398458880 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.543413] next loff: 398458880 next poff: 398458880 [D 20:01:00.543438] return value 2097152 [D 20:01:00.543462] finished [D 20:01:00.543486] retval = 2097152 [D 20:01:00.543510] exiting distribute returned less than expected [D 20:01:00.543534] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 398458880 fo 402653184 eof 0 [D 20:01:00.543560] ========================================================= [D 20:01:00.543584] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:01:00.543610] element 0: offset: 0xade5b000, size: 2097152 [D 20:01:00.543637] post_sr: da13:3346 bh 38 len 32 wr_id 15e72f10. [D 20:01:00.543666] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:00.543692] ========================================================= [D 20:01:00.543716] PINT_process_request [D 20:01:00.543741] tiling 16 copies [D 20:01:00.543765] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.543791] lvl 0 el 0 blk 0 by 488636416 [D 20:01:00.543816] to 488636416 ta 469762048 fi 503316480 [D 20:01:00.543841] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:00.543866] basic type or contiguous data [D 20:01:00.543889] Request disp [D 20:01:00.543913] contig_offset = 488636416 contig_size = 48234496 lvl_flag = 1 [D 20:01:00.543938] calling distribute [D 20:01:00.543963] PINT_distribute [D 20:01:00.543987] of 488636416 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 488636416 exfl 1 [D 20:01:00.544013] begin iteration loff: 488636416 [D 20:01:00.544038] segment exceeds byte limit [D 20:01:00.544063] file being extended [D 20:01:00.544087] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:00.544111] **********CALL***PROCESS********* [D 20:01:00.544136] segment of 18874368 sz 2097152 [D 20:01:00.544159] PINT_process_request [D 20:01:00.544184] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.544210] lvl 0 el 0 blk 0 by 18874368 [D 20:01:00.544235] to 18874368 ta 18874368 fi 20971520 [D 20:01:00.544260] basic type or contiguous data [D 20:01:00.544283] Request disp [D 20:01:00.544307] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:00.544332] process a segment [D 20:01:00.544356] of 18874368 sz 2097152 [D 20:01:00.544380] add a segment 0 [D 20:01:00.544404] retval = 2097152 [D 20:01:00.544427] exiting distribute returned less than expected [D 20:01:00.544451] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:00.544478] *****RETURN***FROM***PROCESS***** [D 20:01:00.544502] end iteration [D 20:01:00.544526] done with segments or bytes [D 20:01:00.544549] of 490733568 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.544575] next loff: 490733568 next poff: 490733568 [D 20:01:00.544600] return value 2097152 [D 20:01:00.544624] finished [D 20:01:00.544648] retval = 2097152 [D 20:01:00.544672] exiting distribute returned less than expected [D 20:01:00.544695] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 490733568 fo 503316480 eof 0 [D 20:01:00.544721] ========================================================= [D 20:01:00.544746] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:00.544772] element 0: offset: 0xa7655000, size: 2097152 [D 20:01:00.544798] post_sr: da13:3346 bh 39 len 32 wr_id 15e72f28. [D 20:01:00.544827] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:01:00.544854] ========================================================= [D 20:01:00.544878] PINT_process_request [D 20:01:00.544902] tiling 14 copies [D 20:01:00.544926] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.544953] lvl 0 el 0 blk 0 by 429916160 [D 20:01:00.544977] to 429916160 ta 402653184 fi 436207616 [D 20:01:00.545004] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.545030] basic type or contiguous data [D 20:01:00.545054] Request disp [D 20:01:00.545078] contig_offset = 429916160 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.545103] calling distribute [D 20:01:00.545126] PINT_distribute [D 20:01:00.545150] of 429916160 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 429916160 exfl 1 [D 20:01:00.545177] begin iteration loff: 429916160 [D 20:01:00.545206] segment exceeds byte limit [D 20:01:00.545229] file being extended [D 20:01:00.545253] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.545278] **********CALL***PROCESS********* [D 20:01:00.545302] segment of 27262976 sz 2097152 [D 20:01:00.545327] PINT_process_request [D 20:01:00.545351] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.545376] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.545401] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.545426] basic type or contiguous data [D 20:01:00.545450] Request disp [D 20:01:00.545474] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.545498] process a segment [D 20:01:00.545522] of 27262976 sz 2097152 [D 20:01:00.545546] add a segment 0 [D 20:01:00.545571] retval = 2097152 [D 20:01:00.545595] exiting distribute returned less than expected [D 20:01:00.545619] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.545645] *****RETURN***FROM***PROCESS***** [D 20:01:00.545669] end iteration [D 20:01:00.545693] done with segments or bytes [D 20:01:00.545717] of 432013312 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.545742] next loff: 432013312 next poff: 432013312 [D 20:01:00.545766] return value 2097152 [D 20:01:00.545791] finished [D 20:01:00.545815] retval = 2097152 [D 20:01:00.545839] exiting distribute returned less than expected [D 20:01:00.545863] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 432013312 fo 436207616 eof 0 [D 20:01:00.545888] ========================================================= [D 20:01:00.545913] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:01:00.545939] element 0: offset: 0xabe59000, size: 2097152 [D 20:01:00.545966] post_sr: da13:3346 bh 40 len 32 wr_id 15e72f40. [D 20:01:00.545995] PINT_client_state_machine_test id 22 [D 20:01:00.546024] openib_check_cq: completed send wr_id 15e72ee0. [D 20:01:00.546050] openib_check_cq: completed send wr_id 15e72ef8. [D 20:01:00.546075] openib_check_cq: completed send wr_id 15e72f10. [D 20:01:00.546100] openib_check_cq: completed send wr_id 15e72f28. [D 20:01:00.546126] openib_check_cq: completed send wr_id 15e72f40. [D 20:01:00.546159] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:01:00.546185] ========================================================= [D 20:01:00.546210] PINT_process_request [D 20:01:00.546234] tiling 15 copies [D 20:01:00.546258] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.546284] lvl 0 el 0 blk 0 by 463470592 [D 20:01:00.546309] to 463470592 ta 436207616 fi 469762048 [D 20:01:00.546334] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:00.546359] basic type or contiguous data [D 20:01:00.546382] Request disp [D 20:01:00.546406] contig_offset = 463470592 contig_size = 39845888 lvl_flag = 1 [D 20:01:00.546431] calling distribute [D 20:01:00.546455] PINT_distribute [D 20:01:00.546479] of 463470592 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 463470592 exfl 1 [D 20:01:00.546505] begin iteration loff: 463470592 [D 20:01:00.546530] segment exceeds byte limit [D 20:01:00.546554] file being extended [D 20:01:00.546578] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:00.546603] **********CALL***PROCESS********* [D 20:01:00.546627] segment of 27262976 sz 2097152 [D 20:01:00.546651] PINT_process_request [D 20:01:00.546675] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.546701] lvl 0 el 0 blk 0 by 27262976 [D 20:01:00.546730] to 27262976 ta 27262976 fi 29360128 [D 20:01:00.546756] basic type or contiguous data [D 20:01:00.546780] Request disp [D 20:01:00.546804] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:00.546829] process a segment [D 20:01:00.546853] of 27262976 sz 2097152 [D 20:01:00.546876] add a segment 0 [D 20:01:00.546900] retval = 2097152 [D 20:01:00.546924] exiting distribute returned less than expected [D 20:01:00.546949] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:00.546975] *****RETURN***FROM***PROCESS***** [D 20:01:00.546998] end iteration [D 20:01:00.547022] done with segments or bytes [D 20:01:00.547046] of 465567744 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.547072] next loff: 465567744 next poff: 465567744 [D 20:01:00.547097] return value 2097152 [D 20:01:00.547122] finished [D 20:01:00.547145] retval = 2097152 [D 20:01:00.547168] exiting distribute returned less than expected [D 20:01:00.547196] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 465567744 fo 469762048 eof 0 [D 20:01:00.547226] ========================================================= [D 20:01:00.547255] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:01:00.547285] element 0: offset: 0xa9e57000, size: 2097152 [D 20:01:00.547312] post_sr: da13:3346 bh 41 len 32 wr_id 15e72f58. [D 20:01:00.547342] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:01:00.547368] ========================================================= [D 20:01:00.547392] PINT_process_request [D 20:01:00.547416] tiling 2 copies [D 20:01:00.547441] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.547467] lvl 0 el 0 blk 0 by 29360128 [D 20:01:00.547492] to 29360128 ta 0 fi 33554432 [D 20:01:00.547516] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:00.547541] basic type or contiguous data [D 20:01:00.547565] Request disp [D 20:01:00.547589] contig_offset = 29360128 contig_size = 37748736 lvl_flag = 1 [D 20:01:00.547615] calling distribute [D 20:01:00.547638] PINT_distribute [D 20:01:00.547662] of 29360128 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 29360128 exfl 1 [D 20:01:00.547688] begin iteration loff: 29360128 [D 20:01:00.547713] segment exceeds byte limit [D 20:01:00.547738] file being extended [D 20:01:00.547760] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:00.547785] **********CALL***PROCESS********* [D 20:01:00.547809] segment of 29360128 sz 2097152 [D 20:01:00.547834] PINT_process_request [D 20:01:00.547858] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.547883] lvl 0 el 0 blk 0 by 29360128 [D 20:01:00.547944] to 29360128 ta 29360128 fi 31457280 [D 20:01:00.547969] basic type or contiguous data [D 20:01:00.547993] Request disp [D 20:01:00.548017] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:00.548042] process a segment [D 20:01:00.548066] of 29360128 sz 2097152 [D 20:01:00.548090] add a segment 0 [D 20:01:00.548114] retval = 2097152 [D 20:01:00.548138] exiting distribute returned less than expected [D 20:01:00.548162] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:00.548189] *****RETURN***FROM***PROCESS***** [D 20:01:00.548213] end iteration [D 20:01:00.548236] done with segments or bytes [D 20:01:00.548260] of 31457280 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.548286] next loff: 31457280 next poff: 31457280 [D 20:01:00.548311] return value 2097152 [D 20:01:00.548335] finished [D 20:01:00.548358] retval = 2097152 [D 20:01:00.548382] exiting distribute returned less than expected [D 20:01:00.548406] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 31457280 fo 33554432 eof 0 [D 20:01:00.548432] ========================================================= [D 20:01:00.548457] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:01:00.548481] element 0: offset: 0xc4071000, size: 2097152 [D 20:01:00.548512] post_sr: da13:3346 bh 42 len 32 wr_id 15e72f70. [D 20:01:00.548542] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:01:00.548567] ========================================================= [D 20:01:00.548591] PINT_process_request [D 20:01:00.548615] tiling 3 copies [D 20:01:00.548639] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.548666] lvl 0 el 0 blk 0 by 62914560 [D 20:01:00.548690] to 62914560 ta 33554432 fi 67108864 [D 20:01:00.548715] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:00.548740] basic type or contiguous data [D 20:01:00.548764] Request disp [D 20:01:00.548788] contig_offset = 62914560 contig_size = 37748736 lvl_flag = 1 [D 20:01:00.548812] calling distribute [D 20:01:00.548837] PINT_distribute [D 20:01:00.548861] of 62914560 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 62914560 exfl 1 [D 20:01:00.548887] begin iteration loff: 62914560 [D 20:01:00.548912] segment exceeds byte limit [D 20:01:00.548935] file being extended [D 20:01:00.548959] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:00.548984] **********CALL***PROCESS********* [D 20:01:00.549008] segment of 29360128 sz 2097152 [D 20:01:00.549032] PINT_process_request [D 20:01:00.549056] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.549081] lvl 0 el 0 blk 0 by 29360128 [D 20:01:00.549106] to 29360128 ta 29360128 fi 31457280 [D 20:01:00.549131] basic type or contiguous data [D 20:01:00.549155] Request disp [D 20:01:00.549179] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:00.549206] process a segment [D 20:01:00.549230] of 29360128 sz 2097152 [D 20:01:00.549254] add a segment 0 [D 20:01:00.549278] retval = 2097152 [D 20:01:00.549302] exiting distribute returned less than expected [D 20:01:00.549326] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:00.549352] *****RETURN***FROM***PROCESS***** [D 20:01:00.549377] end iteration [D 20:01:00.549400] done with segments or bytes [D 20:01:00.549424] of 65011712 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.549449] next loff: 65011712 next poff: 65011712 [D 20:01:00.549474] return value 2097152 [D 20:01:00.549499] finished [D 20:01:00.549522] retval = 2097152 [D 20:01:00.549547] exiting distribute returned less than expected [D 20:01:00.549571] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 65011712 fo 67108864 eof 0 [D 20:01:00.549596] ========================================================= [D 20:01:00.549621] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:01:00.549646] element 0: offset: 0xc206f000, size: 2097152 [D 20:01:00.549674] post_sr: da13:3346 bh 43 len 32 wr_id 15e72f88. [D 20:01:00.549702] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:01:00.549728] ========================================================= [D 20:01:00.549753] PINT_process_request [D 20:01:00.549777] tiling 4 copies [D 20:01:00.549800] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.549826] lvl 0 el 0 blk 0 by 96468992 [D 20:01:00.549851] to 96468992 ta 67108864 fi 100663296 [D 20:01:00.549876] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:00.549901] basic type or contiguous data [D 20:01:00.549924] Request disp [D 20:01:00.549948] contig_offset = 96468992 contig_size = 37748736 lvl_flag = 1 [D 20:01:00.549974] calling distribute [D 20:01:00.549998] PINT_distribute [D 20:01:00.550022] of 96468992 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 96468992 exfl 1 [D 20:01:00.550048] begin iteration loff: 96468992 [D 20:01:00.550073] segment exceeds byte limit [D 20:01:00.550097] file being extended [D 20:01:00.550121] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:00.550146] **********CALL***PROCESS********* [D 20:01:00.550169] segment of 29360128 sz 2097152 [D 20:01:00.550193] PINT_process_request [D 20:01:00.550220] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.550247] lvl 0 el 0 blk 0 by 29360128 [D 20:01:00.550271] to 29360128 ta 29360128 fi 31457280 [D 20:01:00.550296] basic type or contiguous data [D 20:01:00.550320] Request disp [D 20:01:00.550344] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:00.550369] process a segment [D 20:01:00.550393] of 29360128 sz 2097152 [D 20:01:00.550417] add a segment 0 [D 20:01:00.550440] retval = 2097152 [D 20:01:00.550465] exiting distribute returned less than expected [D 20:01:00.550489] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:00.550515] *****RETURN***FROM***PROCESS***** [D 20:01:00.550539] end iteration [D 20:01:00.550562] done with segments or bytes [D 20:01:00.550586] of 98566144 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.550612] next loff: 98566144 next poff: 98566144 [D 20:01:00.550637] return value 2097152 [D 20:01:00.550661] finished [D 20:01:00.550684] retval = 2097152 [D 20:01:00.550708] exiting distribute returned less than expected [D 20:01:00.550732] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 98566144 fo 100663296 eof 0 [D 20:01:00.550759] ========================================================= [D 20:01:00.550783] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:01:00.550808] element 0: offset: 0xc006d000, size: 2097152 [D 20:01:00.550835] post_sr: da13:3346 bh 44 len 32 wr_id 15e72fa0. [D 20:01:00.550864] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:01:00.550890] ========================================================= [D 20:01:00.550914] PINT_process_request [D 20:01:00.550939] tiling 5 copies [D 20:01:00.550963] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.550989] lvl 0 el 0 blk 0 by 130023424 [D 20:01:00.551013] to 130023424 ta 100663296 fi 134217728 [D 20:01:00.551039] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:00.551064] basic type or contiguous data [D 20:01:00.551088] Request disp [D 20:01:00.551112] contig_offset = 130023424 contig_size = 37748736 lvl_flag = 1 [D 20:01:00.551136] calling distribute [D 20:01:00.551160] PINT_distribute [D 20:01:00.551187] of 130023424 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 130023424 exfl 1 [D 20:01:00.551218] begin iteration loff: 130023424 [D 20:01:00.551246] segment exceeds byte limit [D 20:01:00.551274] file being extended [D 20:01:00.551301] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:00.551325] **********CALL***PROCESS********* [D 20:01:00.551349] segment of 29360128 sz 2097152 [D 20:01:00.551374] PINT_process_request [D 20:01:00.551397] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.551424] lvl 0 el 0 blk 0 by 29360128 [D 20:01:00.551448] to 29360128 ta 29360128 fi 31457280 [D 20:01:00.551473] basic type or contiguous data [D 20:01:00.551497] Request disp [D 20:01:00.551521] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:00.551546] process a segment [D 20:01:00.551569] of 29360128 sz 2097152 [D 20:01:00.551593] add a segment 0 [D 20:01:00.551617] retval = 2097152 [D 20:01:00.551641] exiting distribute returned less than expected [D 20:01:00.551665] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:00.551692] *****RETURN***FROM***PROCESS***** [D 20:01:00.551715] end iteration [D 20:01:00.551739] done with segments or bytes [D 20:01:00.551763] of 132120576 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.551789] next loff: 132120576 next poff: 132120576 [D 20:01:00.551813] return value 2097152 [D 20:01:00.551837] finished [D 20:01:00.551860] retval = 2097152 [D 20:01:00.551884] exiting distribute returned less than expected [D 20:01:00.551909] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 132120576 fo 134217728 eof 0 [D 20:01:00.551935] ========================================================= [D 20:01:00.551961] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:01:00.551987] element 0: offset: 0xbe06b000, size: 2097152 [D 20:01:00.552015] post_sr: da13:3346 bh 45 len 32 wr_id 15e72fb8. [D 20:01:00.552045] openib_check_cq: completed send wr_id 15e72f58. [D 20:01:00.552072] openib_check_cq: completed send wr_id 15e72f70. [D 20:01:00.552097] openib_check_cq: completed send wr_id 15e72f88. [D 20:01:00.552123] openib_check_cq: completed send wr_id 15e72fa0. [D 20:01:00.552147] openib_check_cq: completed send wr_id 15e72fb8. [D 20:01:00.552179] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:00.552206] ========================================================= [D 20:01:00.552229] PINT_process_request [D 20:01:00.552253] tiling 2 copies [D 20:01:00.552278] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.552304] lvl 0 el 0 blk 0 by 2097152 [D 20:01:00.552329] to 2097152 ta 0 fi 33554432 [D 20:01:00.552355] mto 2097152 mta 0 mfi 2097152 [D 20:01:00.552379] basic type or contiguous data [D 20:01:00.552404] Request disp [D 20:01:00.552428] contig_offset = 2097152 contig_size = 65011712 lvl_flag = 1 [D 20:01:00.552453] calling distribute [D 20:01:00.552476] PINT_distribute [D 20:01:00.552500] of 2097152 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:00.552526] begin iteration loff: 2097152 [D 20:01:00.552551] segment exceeds byte limit [D 20:01:00.552576] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:00.552600] **********CALL***PROCESS********* [D 20:01:00.552624] segment of 2097152 sz 2097152 [D 20:01:00.552648] PINT_process_request [D 20:01:00.552672] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.552699] lvl 0 el 0 blk 0 by 2097152 [D 20:01:00.552722] to 2097152 ta 2097152 fi 4194304 [D 20:01:00.552747] basic type or contiguous data [D 20:01:00.552771] Request disp [D 20:01:00.552795] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:00.552820] process a segment [D 20:01:00.552844] of 2097152 sz 2097152 [D 20:01:00.552868] add a segment 0 [D 20:01:00.552892] retval = 2097152 [D 20:01:00.552916] exiting distribute returned less than expected [D 20:01:00.552940] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:00.552966] *****RETURN***FROM***PROCESS***** [D 20:01:00.552990] end iteration [D 20:01:00.553014] done with segments or bytes [D 20:01:00.553038] of 4194304 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.553064] next loff: 4194304 next poff: 4194304 [D 20:01:00.553088] return value 2097152 [D 20:01:00.553112] finished [D 20:01:00.553136] retval = 2097152 [D 20:01:00.553160] exiting distribute returned less than expected [D 20:01:00.553186] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 4194304 fo 33554432 eof 0 [D 20:01:00.553213] ========================================================= [D 20:01:00.553237] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:00.553263] element 0: offset: 0xa2651000, size: 2097152 [D 20:01:00.553294] post_sr: da13:3346 bh 46 len 48 wr_id 15e72fd0. [D 20:01:00.553321] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:00.553347] ========================================================= [D 20:01:00.553372] PINT_process_request [D 20:01:00.553396] tiling 15 copies [D 20:01:00.553420] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.553445] lvl 0 el 0 blk 0 by 440401920 [D 20:01:00.553470] to 440401920 ta 436207616 fi 469762048 [D 20:01:00.553495] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:00.553521] basic type or contiguous data [D 20:01:00.553544] Request disp [D 20:01:00.553568] contig_offset = 440401920 contig_size = 62914560 lvl_flag = 1 [D 20:01:00.553593] calling distribute [D 20:01:00.553617] PINT_distribute [D 20:01:00.553641] of 440401920 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:00.553670] begin iteration loff: 440401920 [D 20:01:00.553695] segment exceeds byte limit [D 20:01:00.553719] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:00.553745] **********CALL***PROCESS********* [D 20:01:00.553768] segment of 4194304 sz 2097152 [D 20:01:00.553792] PINT_process_request [D 20:01:00.553816] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:00.553842] lvl 0 el 0 blk 0 by 4194304 [D 20:01:00.553867] to 4194304 ta 4194304 fi 6291456 [D 20:01:00.553892] basic type or contiguous data [D 20:01:00.553915] Request disp [D 20:01:00.553939] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:00.553964] process a segment [D 20:01:00.553988] of 4194304 sz 2097152 [D 20:01:00.554012] add a segment 0 [D 20:01:00.554036] retval = 2097152 [D 20:01:00.554059] exiting distribute returned less than expected [D 20:01:00.554083] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:00.554110] *****RETURN***FROM***PROCESS***** [D 20:01:00.554134] end iteration [D 20:01:00.554158] done with segments or bytes [D 20:01:00.554182] of 442499072 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:00.554207] next loff: 442499072 next poff: 442499072 [D 20:01:00.554232] return value 2097152 [D 20:01:00.554257] finished [D 20:01:00.554281] retval = 2097152 [D 20:01:00.554305] exiting distribute returned less than expected [D 20:01:00.554328] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 442499072 fo 469762048 eof 0 [D 20:01:00.554354] ========================================================= [D 20:01:00.554379] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:00.554405] element 0: offset: 0x88837000, size: 2097152 [D 20:01:00.554430] openib_check_cq: completed send wr_id 15e72fd0. [D 20:01:00.554460] post_sr: da13:3346 bh 47 len 48 wr_id 15e72fe8. [D 20:01:00.554495] openib_check_cq: completed send wr_id 15e72fe8. [D 20:01:00.556027] PINT_client_state_machine_test id 22 [D 20:01:00.566057] PINT_client_state_machine_test id 22 [D 20:01:00.576086] PINT_client_state_machine_test id 22 [D 20:01:00.586117] PINT_client_state_machine_test id 22 [D 20:01:00.596147] PINT_client_state_machine_test id 22 [D 20:01:00.606180] PINT_client_state_machine_test id 22 [D 20:01:00.621189] PINT_client_state_machine_test id 22 [D 20:01:00.633188] PINT_client_state_machine_test id 22 [D 20:01:00.645200] PINT_client_state_machine_test id 22 [D 20:01:00.657188] PINT_client_state_machine_test id 22 [D 20:01:00.669188] PINT_client_state_machine_test id 22 [D 20:01:00.681188] PINT_client_state_machine_test id 22 [D 20:01:00.693188] PINT_client_state_machine_test id 22 [D 20:01:00.705187] PINT_client_state_machine_test id 22 [D 20:01:00.717187] PINT_client_state_machine_test id 22 [D 20:01:00.729187] PINT_client_state_machine_test id 22 [D 20:01:00.741188] PINT_client_state_machine_test id 22 [D 20:01:00.753189] PINT_client_state_machine_test id 22 [D 20:01:00.765188] PINT_client_state_machine_test id 22 [D 20:01:00.777194] PINT_client_state_machine_test id 22 [D 20:01:00.789188] PINT_client_state_machine_test id 22 [D 20:01:00.801188] PINT_client_state_machine_test id 22 [D 20:01:00.813190] PINT_client_state_machine_test id 22 [D 20:01:00.825188] PINT_client_state_machine_test id 22 [D 20:01:00.837188] PINT_client_state_machine_test id 22 [D 20:01:00.849189] PINT_client_state_machine_test id 22 [D 20:01:00.861188] PINT_client_state_machine_test id 22 [D 20:01:00.873188] PINT_client_state_machine_test id 22 [D 20:01:00.885189] PINT_client_state_machine_test id 22 [D 20:01:00.897189] PINT_client_state_machine_test id 22 [D 20:01:00.909189] PINT_client_state_machine_test id 22 [D 20:01:00.921189] PINT_client_state_machine_test id 22 [D 20:01:00.933189] PINT_client_state_machine_test id 22 [D 20:01:00.945190] PINT_client_state_machine_test id 22 [D 20:01:00.957189] PINT_client_state_machine_test id 22 [D 20:01:00.969189] PINT_client_state_machine_test id 22 [D 20:01:00.981192] PINT_client_state_machine_test id 22 [D 20:01:00.993191] PINT_client_state_machine_test id 22 [D 20:01:01.005189] PINT_client_state_machine_test id 22 [D 20:01:01.017189] PINT_client_state_machine_test id 22 [D 20:01:01.029190] PINT_client_state_machine_test id 22 [D 20:01:01.041190] PINT_client_state_machine_test id 22 [D 20:01:01.053190] PINT_client_state_machine_test id 22 [D 20:01:01.065190] PINT_client_state_machine_test id 22 [D 20:01:01.077189] PINT_client_state_machine_test id 22 [D 20:01:01.089190] PINT_client_state_machine_test id 22 [D 20:01:01.101222] PINT_client_state_machine_test id 22 [D 20:01:01.113194] PINT_client_state_machine_test id 22 [D 20:01:01.125189] PINT_client_state_machine_test id 22 [D 20:01:01.137190] PINT_client_state_machine_test id 22 [D 20:01:01.149190] PINT_client_state_machine_test id 22 [D 20:01:01.161190] PINT_client_state_machine_test id 22 [D 20:01:01.173190] PINT_client_state_machine_test id 22 [D 20:01:01.185190] PINT_client_state_machine_test id 22 [D 20:01:01.197191] PINT_client_state_machine_test id 22 [D 20:01:01.209190] PINT_client_state_machine_test id 22 [D 20:01:01.221190] PINT_client_state_machine_test id 22 [D 20:01:01.233192] PINT_client_state_machine_test id 22 [D 20:01:01.245190] PINT_client_state_machine_test id 22 [D 20:01:01.257190] PINT_client_state_machine_test id 22 [D 20:01:01.269190] PINT_client_state_machine_test id 22 [D 20:01:01.281190] PINT_client_state_machine_test id 22 [D 20:01:01.293190] PINT_client_state_machine_test id 22 [D 20:01:01.305191] PINT_client_state_machine_test id 22 [D 20:01:01.317191] PINT_client_state_machine_test id 22 [D 20:01:01.329192] PINT_client_state_machine_test id 22 [D 20:01:01.341191] PINT_client_state_machine_test id 22 [D 20:01:01.353191] PINT_client_state_machine_test id 22 [D 20:01:01.365191] PINT_client_state_machine_test id 22 [D 20:01:01.377191] PINT_client_state_machine_test id 22 [D 20:01:01.389191] PINT_client_state_machine_test id 22 [D 20:01:01.401191] PINT_client_state_machine_test id 22 [D 20:01:01.413191] PINT_client_state_machine_test id 22 [D 20:01:01.413226] sm control op 300 [D 20:01:01.413255] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:01:01.413281] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:01:01.413311] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:01:01.413337] PINT_client_state_machine_test id 22 [D 20:01:01.425192] PINT_client_state_machine_test id 22 [D 20:01:01.437191] PINT_client_state_machine_test id 22 [D 20:01:01.449192] PINT_client_state_machine_test id 22 [D 20:01:01.461192] PINT_client_state_machine_test id 22 [D 20:01:01.473192] PINT_client_state_machine_test id 22 [D 20:01:01.485192] PINT_client_state_machine_test id 22 [D 20:01:01.497192] PINT_client_state_machine_test id 22 [D 20:01:01.509191] PINT_client_state_machine_test id 22 [D 20:01:01.521193] PINT_client_state_machine_test id 22 [D 20:01:01.533192] PINT_client_state_machine_test id 22 [D 20:01:01.545192] PINT_client_state_machine_test id 22 [D 20:01:01.557192] PINT_client_state_machine_test id 22 [D 20:01:01.569191] PINT_client_state_machine_test id 22 [D 20:01:01.581192] PINT_client_state_machine_test id 22 [D 20:01:01.593193] PINT_client_state_machine_test id 22 [D 20:01:01.605192] PINT_client_state_machine_test id 22 [D 20:01:01.617193] PINT_client_state_machine_test id 22 [D 20:01:01.629193] PINT_client_state_machine_test id 22 [D 20:01:01.641192] PINT_client_state_machine_test id 22 [D 20:01:01.653192] PINT_client_state_machine_test id 22 [D 20:01:01.665192] PINT_client_state_machine_test id 22 [D 20:01:01.677192] PINT_client_state_machine_test id 22 [D 20:01:01.689192] PINT_client_state_machine_test id 22 [D 20:01:01.701192] PINT_client_state_machine_test id 22 [D 20:01:01.713193] PINT_client_state_machine_test id 22 [D 20:01:01.725193] PINT_client_state_machine_test id 22 [D 20:01:01.737193] PINT_client_state_machine_test id 22 [D 20:01:01.749193] PINT_client_state_machine_test id 22 [D 20:01:01.761193] PINT_client_state_machine_test id 22 [D 20:01:01.773197] PINT_client_state_machine_test id 22 [D 20:01:01.785193] PINT_client_state_machine_test id 22 [D 20:01:01.797193] PINT_client_state_machine_test id 22 [D 20:01:01.809194] PINT_client_state_machine_test id 22 [D 20:01:01.821193] PINT_client_state_machine_test id 22 [D 20:01:01.833193] PINT_client_state_machine_test id 22 [D 20:01:01.845193] PINT_client_state_machine_test id 22 [D 20:01:01.857193] PINT_client_state_machine_test id 22 [D 20:01:01.869194] PINT_client_state_machine_test id 22 [D 20:01:01.881194] PINT_client_state_machine_test id 22 [D 20:01:01.893194] PINT_client_state_machine_test id 22 [D 20:01:01.905195] PINT_client_state_machine_test id 22 [D 20:01:01.917194] PINT_client_state_machine_test id 22 [D 20:01:01.929194] PINT_client_state_machine_test id 22 [D 20:01:01.941194] PINT_client_state_machine_test id 22 [D 20:01:01.953194] PINT_client_state_machine_test id 22 [D 20:01:01.965194] PINT_client_state_machine_test id 22 [D 20:01:01.977194] PINT_client_state_machine_test id 22 [D 20:01:01.989194] PINT_client_state_machine_test id 22 [D 20:01:02.001195] PINT_client_state_machine_test id 22 [D 20:01:02.013195] PINT_client_state_machine_test id 22 [D 20:01:02.025197] PINT_client_state_machine_test id 22 [D 20:01:02.037195] PINT_client_state_machine_test id 22 [D 20:01:02.049195] PINT_client_state_machine_test id 22 [D 20:01:02.061194] PINT_client_state_machine_test id 22 [D 20:01:02.073194] PINT_client_state_machine_test id 22 [D 20:01:02.085195] PINT_client_state_machine_test id 22 [D 20:01:02.097195] PINT_client_state_machine_test id 22 [D 20:01:02.109199] PINT_client_state_machine_test id 22 [D 20:01:02.121196] PINT_client_state_machine_test id 22 [D 20:01:02.133194] PINT_client_state_machine_test id 22 [D 20:01:02.145195] PINT_client_state_machine_test id 22 [D 20:01:02.157196] PINT_client_state_machine_test id 22 [D 20:01:02.169195] PINT_client_state_machine_test id 22 [D 20:01:02.181195] PINT_client_state_machine_test id 22 [D 20:01:02.193196] PINT_client_state_machine_test id 22 [D 20:01:02.205195] PINT_client_state_machine_test id 22 [D 20:01:02.217195] PINT_client_state_machine_test id 22 [D 20:01:02.229195] PINT_client_state_machine_test id 22 -[D 20:01:02.240056] openib_check_cq: completed recv wr_id 15e739e0. [D 20:01:02.240097] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab6027010 rkey 48007000 nic_wr_credit 1. [D 20:01:02.240125] openib_post_rr: da13:3346 bh 53 wr_id 15e739e0. [D 20:01:02.240152] openib_check_cq: completed recv wr_id 15e739f8. [D 20:01:02.240184] openib_post_rr: da13:3346 bh 54 wr_id 15e739f8. [D 20:01:02.240216] openib_check_cq: completed recv wr_id 15e73a10. [D 20:01:02.240246] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab6228010 rkey 48007100 nic_wr_credit 2. [D 20:01:02.240278] openib_post_rr: da13:3346 bh 55 wr_id 15e73a10. [D 20:01:02.240307] openib_check_cq: completed recv wr_id 15e73a28. [D 20:01:02.240334] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab642a010 rkey 48007200 nic_wr_credit 3. [D 20:01:02.240361] openib_post_rr: da13:3346 bh 56 wr_id 15e73a28. [D 20:01:02.240387] openib_check_cq: completed recv wr_id 15e73a40. [D 20:01:02.240414] openib_post_rr: da13:3346 bh 57 wr_id 15e73a40. [D 20:01:02.240440] openib_check_cq: completed recv wr_id 15e73a58. [D 20:01:02.240467] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab662b010 rkey 48007300 nic_wr_credit 4. [D 20:01:02.240493] openib_post_rr: da13:3346 bh 58 wr_id 15e73a58. [D 20:01:02.240519] openib_check_cq: completed recv wr_id 15e73a70. [D 20:01:02.240546] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaab682c010 rkey 48007400 nic_wr_credit 5. [D 20:01:02.240572] openib_post_rr: da13:3346 bh 59 wr_id 15e73a70. [D 20:01:02.240598] openib_check_cq: completed recv wr_id 15e73a88. [D 20:01:02.240625] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab6a2d010 rkey 48007500 nic_wr_credit 6. [D 20:01:02.240656] openib_post_rr: da13:3346 bh 60 wr_id 15e73a88. [D 20:01:02.240683] openib_check_cq: completed recv wr_id 15e73aa0. [D 20:01:02.240710] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab6c2e010 rkey 48007600 nic_wr_credit 7. [D 20:01:02.240736] openib_post_rr: da13:3346 bh 61 wr_id 15e73aa0. [D 20:01:02.240762] openib_check_cq: completed recv wr_id 15e73ab8. [D 20:01:02.240790] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaab6e2f010 rkey 48007700 nic_wr_credit 8. [D 20:01:02.240817] openib_post_rr: da13:3346 bh 62 wr_id 15e73ab8. [D 20:01:02.240842] openib_check_cq: completed recv wr_id 15e73ad0. [D 20:01:02.240869] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaab5c25010 rkey c8006e00 nic_wr_credit 9. [D 20:01:02.240896] openib_post_rr: da13:3346 bh 63 wr_id 15e73ad0. [D 20:01:02.240921] openib_check_cq: completed recv wr_id 15e73ae8. [D 20:01:02.240948] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa5b0 to da13:3346 remote addr 2aaab7030010 rkey c8007800 nic_wr_credit 10. [D 20:01:02.240975] openib_post_rr: da13:3346 bh 64 wr_id 15e73ae8. [D 20:01:02.241002] openib_check_cq: completed recv wr_id 15e73b00. [D 20:01:02.241028] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab7231010 rkey c8007900 nic_wr_credit 11. [D 20:01:02.241055] openib_post_rr: da13:3346 bh 65 wr_id 15e73b00. [D 20:01:02.241081] openib_check_cq: completed recv wr_id 15e73b18. [D 20:01:02.241108] openib_post_rr: da13:3346 bh 66 wr_id 15e73b18. [D 20:01:02.241135] openib_check_cq: completed recv wr_id 15e73b30. [D 20:01:02.241162] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab7432010 rkey c8007a00 nic_wr_credit 12. [D 20:01:02.241189] openib_post_rr: da13:3346 bh 67 wr_id 15e73b30. [D 20:01:02.241214] openib_check_cq: completed recv wr_id 15e73b48. [D 20:01:02.241241] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab7633010 rkey c8007b00 nic_wr_credit 13. [D 20:01:02.241269] openib_post_rr: da13:3346 bh 68 wr_id 15e73b48. [D 20:01:02.241295] openib_check_cq: completed recv wr_id 15e73b60. [D 20:01:02.241322] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab7834010 rkey c8007c00 nic_wr_credit 14. [D 20:01:02.241350] openib_post_rr: da13:3346 bh 69 wr_id 15e73b60. [D 20:01:02.241375] openib_check_cq: completed recv wr_id 15e73b78. [D 20:01:02.241402] openib_post_sr_rdmaw: ibv_post_send wr_id 15efcfd0 to da13:3346 remote addr 2aaab7a35010 rkey c8007d00 nic_wr_credit 15. [D 20:01:02.241430] openib_post_rr: da13:3346 bh 70 wr_id 15e73b78. [D 20:01:02.241456] openib_check_cq: completed recv wr_id 15e73b90. [D 20:01:02.241482] openib_post_rr: da13:3346 bh 71 wr_id 15e73b90. [D 20:01:02.241519] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.241546] ========================================================= [D 20:01:02.241571] PINT_process_request [D 20:01:02.241595] tiling 3 copies [D 20:01:02.241620] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.241646] lvl 0 el 0 blk 0 by 35651584 [D 20:01:02.241671] to 35651584 ta 33554432 fi 67108864 [D 20:01:02.241696] mto 2097152 mta 0 mfi 2097152 [D 20:01:02.241721] basic type or contiguous data [D 20:01:02.241744] Request disp [D 20:01:02.241768] contig_offset = 35651584 contig_size = 65011712 lvl_flag = 1 [D 20:01:02.241793] calling distribute [D 20:01:02.241817] PINT_distribute [D 20:01:02.241841] of 35651584 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.241868] begin iteration loff: 35651584 [D 20:01:02.241893] segment exceeds byte limit [D 20:01:02.241917] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:02.241942] **********CALL***PROCESS********* [D 20:01:02.241966] segment of 2097152 sz 2097152 [D 20:01:02.241991] PINT_process_request [D 20:01:02.242014] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.242043] lvl 0 el 0 blk 0 by 2097152 [D 20:01:02.242068] to 2097152 ta 2097152 fi 4194304 [D 20:01:02.242093] basic type or contiguous data [D 20:01:02.242116] Request disp [D 20:01:02.242140] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:02.242165] process a segment [D 20:01:02.242192] of 2097152 sz 2097152 [D 20:01:02.242218] add a segment 0 [D 20:01:02.242241] retval = 2097152 [D 20:01:02.242265] exiting distribute returned less than expected [D 20:01:02.242289] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:02.242315] *****RETURN***FROM***PROCESS***** [D 20:01:02.242340] end iteration [D 20:01:02.242363] done with segments or bytes [D 20:01:02.242387] of 37748736 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.242413] next loff: 37748736 next poff: 37748736 [D 20:01:02.242438] return value 2097152 [D 20:01:02.242462] finished [D 20:01:02.242486] retval = 2097152 [D 20:01:02.242509] exiting distribute returned less than expected [D 20:01:02.242533] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 37748736 fo 67108864 eof 0 [D 20:01:02.242559] ========================================================= [D 20:01:02.242584] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.242609] element 0: offset: 0xa064f000, size: 2097152 [D 20:01:02.242636] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 14. [D 20:01:02.242663] post_sr: da13:3346 bh 48 len 16 wr_id 15e73000. [D 20:01:02.242694] post_sr: da13:3346 bh 49 len 48 wr_id 15e73018. [D 20:01:02.242722] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.242748] ========================================================= [D 20:01:02.242773] PINT_process_request [D 20:01:02.242796] tiling 15 copies [D 20:01:02.242820] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.242847] lvl 0 el 0 blk 0 by 442499072 [D 20:01:02.242872] to 442499072 ta 436207616 fi 469762048 [D 20:01:02.242897] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:02.242921] basic type or contiguous data [D 20:01:02.242945] Request disp [D 20:01:02.242969] contig_offset = 442499072 contig_size = 60817408 lvl_flag = 1 [D 20:01:02.242994] calling distribute [D 20:01:02.243018] PINT_distribute [D 20:01:02.243041] of 442499072 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.243068] begin iteration loff: 442499072 [D 20:01:02.243093] segment exceeds byte limit [D 20:01:02.243117] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:02.243142] **********CALL***PROCESS********* [D 20:01:02.243165] segment of 6291456 sz 2097152 [D 20:01:02.243190] PINT_process_request [D 20:01:02.243214] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.243240] lvl 0 el 0 blk 0 by 6291456 [D 20:01:02.243265] to 6291456 ta 6291456 fi 8388608 [D 20:01:02.243289] basic type or contiguous data [D 20:01:02.243313] Request disp [D 20:01:02.243337] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:02.243362] process a segment [D 20:01:02.243386] of 6291456 sz 2097152 [D 20:01:02.243410] add a segment 0 [D 20:01:02.243433] retval = 2097152 [D 20:01:02.243457] exiting distribute returned less than expected [D 20:01:02.243481] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:02.243507] *****RETURN***FROM***PROCESS***** [D 20:01:02.243532] end iteration [D 20:01:02.243556] done with segments or bytes [D 20:01:02.243579] of 444596224 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.243605] next loff: 444596224 next poff: 444596224 [D 20:01:02.243630] return value 2097152 [D 20:01:02.243655] finished [D 20:01:02.243679] retval = 2097152 [D 20:01:02.243702] exiting distribute returned less than expected [D 20:01:02.243726] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 444596224 fo 469762048 eof 0 [D 20:01:02.243752] ========================================================= [D 20:01:02.243780] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.243804] element 0: offset: 0x88a37000, size: 2097152 [D 20:01:02.243834] post_sr: da13:3346 bh 50 len 48 wr_id 15e73030. [D 20:01:02.243861] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.243886] ========================================================= [D 20:01:02.243911] PINT_process_request [D 20:01:02.243935] tiling 2 copies [D 20:01:02.243959] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.243985] lvl 0 el 0 blk 0 by 4194304 [D 20:01:02.244009] to 4194304 ta 0 fi 33554432 [D 20:01:02.244034] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:02.244059] basic type or contiguous data [D 20:01:02.244083] Request disp [D 20:01:02.244107] contig_offset = 4194304 contig_size = 62914560 lvl_flag = 1 [D 20:01:02.244132] calling distribute [D 20:01:02.244155] PINT_distribute [D 20:01:02.244179] of 4194304 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.244210] begin iteration loff: 4194304 [D 20:01:02.244238] segment exceeds byte limit [D 20:01:02.244266] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:02.244295] **********CALL***PROCESS********* [D 20:01:02.244319] segment of 4194304 sz 2097152 [D 20:01:02.244344] PINT_process_request [D 20:01:02.244368] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.244394] lvl 0 el 0 blk 0 by 4194304 [D 20:01:02.244419] to 4194304 ta 4194304 fi 6291456 [D 20:01:02.244443] basic type or contiguous data [D 20:01:02.244467] Request disp [D 20:01:02.244491] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:02.244516] process a segment [D 20:01:02.244540] of 4194304 sz 2097152 [D 20:01:02.244564] add a segment 0 [D 20:01:02.244588] retval = 2097152 [D 20:01:02.244612] exiting distribute returned less than expected [D 20:01:02.244636] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:02.244662] *****RETURN***FROM***PROCESS***** [D 20:01:02.244686] end iteration [D 20:01:02.244709] done with segments or bytes [D 20:01:02.244733] of 6291456 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.244759] next loff: 6291456 next poff: 6291456 [D 20:01:02.244784] return value 2097152 [D 20:01:02.244808] finished [D 20:01:02.244832] retval = 2097152 [D 20:01:02.244855] exiting distribute returned less than expected [D 20:01:02.244879] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 6291456 fo 33554432 eof 0 [D 20:01:02.244905] ========================================================= [D 20:01:02.244930] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.244955] element 0: offset: 0xa2851000, size: 2097152 [D 20:01:02.244980] openib_check_cq: completed recv wr_id 15e73ba8. [D 20:01:02.245008] openib_post_rr: da13:3346 bh 72 wr_id 15e73ba8. [D 20:01:02.245038] post_sr: da13:3346 bh 51 len 48 wr_id 15e73048. [D 20:01:02.245068] PINT_client_state_machine_test id 22 [D 20:01:02.245098] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 13. [D 20:01:02.245124] post_sr: da13:3346 bh 52 len 16 wr_id 15e73060. [D 20:01:02.245151] openib_check_cq: completed recv wr_id 15e73bc0. [D 20:01:02.245179] openib_post_rr: da13:3346 bh 73 wr_id 15e73bc0. [D 20:01:02.247512] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 12. [D 20:01:02.247540] post_sr: da13:3346 bh 53 len 16 wr_id 15e73078. [D 20:01:02.249971] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 11. [D 20:01:02.249999] post_sr: da13:3346 bh 54 len 16 wr_id 15e73090. [D 20:01:02.252428] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 10. [D 20:01:02.252455] post_sr: da13:3346 bh 55 len 16 wr_id 15e730a8. [D 20:01:02.254885] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 9. [D 20:01:02.254913] post_sr: da13:3346 bh 56 len 16 wr_id 15e730c0. [D 20:01:02.255096] PINT_client_state_machine_test id 22 [D 20:01:02.257347] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 8. [D 20:01:02.257375] post_sr: da13:3346 bh 57 len 16 wr_id 15e730d8. [D 20:01:02.259812] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 7. [D 20:01:02.259840] post_sr: da13:3346 bh 58 len 16 wr_id 15e730f0. [D 20:01:02.262269] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 6. [D 20:01:02.262297] post_sr: da13:3346 bh 59 len 16 wr_id 15e73108. [D 20:01:02.264726] openib_check_cq: completed rdma wr_id 15efa5b0, nic_wr_credit 5. [D 20:01:02.264754] post_sr: da13:3346 bh 60 len 16 wr_id 15e73120. [D 20:01:02.265129] PINT_client_state_machine_test id 22 [D 20:01:02.267189] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 4. [D 20:01:02.267217] post_sr: da13:3346 bh 61 len 16 wr_id 15e73138. [D 20:01:02.269653] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 3. [D 20:01:02.269681] post_sr: da13:3346 bh 62 len 16 wr_id 15e73150. [D 20:01:02.272116] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 2. [D 20:01:02.272144] post_sr: da13:3346 bh 63 len 16 wr_id 15e73168. [D 20:01:02.274577] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 1. [D 20:01:02.274605] post_sr: da13:3346 bh 64 len 16 wr_id 15e73180. [D 20:01:02.275156] PINT_client_state_machine_test id 22 [D 20:01:02.277034] openib_check_cq: completed rdma wr_id 15efcfd0, nic_wr_credit 0. [D 20:01:02.277062] post_sr: da13:3346 bh 65 len 16 wr_id 15e73198. [D 20:01:02.277089] openib_check_cq: completed send wr_id 15e73000. [D 20:01:02.277114] openib_check_cq: completed send wr_id 15e73018. [D 20:01:02.277140] openib_check_cq: completed send wr_id 15e73030. [D 20:01:02.277165] openib_check_cq: completed send wr_id 15e73048. [D 20:01:02.277191] openib_check_cq: completed send wr_id 15e73060. [D 20:01:02.277215] openib_check_cq: completed send wr_id 15e73078. [D 20:01:02.277240] openib_check_cq: completed send wr_id 15e73090. [D 20:01:02.277266] openib_check_cq: completed send wr_id 15e730a8. [D 20:01:02.277291] openib_check_cq: completed send wr_id 15e730c0. [D 20:01:02.277317] openib_check_cq: completed send wr_id 15e730d8. [D 20:01:02.277341] openib_check_cq: completed send wr_id 15e730f0. [D 20:01:02.277366] openib_check_cq: completed send wr_id 15e73108. [D 20:01:02.277392] openib_check_cq: completed send wr_id 15e73120. [D 20:01:02.277417] openib_check_cq: completed send wr_id 15e73138. [D 20:01:02.277441] openib_check_cq: completed send wr_id 15e73150. [D 20:01:02.277467] openib_check_cq: completed send wr_id 15e73168. [D 20:01:02.277492] openib_check_cq: completed send wr_id 15e73180. [D 20:01:02.277518] openib_check_cq: completed send wr_id 15e73198. [D 20:01:02.277551] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:01:02.277577] ========================================================= [D 20:01:02.277601] PINT_process_request [D 20:01:02.277627] tiling 6 copies [D 20:01:02.277652] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.277678] lvl 0 el 0 blk 0 by 163577856 [D 20:01:02.277703] to 163577856 ta 134217728 fi 167772160 [D 20:01:02.277728] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.277752] basic type or contiguous data [D 20:01:02.277776] Request disp [D 20:01:02.277800] contig_offset = 163577856 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.277825] calling distribute [D 20:01:02.277849] PINT_distribute [D 20:01:02.277873] of 163577856 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 163577856 exfl 1 [D 20:01:02.277899] begin iteration loff: 163577856 [D 20:01:02.277924] segment exceeds byte limit [D 20:01:02.277948] file being extended [D 20:01:02.277972] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.277997] **********CALL***PROCESS********* [D 20:01:02.278020] segment of 29360128 sz 2097152 [D 20:01:02.278044] PINT_process_request [D 20:01:02.278068] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.278095] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.278121] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.278147] basic type or contiguous data [D 20:01:02.278171] Request disp [D 20:01:02.278199] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.278224] process a segment [D 20:01:02.278248] of 29360128 sz 2097152 [D 20:01:02.278272] add a segment 0 [D 20:01:02.278297] retval = 2097152 [D 20:01:02.278321] exiting distribute returned less than expected [D 20:01:02.278345] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.278370] *****RETURN***FROM***PROCESS***** [D 20:01:02.278395] end iteration [D 20:01:02.278419] done with segments or bytes [D 20:01:02.278443] of 165675008 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.278469] next loff: 165675008 next poff: 165675008 [D 20:01:02.278493] return value 2097152 [D 20:01:02.278517] finished [D 20:01:02.278541] retval = 2097152 [D 20:01:02.278565] exiting distribute returned less than expected [D 20:01:02.278589] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 165675008 fo 167772160 eof 0 [D 20:01:02.278614] ========================================================= [D 20:01:02.278639] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:01:02.278665] element 0: offset: 0xbc069000, size: 2097152 [D 20:01:02.278692] post_sr: da13:3346 bh 66 len 32 wr_id 15e731b0. [D 20:01:02.278721] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:01:02.278747] ========================================================= [D 20:01:02.278772] PINT_process_request [D 20:01:02.278796] tiling 7 copies [D 20:01:02.278820] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.278846] lvl 0 el 0 blk 0 by 197132288 [D 20:01:02.278871] to 197132288 ta 167772160 fi 201326592 [D 20:01:02.278896] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.278922] basic type or contiguous data [D 20:01:02.278945] Request disp [D 20:01:02.278969] contig_offset = 197132288 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.278995] calling distribute [D 20:01:02.279019] PINT_distribute [D 20:01:02.279043] of 197132288 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 197132288 exfl 1 [D 20:01:02.279069] begin iteration loff: 197132288 [D 20:01:02.279094] segment exceeds byte limit [D 20:01:02.279118] file being extended [D 20:01:02.279142] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.279167] **********CALL***PROCESS********* [D 20:01:02.279191] segment of 29360128 sz 2097152 [D 20:01:02.279215] PINT_process_request [D 20:01:02.279239] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.279266] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.279291] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.279314] basic type or contiguous data [D 20:01:02.279338] Request disp [D 20:01:02.279362] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.279387] process a segment [D 20:01:02.279411] of 29360128 sz 2097152 [D 20:01:02.279435] add a segment 0 [D 20:01:02.279458] retval = 2097152 [D 20:01:02.279482] exiting distribute returned less than expected [D 20:01:02.279507] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.279533] *****RETURN***FROM***PROCESS***** [D 20:01:02.279557] end iteration [D 20:01:02.279581] done with segments or bytes [D 20:01:02.279604] of 199229440 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.279630] next loff: 199229440 next poff: 199229440 [D 20:01:02.279655] return value 2097152 [D 20:01:02.279679] finished [D 20:01:02.279703] retval = 2097152 [D 20:01:02.279726] exiting distribute returned less than expected [D 20:01:02.279750] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 199229440 fo 201326592 eof 0 [D 20:01:02.279777] ========================================================= [D 20:01:02.279801] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:01:02.279829] element 0: offset: 0xba067000, size: 2097152 [D 20:01:02.279857] post_sr: da13:3346 bh 67 len 32 wr_id 15e731c8. [D 20:01:02.279886] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:01:02.279911] ========================================================= [D 20:01:02.279936] PINT_process_request [D 20:01:02.279960] tiling 8 copies [D 20:01:02.279984] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.280011] lvl 0 el 0 blk 0 by 230686720 [D 20:01:02.280035] to 230686720 ta 201326592 fi 234881024 [D 20:01:02.280060] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.280085] basic type or contiguous data [D 20:01:02.280109] Request disp [D 20:01:02.280133] contig_offset = 230686720 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.280158] calling distribute [D 20:01:02.280182] PINT_distribute [D 20:01:02.280209] of 230686720 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 230686720 exfl 1 [D 20:01:02.280240] begin iteration loff: 230686720 [D 20:01:02.280268] segment exceeds byte limit [D 20:01:02.280296] file being extended [D 20:01:02.280321] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.280345] **********CALL***PROCESS********* [D 20:01:02.280369] segment of 29360128 sz 2097152 [D 20:01:02.280394] PINT_process_request [D 20:01:02.280418] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.280444] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.280468] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.280493] basic type or contiguous data [D 20:01:02.280517] Request disp [D 20:01:02.280541] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.280566] process a segment [D 20:01:02.280589] of 29360128 sz 2097152 [D 20:01:02.280613] add a segment 0 [D 20:01:02.280637] retval = 2097152 [D 20:01:02.280661] exiting distribute returned less than expected [D 20:01:02.280685] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.280711] *****RETURN***FROM***PROCESS***** [D 20:01:02.280735] end iteration [D 20:01:02.280758] done with segments or bytes [D 20:01:02.280782] of 232783872 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.280808] next loff: 232783872 next poff: 232783872 [D 20:01:02.280833] return value 2097152 [D 20:01:02.280858] finished [D 20:01:02.280880] retval = 2097152 [D 20:01:02.280905] exiting distribute returned less than expected [D 20:01:02.280929] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 232783872 fo 234881024 eof 0 [D 20:01:02.280955] ========================================================= [D 20:01:02.280979] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:01:02.281003] element 0: offset: 0xb8065000, size: 2097152 [D 20:01:02.281031] post_sr: da13:3346 bh 68 len 32 wr_id 15e731e0. [D 20:01:02.281060] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:01:02.281085] ========================================================= [D 20:01:02.281110] PINT_process_request [D 20:01:02.281134] tiling 9 copies [D 20:01:02.281158] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.281185] lvl 0 el 0 blk 0 by 264241152 [D 20:01:02.281209] to 264241152 ta 234881024 fi 268435456 [D 20:01:02.281234] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.281259] basic type or contiguous data [D 20:01:02.281283] Request disp [D 20:01:02.281307] contig_offset = 264241152 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.281332] calling distribute [D 20:01:02.281355] PINT_distribute [D 20:01:02.281379] of 264241152 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 264241152 exfl 1 [D 20:01:02.281406] begin iteration loff: 264241152 [D 20:01:02.281431] segment exceeds byte limit [D 20:01:02.281455] file being extended [D 20:01:02.281478] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.281502] **********CALL***PROCESS********* [D 20:01:02.281526] segment of 29360128 sz 2097152 [D 20:01:02.281553] PINT_process_request [D 20:01:02.281578] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.281603] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.281629] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.281653] basic type or contiguous data [D 20:01:02.281678] Request disp [D 20:01:02.281701] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.281726] process a segment [D 20:01:02.281750] of 29360128 sz 2097152 [D 20:01:02.281774] add a segment 0 [D 20:01:02.281798] retval = 2097152 [D 20:01:02.281822] exiting distribute returned less than expected [D 20:01:02.281846] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.281872] *****RETURN***FROM***PROCESS***** [D 20:01:02.281896] end iteration [D 20:01:02.281920] done with segments or bytes [D 20:01:02.281944] of 266338304 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.281970] next loff: 266338304 next poff: 266338304 [D 20:01:02.281994] return value 2097152 [D 20:01:02.282018] finished [D 20:01:02.282042] retval = 2097152 [D 20:01:02.282066] exiting distribute returned less than expected [D 20:01:02.282090] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 266338304 fo 268435456 eof 0 [D 20:01:02.282115] ========================================================= [D 20:01:02.282140] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:01:02.282165] element 0: offset: 0xb6063000, size: 2097152 [D 20:01:02.282196] post_sr: da13:3346 bh 69 len 32 wr_id 15e731f8. [D 20:01:02.282225] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:01:02.282251] ========================================================= [D 20:01:02.282275] PINT_process_request [D 20:01:02.282299] tiling 10 copies [D 20:01:02.282323] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.282350] lvl 0 el 0 blk 0 by 297795584 [D 20:01:02.282374] to 297795584 ta 268435456 fi 301989888 [D 20:01:02.282400] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.282423] basic type or contiguous data [D 20:01:02.282447] Request disp [D 20:01:02.282472] contig_offset = 297795584 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.282497] calling distribute [D 20:01:02.282521] PINT_distribute [D 20:01:02.282545] of 297795584 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 297795584 exfl 1 [D 20:01:02.282570] begin iteration loff: 297795584 [D 20:01:02.282595] segment exceeds byte limit [D 20:01:02.282620] file being extended [D 20:01:02.282643] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.282668] **********CALL***PROCESS********* [D 20:01:02.282691] segment of 29360128 sz 2097152 [D 20:01:02.282715] PINT_process_request [D 20:01:02.282739] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.282766] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.282790] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.282814] basic type or contiguous data [D 20:01:02.282838] Request disp [D 20:01:02.282862] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.282887] process a segment [D 20:01:02.282911] of 29360128 sz 2097152 [D 20:01:02.282935] add a segment 0 [D 20:01:02.282959] retval = 2097152 [D 20:01:02.282983] exiting distribute returned less than expected [D 20:01:02.283007] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.283033] *****RETURN***FROM***PROCESS***** [D 20:01:02.283058] end iteration [D 20:01:02.283082] done with segments or bytes [D 20:01:02.283105] of 299892736 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.283130] next loff: 299892736 next poff: 299892736 [D 20:01:02.283155] return value 2097152 [D 20:01:02.283179] finished [D 20:01:02.283203] retval = 2097152 [D 20:01:02.283226] exiting distribute returned less than expected [D 20:01:02.283250] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 299892736 fo 301989888 eof 0 [D 20:01:02.283280] ========================================================= [D 20:01:02.283305] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:01:02.283329] element 0: offset: 0xb4061000, size: 2097152 [D 20:01:02.283357] post_sr: da13:3346 bh 70 len 32 wr_id 15e73210. [D 20:01:02.283387] openib_check_cq: completed send wr_id 15e731b0. [D 20:01:02.283413] openib_check_cq: completed send wr_id 15e731c8. [D 20:01:02.283438] openib_check_cq: completed send wr_id 15e731e0. [D 20:01:02.283464] openib_check_cq: completed send wr_id 15e731f8. [D 20:01:02.283489] openib_check_cq: completed send wr_id 15e73210. [D 20:01:02.283522] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:01:02.283548] ========================================================= [D 20:01:02.283572] PINT_process_request [D 20:01:02.283596] tiling 11 copies [D 20:01:02.283620] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.283647] lvl 0 el 0 blk 0 by 331350016 [D 20:01:02.283672] to 331350016 ta 301989888 fi 335544320 [D 20:01:02.283697] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.283720] basic type or contiguous data [D 20:01:02.283744] Request disp [D 20:01:02.283768] contig_offset = 331350016 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.283794] calling distribute [D 20:01:02.283818] PINT_distribute [D 20:01:02.283842] of 331350016 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 331350016 exfl 1 [D 20:01:02.283867] begin iteration loff: 331350016 [D 20:01:02.283892] segment exceeds byte limit [D 20:01:02.283916] file being extended [D 20:01:02.283940] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.283965] **********CALL***PROCESS********* [D 20:01:02.283988] segment of 29360128 sz 2097152 [D 20:01:02.284012] PINT_process_request [D 20:01:02.284036] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.284063] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.284088] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.284111] basic type or contiguous data [D 20:01:02.284136] Request disp [D 20:01:02.284160] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.284185] process a segment [D 20:01:02.284213] of 29360128 sz 2097152 [D 20:01:02.284240] add a segment 0 [D 20:01:02.284268] retval = 2097152 [D 20:01:02.284296] exiting distribute returned less than expected [D 20:01:02.284320] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.284347] *****RETURN***FROM***PROCESS***** [D 20:01:02.284371] end iteration [D 20:01:02.284395] done with segments or bytes [D 20:01:02.284419] of 333447168 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.284444] next loff: 333447168 next poff: 333447168 [D 20:01:02.284469] return value 2097152 [D 20:01:02.284493] finished [D 20:01:02.284517] retval = 2097152 [D 20:01:02.284541] exiting distribute returned less than expected [D 20:01:02.284564] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 333447168 fo 335544320 eof 0 [D 20:01:02.284591] ========================================================= [D 20:01:02.284615] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:01:02.284640] element 0: offset: 0xb205f000, size: 2097152 [D 20:01:02.284667] post_sr: da13:3346 bh 71 len 32 wr_id 15e73228. [D 20:01:02.284696] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:01:02.284722] ========================================================= [D 20:01:02.284746] PINT_process_request [D 20:01:02.284770] tiling 12 copies [D 20:01:02.284794] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.284820] lvl 0 el 0 blk 0 by 364904448 [D 20:01:02.284845] to 364904448 ta 335544320 fi 369098752 [D 20:01:02.284869] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.284893] basic type or contiguous data [D 20:01:02.284918] Request disp [D 20:01:02.284942] contig_offset = 364904448 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.284970] calling distribute [D 20:01:02.284994] PINT_distribute [D 20:01:02.285017] of 364904448 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 364904448 exfl 1 [D 20:01:02.285044] begin iteration loff: 364904448 [D 20:01:02.285069] segment exceeds byte limit [D 20:01:02.285093] file being extended [D 20:01:02.285117] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.285141] **********CALL***PROCESS********* [D 20:01:02.285165] segment of 29360128 sz 2097152 [D 20:01:02.285190] PINT_process_request [D 20:01:02.285214] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.285240] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.285264] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.285288] basic type or contiguous data [D 20:01:02.285312] Request disp [D 20:01:02.285337] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.285362] process a segment [D 20:01:02.285386] of 29360128 sz 2097152 [D 20:01:02.285409] add a segment 0 [D 20:01:02.285433] retval = 2097152 [D 20:01:02.285457] exiting distribute returned less than expected [D 20:01:02.285482] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.285508] *****RETURN***FROM***PROCESS***** [D 20:01:02.285531] end iteration [D 20:01:02.285555] done with segments or bytes [D 20:01:02.285579] of 367001600 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.285605] next loff: 367001600 next poff: 367001600 [D 20:01:02.285630] return value 2097152 [D 20:01:02.285654] finished [D 20:01:02.285677] retval = 2097152 [D 20:01:02.285701] exiting distribute returned less than expected [D 20:01:02.285725] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 367001600 fo 369098752 eof 0 [D 20:01:02.285751] ========================================================= [D 20:01:02.285776] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:01:02.285800] element 0: offset: 0xb005d000, size: 2097152 [D 20:01:02.285828] post_sr: da13:3346 bh 72 len 32 wr_id 15e73240. [D 20:01:02.285857] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:01:02.285882] ========================================================= [D 20:01:02.285906] PINT_process_request [D 20:01:02.285930] tiling 13 copies [D 20:01:02.285955] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.285981] lvl 0 el 0 blk 0 by 398458880 [D 20:01:02.286005] to 398458880 ta 369098752 fi 402653184 [D 20:01:02.286030] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.286055] basic type or contiguous data [D 20:01:02.286079] Request disp [D 20:01:02.286103] contig_offset = 398458880 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.286128] calling distribute [D 20:01:02.286152] PINT_distribute [D 20:01:02.286176] of 398458880 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 398458880 exfl 1 [D 20:01:02.286206] begin iteration loff: 398458880 [D 20:01:02.286231] segment exceeds byte limit [D 20:01:02.286255] file being extended [D 20:01:02.286279] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.286304] **********CALL***PROCESS********* [D 20:01:02.286327] segment of 29360128 sz 2097152 [D 20:01:02.286352] PINT_process_request [D 20:01:02.286375] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.286402] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.286427] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.286452] basic type or contiguous data [D 20:01:02.286476] Request disp [D 20:01:02.286499] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.286524] process a segment [D 20:01:02.286548] of 29360128 sz 2097152 [D 20:01:02.286573] add a segment 0 [D 20:01:02.286597] retval = 2097152 [D 20:01:02.286621] exiting distribute returned less than expected [D 20:01:02.286644] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.286671] *****RETURN***FROM***PROCESS***** [D 20:01:02.286698] end iteration [D 20:01:02.286722] done with segments or bytes [D 20:01:02.286746] of 400556032 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.286771] next loff: 400556032 next poff: 400556032 [D 20:01:02.286796] return value 2097152 [D 20:01:02.286821] finished [D 20:01:02.286845] retval = 2097152 [D 20:01:02.286869] exiting distribute returned less than expected [D 20:01:02.286892] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 400556032 fo 402653184 eof 0 [D 20:01:02.286918] ========================================================= [D 20:01:02.286943] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:01:02.286968] element 0: offset: 0xae05b000, size: 2097152 [D 20:01:02.286995] post_sr: da13:3346 bh 73 len 32 wr_id 15e73258. [D 20:01:02.287024] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:02.287049] ========================================================= [D 20:01:02.287074] PINT_process_request [D 20:01:02.287098] tiling 16 copies [D 20:01:02.287122] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.287148] lvl 0 el 0 blk 0 by 490733568 [D 20:01:02.287173] to 490733568 ta 469762048 fi 503316480 [D 20:01:02.287198] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:02.287222] basic type or contiguous data [D 20:01:02.287247] Request disp [D 20:01:02.287271] contig_offset = 490733568 contig_size = 46137344 lvl_flag = 1 [D 20:01:02.287296] calling distribute [D 20:01:02.287320] PINT_distribute [D 20:01:02.287343] of 490733568 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 490733568 exfl 1 [D 20:01:02.287370] begin iteration loff: 490733568 [D 20:01:02.287395] segment exceeds byte limit [D 20:01:02.287419] file being extended [D 20:01:02.287443] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:02.287467] **********CALL***PROCESS********* [D 20:01:02.287491] segment of 20971520 sz 2097152 [D 20:01:02.287516] PINT_process_request [D 20:01:02.287540] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.287567] lvl 0 el 0 blk 0 by 20971520 [D 20:01:02.287591] to 20971520 ta 20971520 fi 23068672 [D 20:01:02.287618] basic type or contiguous data [D 20:01:02.287642] Request disp [D 20:01:02.287666] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:02.287692] process a segment [D 20:01:02.287715] of 20971520 sz 2097152 [D 20:01:02.287739] add a segment 0 [D 20:01:02.287763] retval = 2097152 [D 20:01:02.287788] exiting distribute returned less than expected [D 20:01:02.287812] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:02.287837] *****RETURN***FROM***PROCESS***** [D 20:01:02.287862] end iteration [D 20:01:02.287886] done with segments or bytes [D 20:01:02.287910] of 492830720 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.287935] next loff: 492830720 next poff: 492830720 [D 20:01:02.287960] return value 2097152 [D 20:01:02.287984] finished [D 20:01:02.288008] retval = 2097152 [D 20:01:02.288031] exiting distribute returned less than expected [D 20:01:02.288056] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 492830720 fo 503316480 eof 0 [D 20:01:02.288082] ========================================================= [D 20:01:02.288105] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:02.288131] element 0: offset: 0xa7855000, size: 2097152 [D 20:01:02.288158] post_sr: da13:3346 bh 74 len 32 wr_id 15e73270. [D 20:01:02.288187] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:01:02.288218] ========================================================= [D 20:01:02.288247] PINT_process_request [D 20:01:02.288274] tiling 14 copies [D 20:01:02.288302] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.288328] lvl 0 el 0 blk 0 by 432013312 [D 20:01:02.288353] to 432013312 ta 402653184 fi 436207616 [D 20:01:02.288378] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.288405] basic type or contiguous data [D 20:01:02.288430] Request disp [D 20:01:02.288454] contig_offset = 432013312 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.288478] calling distribute [D 20:01:02.288502] PINT_distribute [D 20:01:02.288526] of 432013312 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 432013312 exfl 1 [D 20:01:02.288552] begin iteration loff: 432013312 [D 20:01:02.288578] segment exceeds byte limit [D 20:01:02.288601] file being extended [D 20:01:02.288625] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.288649] **********CALL***PROCESS********* [D 20:01:02.288673] segment of 29360128 sz 2097152 [D 20:01:02.288698] PINT_process_request [D 20:01:02.288722] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.288747] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.288772] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.288797] basic type or contiguous data [D 20:01:02.288821] Request disp [D 20:01:02.288845] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.288869] process a segment [D 20:01:02.288893] of 29360128 sz 2097152 [D 20:01:02.288917] add a segment 0 [D 20:01:02.288941] retval = 2097152 [D 20:01:02.288965] exiting distribute returned less than expected [D 20:01:02.288990] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.289015] *****RETURN***FROM***PROCESS***** [D 20:01:02.289040] end iteration [D 20:01:02.289064] done with segments or bytes [D 20:01:02.289088] of 434110464 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.289113] next loff: 434110464 next poff: 434110464 [D 20:01:02.289137] return value 2097152 [D 20:01:02.289162] finished [D 20:01:02.289185] retval = 2097152 [D 20:01:02.289210] exiting distribute returned less than expected [D 20:01:02.289234] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 434110464 fo 436207616 eof 0 [D 20:01:02.289259] ========================================================= [D 20:01:02.289284] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:01:02.289310] element 0: offset: 0xac059000, size: 2097152 [D 20:01:02.289337] post_sr: da13:3346 bh 75 len 32 wr_id 15e73288. [D 20:01:02.289366] PINT_client_state_machine_test id 22 [D 20:01:02.289395] openib_check_cq: completed send wr_id 15e73228. [D 20:01:02.289422] openib_check_cq: completed send wr_id 15e73240. [D 20:01:02.289446] openib_check_cq: completed send wr_id 15e73258. [D 20:01:02.289471] openib_check_cq: completed send wr_id 15e73270. [D 20:01:02.289497] openib_check_cq: completed send wr_id 15e73288. [D 20:01:02.289529] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:01:02.289555] ========================================================= [D 20:01:02.289580] PINT_process_request [D 20:01:02.289604] tiling 15 copies [D 20:01:02.289628] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.289654] lvl 0 el 0 blk 0 by 465567744 [D 20:01:02.289679] to 465567744 ta 436207616 fi 469762048 [D 20:01:02.289704] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.289729] basic type or contiguous data [D 20:01:02.289752] Request disp [D 20:01:02.289777] contig_offset = 465567744 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.289802] calling distribute [D 20:01:02.289826] PINT_distribute [D 20:01:02.289850] of 465567744 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 465567744 exfl 1 [D 20:01:02.289876] begin iteration loff: 465567744 [D 20:01:02.289901] segment exceeds byte limit [D 20:01:02.289925] file being extended [D 20:01:02.289949] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.289974] **********CALL***PROCESS********* [D 20:01:02.289998] segment of 29360128 sz 2097152 [D 20:01:02.290021] PINT_process_request [D 20:01:02.290045] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.290071] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.290096] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.290123] basic type or contiguous data [D 20:01:02.290148] Request disp [D 20:01:02.290172] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.290201] process a segment [D 20:01:02.290225] of 29360128 sz 2097152 [D 20:01:02.290249] add a segment 0 [D 20:01:02.290273] retval = 2097152 [D 20:01:02.290297] exiting distribute returned less than expected [D 20:01:02.290321] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.290348] *****RETURN***FROM***PROCESS***** [D 20:01:02.290371] end iteration [D 20:01:02.290395] done with segments or bytes [D 20:01:02.290419] of 467664896 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.290445] next loff: 467664896 next poff: 467664896 [D 20:01:02.290469] return value 2097152 [D 20:01:02.290493] finished [D 20:01:02.290516] retval = 2097152 [D 20:01:02.290541] exiting distribute returned less than expected [D 20:01:02.290565] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 467664896 fo 469762048 eof 0 [D 20:01:02.290591] ========================================================= [D 20:01:02.290616] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:01:02.290641] element 0: offset: 0xaa057000, size: 2097152 [D 20:01:02.290668] post_sr: da13:3346 bh 76 len 32 wr_id 15e732a0. [D 20:01:02.290697] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:01:02.290723] ========================================================= [D 20:01:02.290747] PINT_process_request [D 20:01:02.290772] tiling 2 copies [D 20:01:02.290796] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.290822] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.290846] to 31457280 ta 0 fi 33554432 [D 20:01:02.290872] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.290897] basic type or contiguous data [D 20:01:02.290921] Request disp [D 20:01:02.290945] contig_offset = 31457280 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.290969] calling distribute [D 20:01:02.290993] PINT_distribute [D 20:01:02.291017] of 31457280 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 31457280 exfl 1 [D 20:01:02.291043] begin iteration loff: 31457280 [D 20:01:02.291069] file being extended [D 20:01:02.291091] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.291116] **********CALL***PROCESS********* [D 20:01:02.291140] segment of 31457280 sz 2097152 [D 20:01:02.291165] PINT_process_request [D 20:01:02.291189] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.291215] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.291240] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.291265] basic type or contiguous data [D 20:01:02.291289] Request disp [D 20:01:02.291313] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.291337] process a segment [D 20:01:02.291361] of 31457280 sz 2097152 [D 20:01:02.291386] add a segment 0 [D 20:01:02.291410] retval = 2097152 [D 20:01:02.291434] return from level -1 [D 20:01:02.291458] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.291483] *****RETURN***FROM***PROCESS***** [D 20:01:02.291508] end iteration [D 20:01:02.291531] done with segments or bytes [D 20:01:02.291556] of 33554432 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.291581] next loff: 33554432 next poff: 33554432 [D 20:01:02.291606] (return value) 2097152 [D 20:01:02.291630] finished [D 20:01:02.291654] retval = 2097152 [D 20:01:02.291678] exiting distribute returned less than expected [D 20:01:02.291702] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 33554432 fo 33554432 eof 0 [D 20:01:02.291728] ========================================================= [D 20:01:02.291752] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 8 [D 20:01:02.291778] element 0: offset: 0xc4271000, size: 2097152 [D 20:01:02.291805] post_sr: da13:3346 bh 77 len 32 wr_id 15e732b8. [D 20:01:02.291833] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:01:02.291861] ========================================================= [D 20:01:02.291886] PINT_process_request [D 20:01:02.291910] tiling 3 copies [D 20:01:02.291934] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.291960] lvl 0 el 0 blk 0 by 65011712 [D 20:01:02.291984] to 65011712 ta 33554432 fi 67108864 [D 20:01:02.292009] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.292034] basic type or contiguous data [D 20:01:02.292057] Request disp [D 20:01:02.292081] contig_offset = 65011712 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.292106] calling distribute [D 20:01:02.292131] PINT_distribute [D 20:01:02.292154] of 65011712 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 65011712 exfl 1 [D 20:01:02.292181] begin iteration loff: 65011712 [D 20:01:02.292209] file being extended [D 20:01:02.292237] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.292265] **********CALL***PROCESS********* [D 20:01:02.292292] segment of 31457280 sz 2097152 [D 20:01:02.292318] PINT_process_request [D 20:01:02.292343] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.292368] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.292393] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.292418] basic type or contiguous data [D 20:01:02.292442] Request disp [D 20:01:02.292466] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.292490] process a segment [D 20:01:02.292514] of 31457280 sz 2097152 [D 20:01:02.292538] add a segment 0 [D 20:01:02.292562] retval = 2097152 [D 20:01:02.292586] return from level -1 [D 20:01:02.292611] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.292636] *****RETURN***FROM***PROCESS***** [D 20:01:02.292661] end iteration [D 20:01:02.292685] done with segments or bytes [D 20:01:02.292708] of 67108864 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.292734] next loff: 67108864 next poff: 67108864 [D 20:01:02.292758] (return value) 2097152 [D 20:01:02.292782] finished [D 20:01:02.292806] retval = 2097152 [D 20:01:02.292830] exiting distribute returned less than expected [D 20:01:02.292854] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 67108864 fo 67108864 eof 0 [D 20:01:02.292880] ========================================================= [D 20:01:02.292904] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 9 [D 20:01:02.292929] element 0: offset: 0xc226f000, size: 2097152 [D 20:01:02.292957] post_sr: da13:3346 bh 78 len 32 wr_id 15e732d0. [D 20:01:02.292985] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:01:02.293011] ========================================================= [D 20:01:02.293035] PINT_process_request [D 20:01:02.293060] tiling 4 copies [D 20:01:02.293084] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.293110] lvl 0 el 0 blk 0 by 98566144 [D 20:01:02.293135] to 98566144 ta 67108864 fi 100663296 [D 20:01:02.293160] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.293185] basic type or contiguous data [D 20:01:02.293209] Request disp [D 20:01:02.293232] contig_offset = 98566144 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.293257] calling distribute [D 20:01:02.293282] PINT_distribute [D 20:01:02.293306] of 98566144 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 98566144 exfl 1 [D 20:01:02.293332] begin iteration loff: 98566144 [D 20:01:02.293356] file being extended [D 20:01:02.293380] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.293405] **********CALL***PROCESS********* [D 20:01:02.293429] segment of 31457280 sz 2097152 [D 20:01:02.293453] PINT_process_request [D 20:01:02.293477] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.293503] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.293528] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.293553] basic type or contiguous data [D 20:01:02.293577] Request disp [D 20:01:02.293604] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.293630] process a segment [D 20:01:02.293654] of 31457280 sz 2097152 [D 20:01:02.293679] add a segment 0 [D 20:01:02.293703] retval = 2097152 [D 20:01:02.293726] return from level -1 [D 20:01:02.293751] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.293777] *****RETURN***FROM***PROCESS***** [D 20:01:02.293801] end iteration [D 20:01:02.293825] done with segments or bytes [D 20:01:02.293848] of 100663296 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.293874] next loff: 100663296 next poff: 100663296 [D 20:01:02.293899] (return value) 2097152 [D 20:01:02.293923] finished [D 20:01:02.293947] retval = 2097152 [D 20:01:02.293971] exiting distribute returned less than expected [D 20:01:02.293994] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 100663296 fo 100663296 eof 0 [D 20:01:02.294021] ========================================================= [D 20:01:02.294045] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 10 [D 20:01:02.294071] element 0: offset: 0xc026d000, size: 2097152 [D 20:01:02.294097] post_sr: da13:3346 bh 79 len 32 wr_id 15e732e8. [D 20:01:02.294126] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:01:02.294152] ========================================================= [D 20:01:02.294177] PINT_process_request [D 20:01:02.294205] tiling 5 copies [D 20:01:02.294229] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.294256] lvl 0 el 0 blk 0 by 132120576 [D 20:01:02.294281] to 132120576 ta 100663296 fi 134217728 [D 20:01:02.294305] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.294330] basic type or contiguous data [D 20:01:02.294354] Request disp [D 20:01:02.294378] contig_offset = 132120576 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.294404] calling distribute [D 20:01:02.294427] PINT_distribute [D 20:01:02.294451] of 132120576 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 132120576 exfl 1 [D 20:01:02.294478] begin iteration loff: 132120576 [D 20:01:02.294503] file being extended [D 20:01:02.294527] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.294551] **********CALL***PROCESS********* [D 20:01:02.294575] segment of 31457280 sz 2097152 [D 20:01:02.294600] PINT_process_request [D 20:01:02.294624] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.294650] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.294674] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.294699] basic type or contiguous data [D 20:01:02.294723] Request disp [D 20:01:02.294747] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.294772] process a segment [D 20:01:02.294795] of 31457280 sz 2097152 [D 20:01:02.294819] add a segment 0 [D 20:01:02.294843] retval = 2097152 [D 20:01:02.294867] return from level -1 [D 20:01:02.294891] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.294918] *****RETURN***FROM***PROCESS***** [D 20:01:02.294941] end iteration [D 20:01:02.294965] done with segments or bytes [D 20:01:02.294989] of 134217728 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.295015] next loff: 134217728 next poff: 134217728 [D 20:01:02.295040] (return value) 2097152 [D 20:01:02.295063] finished [D 20:01:02.295087] retval = 2097152 [D 20:01:02.295111] exiting distribute returned less than expected [D 20:01:02.295135] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 134217728 fo 134217728 eof 0 [D 20:01:02.295162] ========================================================= [D 20:01:02.295186] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 11 [D 20:01:02.295211] element 0: offset: 0xbe26b000, size: 2097152 [D 20:01:02.295239] post_sr: da13:3346 bh 80 len 32 wr_id 15e73300. [D 20:01:02.295269] openib_check_cq: completed send wr_id 15e732a0. [D 20:01:02.295295] openib_check_cq: completed send wr_id 15e732b8. [D 20:01:02.295323] openib_check_cq: completed send wr_id 15e732d0. [D 20:01:02.295349] openib_check_cq: completed send wr_id 15e732e8. [D 20:01:02.295374] openib_check_cq: completed send wr_id 15e73300. [D 20:01:02.299396] PINT_client_state_machine_test id 22 [D 20:01:02.309428] PINT_client_state_machine_test id 22 [D 20:01:02.319460] PINT_client_state_machine_test id 22 [D 20:01:02.329489] PINT_client_state_machine_test id 22 [D 20:01:02.339520] PINT_client_state_machine_test id 22 [D 20:01:02.346350] openib_check_cq: completed recv wr_id 15e73bd8. [D 20:01:02.346379] openib_post_rr: da13:3346 bh 74 wr_id 15e73bd8. [D 20:01:02.346405] openib_check_cq: completed recv wr_id 15e73bf0. [D 20:01:02.346432] openib_post_rr: da13:3346 bh 75 wr_id 15e73bf0. [D 20:01:02.346458] openib_check_cq: completed recv wr_id 15e73c08. [D 20:01:02.346485] openib_post_sr_rdmaw: ibv_post_send wr_id 15f1d800 to da13:3346 remote addr 2aaab7c36010 rkey c8007e00 nic_wr_credit 1. [D 20:01:02.346511] openib_post_rr: da13:3346 bh 76 wr_id 15e73c08. [D 20:01:02.346537] openib_check_cq: completed recv wr_id 15e73c20. [D 20:01:02.346564] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaab7e37010 rkey c8007f00 nic_wr_credit 2. [D 20:01:02.346590] openib_post_rr: da13:3346 bh 77 wr_id 15e73c20. [D 20:01:02.346616] openib_check_cq: completed recv wr_id 15e73c38. [D 20:01:02.346642] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaab8038010 rkey 48008000 nic_wr_credit 3. [D 20:01:02.346669] openib_post_rr: da13:3346 bh 78 wr_id 15e73c38. [D 20:01:02.346695] openib_check_cq: completed recv wr_id 15e73c50. [D 20:01:02.346721] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaab8239010 rkey 48008100 nic_wr_credit 4. [D 20:01:02.346748] openib_post_rr: da13:3346 bh 79 wr_id 15e73c50. [D 20:01:02.346774] openib_check_cq: completed recv wr_id 15e73c68. [D 20:01:02.346800] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaab843a010 rkey 48008200 nic_wr_credit 5. [D 20:01:02.346826] openib_post_rr: da13:3346 bh 80 wr_id 15e73c68. [D 20:01:02.346853] openib_check_cq: completed recv wr_id 15e73c80. [D 20:01:02.346879] openib_post_sr_rdmaw: ibv_post_send wr_id 15effae8 to da13:3346 remote addr 2aaab863b010 rkey 48008300 nic_wr_credit 6. [D 20:01:02.346906] openib_post_rr: da13:3346 bh 81 wr_id 15e73c80. [D 20:01:02.346932] openib_check_cq: completed recv wr_id 15e73c98. [D 20:01:02.346959] openib_post_rr: da13:3346 bh 82 wr_id 15e73c98. [D 20:01:02.346985] openib_check_cq: completed recv wr_id 15e73cb0. [D 20:01:02.347012] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaab883c010 rkey 48008400 nic_wr_credit 7. [D 20:01:02.347038] openib_post_rr: da13:3346 bh 83 wr_id 15e73cb0. [D 20:01:02.347064] openib_check_cq: completed recv wr_id 15e73cc8. [D 20:01:02.347090] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02580 to da13:3346 remote addr 2aaab8a3d010 rkey 48008500 nic_wr_credit 8. [D 20:01:02.347117] openib_post_rr: da13:3346 bh 84 wr_id 15e73cc8. [D 20:01:02.347149] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.347176] ========================================================= [D 20:01:02.347200] PINT_process_request [D 20:01:02.347225] tiling 15 copies [D 20:01:02.347249] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.347274] lvl 0 el 0 blk 0 by 444596224 [D 20:01:02.347299] to 444596224 ta 436207616 fi 469762048 [D 20:01:02.347324] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:02.347349] basic type or contiguous data [D 20:01:02.347373] Request disp [D 20:01:02.347396] contig_offset = 444596224 contig_size = 58720256 lvl_flag = 1 [D 20:01:02.347422] calling distribute [D 20:01:02.347446] PINT_distribute [D 20:01:02.347470] of 444596224 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.347496] begin iteration loff: 444596224 [D 20:01:02.347521] segment exceeds byte limit [D 20:01:02.347545] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:02.347572] **********CALL***PROCESS********* [D 20:01:02.347597] segment of 8388608 sz 2097152 [D 20:01:02.347623] PINT_process_request [D 20:01:02.347647] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.347673] lvl 0 el 0 blk 0 by 8388608 [D 20:01:02.347698] to 8388608 ta 8388608 fi 10485760 [D 20:01:02.347723] basic type or contiguous data [D 20:01:02.347746] Request disp [D 20:01:02.347770] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:02.347795] process a segment [D 20:01:02.347819] of 8388608 sz 2097152 [D 20:01:02.347843] add a segment 0 [D 20:01:02.347867] retval = 2097152 [D 20:01:02.347890] exiting distribute returned less than expected [D 20:01:02.347915] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:02.347941] *****RETURN***FROM***PROCESS***** [D 20:01:02.347965] end iteration [D 20:01:02.347989] done with segments or bytes [D 20:01:02.348012] of 446693376 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.348038] next loff: 446693376 next poff: 446693376 [D 20:01:02.348063] return value 2097152 [D 20:01:02.348087] finished [D 20:01:02.348111] retval = 2097152 [D 20:01:02.348134] exiting distribute returned less than expected [D 20:01:02.348158] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 446693376 fo 469762048 eof 0 [D 20:01:02.348185] ========================================================= [D 20:01:02.348214] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.348242] element 0: offset: 0x88c37000, size: 2097152 [D 20:01:02.348273] openib_check_cq: completed recv wr_id 15e73ce0. [D 20:01:02.348303] openib_post_sr_rdmaw: ibv_post_send wr_id 15f05018 to da13:3346 remote addr 2aaab8e3f010 rkey 48008700 nic_wr_credit 9. [D 20:01:02.348330] openib_post_rr: da13:3346 bh 85 wr_id 15e73ce0. [D 20:01:02.348357] openib_check_cq: completed recv wr_id 15e73cf8. [D 20:01:02.348382] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7c30 to da13:3346 remote addr 2aaab8c3e010 rkey 48008600 nic_wr_credit 10. [D 20:01:02.348409] openib_post_rr: da13:3346 bh 86 wr_id 15e73cf8. [D 20:01:02.348440] post_sr: da13:3346 bh 81 len 48 wr_id 15e73318. [D 20:01:02.348466] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.348492] ========================================================= [D 20:01:02.348517] PINT_process_request [D 20:01:02.348541] tiling 2 copies [D 20:01:02.348566] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.348591] lvl 0 el 0 blk 0 by 6291456 [D 20:01:02.348616] to 6291456 ta 0 fi 33554432 [D 20:01:02.348641] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:02.348666] basic type or contiguous data [D 20:01:02.348690] Request disp [D 20:01:02.348713] contig_offset = 6291456 contig_size = 60817408 lvl_flag = 1 [D 20:01:02.348739] calling distribute [D 20:01:02.348763] PINT_distribute [D 20:01:02.348787] of 6291456 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.348813] begin iteration loff: 6291456 [D 20:01:02.348837] segment exceeds byte limit [D 20:01:02.348861] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:02.348887] **********CALL***PROCESS********* [D 20:01:02.348911] segment of 6291456 sz 2097152 [D 20:01:02.348936] PINT_process_request [D 20:01:02.348958] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.348985] lvl 0 el 0 blk 0 by 6291456 [D 20:01:02.349010] to 6291456 ta 6291456 fi 8388608 [D 20:01:02.349035] basic type or contiguous data [D 20:01:02.349059] Request disp [D 20:01:02.349082] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:02.349107] process a segment [D 20:01:02.349130] of 6291456 sz 2097152 [D 20:01:02.349155] add a segment 0 [D 20:01:02.349179] retval = 2097152 [D 20:01:02.349203] exiting distribute returned less than expected [D 20:01:02.349226] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:02.349256] *****RETURN***FROM***PROCESS***** [D 20:01:02.349280] end iteration [D 20:01:02.349304] done with segments or bytes [D 20:01:02.349328] of 8388608 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.349353] next loff: 8388608 next poff: 8388608 [D 20:01:02.349378] return value 2097152 [D 20:01:02.349402] finished [D 20:01:02.349426] retval = 2097152 [D 20:01:02.349450] exiting distribute returned less than expected [D 20:01:02.349473] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 8388608 fo 33554432 eof 0 [D 20:01:02.349499] ========================================================= [D 20:01:02.349524] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.349549] element 0: offset: 0xa2a51000, size: 2097152 [D 20:01:02.349575] openib_check_cq: completed recv wr_id 15e73d10. [D 20:01:02.349602] openib_post_sr_rdmaw: ibv_post_send wr_id 15efa630 to da13:3346 remote addr 2aaab9040010 rkey c8008800 nic_wr_credit 11. [D 20:01:02.349629] openib_post_rr: da13:3346 bh 87 wr_id 15e73d10. [D 20:01:02.349655] openib_check_cq: completed rdma wr_id 15f1d800, nic_wr_credit 10. [D 20:01:02.349681] post_sr: da13:3346 bh 82 len 16 wr_id 15e73330. [D 20:01:02.349708] openib_check_cq: completed recv wr_id 15e73d28. [D 20:01:02.349734] openib_post_sr_rdmaw: ibv_post_send wr_id 15ecea40 to da13:3346 remote addr 2aaab9241010 rkey c8008900 nic_wr_credit 11. [D 20:01:02.349761] openib_post_rr: da13:3346 bh 88 wr_id 15e73d28. [D 20:01:02.349787] openib_check_cq: completed recv wr_id 15e73d40. [D 20:01:02.349814] openib_post_sr_rdmaw: ibv_post_send wr_id 15eb7950 to da13:3346 remote addr 2aaab9442010 rkey c8008a00 nic_wr_credit 12. [D 20:01:02.349841] openib_post_rr: da13:3346 bh 89 wr_id 15e73d40. [D 20:01:02.349867] openib_check_cq: completed recv wr_id 15e73d58. [D 20:01:02.349894] openib_post_sr_rdmaw: ibv_post_send wr_id 15ea6170 to da13:3346 remote addr 2aaab9643010 rkey c8008b00 nic_wr_credit 13. [D 20:01:02.349921] openib_post_rr: da13:3346 bh 90 wr_id 15e73d58. [D 20:01:02.349950] post_sr: da13:3346 bh 83 len 48 wr_id 15e73348. [D 20:01:02.349978] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.350003] ========================================================= [D 20:01:02.350028] PINT_process_request [D 20:01:02.350051] tiling 3 copies [D 20:01:02.350075] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.350102] lvl 0 el 0 blk 0 by 37748736 [D 20:01:02.350127] to 37748736 ta 33554432 fi 67108864 [D 20:01:02.350152] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:02.350176] basic type or contiguous data [D 20:01:02.350205] Request disp [D 20:01:02.350229] contig_offset = 37748736 contig_size = 62914560 lvl_flag = 1 [D 20:01:02.350254] calling distribute [D 20:01:02.350277] PINT_distribute [D 20:01:02.350301] of 37748736 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.350328] begin iteration loff: 37748736 [D 20:01:02.350353] segment exceeds byte limit [D 20:01:02.350377] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:02.350401] **********CALL***PROCESS********* [D 20:01:02.350425] segment of 4194304 sz 2097152 [D 20:01:02.350450] PINT_process_request [D 20:01:02.350474] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.350500] lvl 0 el 0 blk 0 by 4194304 [D 20:01:02.350524] to 4194304 ta 4194304 fi 6291456 [D 20:01:02.350549] basic type or contiguous data [D 20:01:02.350573] Request disp [D 20:01:02.350597] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:02.350622] process a segment [D 20:01:02.350646] of 4194304 sz 2097152 [D 20:01:02.350669] add a segment 0 [D 20:01:02.350693] retval = 2097152 [D 20:01:02.350717] exiting distribute returned less than expected [D 20:01:02.350741] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:02.350768] *****RETURN***FROM***PROCESS***** [D 20:01:02.350791] end iteration [D 20:01:02.350817] done with segments or bytes [D 20:01:02.350842] of 39845888 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.350867] next loff: 39845888 next poff: 39845888 [D 20:01:02.350892] return value 2097152 [D 20:01:02.350916] finished [D 20:01:02.350940] retval = 2097152 [D 20:01:02.350964] exiting distribute returned less than expected [D 20:01:02.350988] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 39845888 fo 67108864 eof 0 [D 20:01:02.351014] ========================================================= [D 20:01:02.351038] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.351063] element 0: offset: 0xa084f000, size: 2097152 [D 20:01:02.351089] openib_check_cq: completed recv wr_id 15e73d70. [D 20:01:02.351117] openib_post_sr_rdmaw: ibv_post_send wr_id 15e8c1c8 to da13:3346 remote addr 2aaab9a45010 rkey c8008d00 nic_wr_credit 14. [D 20:01:02.351143] openib_post_rr: da13:3346 bh 91 wr_id 15e73d70. [D 20:01:02.351174] post_sr: da13:3346 bh 84 len 48 wr_id 15e73360. [D 20:01:02.351202] PINT_client_state_machine_test id 22 [D 20:01:02.351440] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 13. [D 20:01:02.351467] post_sr: da13:3346 bh 85 len 16 wr_id 15e73378. [D 20:01:02.353891] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 12. [D 20:01:02.353919] post_sr: da13:3346 bh 86 len 16 wr_id 15e73390. [D 20:01:02.356349] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 11. [D 20:01:02.356377] post_sr: da13:3346 bh 87 len 16 wr_id 15e733a8. [D 20:01:02.358803] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 10. [D 20:01:02.358831] post_sr: da13:3346 bh 88 len 16 wr_id 15e733c0. [D 20:01:02.361229] PINT_client_state_machine_test id 22 [D 20:01:02.361260] openib_check_cq: completed rdma wr_id 15effae8, nic_wr_credit 9. [D 20:01:02.361287] post_sr: da13:3346 bh 89 len 16 wr_id 15e733d8. [D 20:01:02.363713] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 8. [D 20:01:02.363740] post_sr: da13:3346 bh 90 len 16 wr_id 15e733f0. [D 20:01:02.366172] openib_check_cq: completed rdma wr_id 15f02580, nic_wr_credit 7. [D 20:01:02.366203] post_sr: da13:3346 bh 91 len 16 wr_id 15e73408. [D 20:01:02.368627] openib_check_cq: completed rdma wr_id 15f05018, nic_wr_credit 6. [D 20:01:02.368655] post_sr: da13:3346 bh 92 len 16 wr_id 15e73420. [D 20:01:02.371080] openib_check_cq: completed rdma wr_id 15ef7c30, nic_wr_credit 5. [D 20:01:02.371108] post_sr: da13:3346 bh 93 len 16 wr_id 15e73438. [D 20:01:02.371133] openib_check_cq: completed send wr_id 15e73318. [D 20:01:02.371264] PINT_client_state_machine_test id 22 [D 20:01:02.373538] openib_check_cq: completed rdma wr_id 15efa630, nic_wr_credit 4. [D 20:01:02.373565] post_sr: da13:3346 bh 94 len 16 wr_id 15e73450. [D 20:01:02.373592] openib_check_cq: completed send wr_id 15e73330. [D 20:01:02.373622] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:01:02.373648] ========================================================= [D 20:01:02.373673] PINT_process_request [D 20:01:02.373697] tiling 6 copies [D 20:01:02.373721] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.373747] lvl 0 el 0 blk 0 by 165675008 [D 20:01:02.373772] to 165675008 ta 134217728 fi 167772160 [D 20:01:02.373797] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.373822] basic type or contiguous data [D 20:01:02.373846] Request disp [D 20:01:02.373869] contig_offset = 165675008 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.373894] calling distribute [D 20:01:02.373919] PINT_distribute [D 20:01:02.373942] of 165675008 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 165675008 exfl 1 [D 20:01:02.373969] begin iteration loff: 165675008 [D 20:01:02.373993] file being extended [D 20:01:02.374017] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.374042] **********CALL***PROCESS********* [D 20:01:02.374066] segment of 31457280 sz 2097152 [D 20:01:02.374091] PINT_process_request [D 20:01:02.374117] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.374144] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.374169] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.374197] basic type or contiguous data [D 20:01:02.374221] Request disp [D 20:01:02.374245] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.374270] process a segment [D 20:01:02.374294] of 31457280 sz 2097152 [D 20:01:02.374319] add a segment 0 [D 20:01:02.374342] retval = 2097152 [D 20:01:02.374366] return from level -1 [D 20:01:02.374390] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.374417] *****RETURN***FROM***PROCESS***** [D 20:01:02.374441] end iteration [D 20:01:02.374464] done with segments or bytes [D 20:01:02.374488] of 167772160 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.374514] next loff: 167772160 next poff: 167772160 [D 20:01:02.374539] (return value) 2097152 [D 20:01:02.374563] finished [D 20:01:02.374587] retval = 2097152 [D 20:01:02.374610] exiting distribute returned less than expected [D 20:01:02.374634] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 167772160 fo 167772160 eof 0 [D 20:01:02.374661] ========================================================= [D 20:01:02.374685] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 12 [D 20:01:02.374711] element 0: offset: 0xbc269000, size: 2097152 [D 20:01:02.374737] post_sr: da13:3346 bh 95 len 32 wr_id 15e73468. [D 20:01:02.375995] openib_check_cq: completed rdma wr_id 15ecea40, nic_wr_credit 3. [D 20:01:02.376023] post_sr: da13:3346 bh 96 len 16 wr_id 15e73480. [D 20:01:02.378445] openib_check_cq: completed rdma wr_id 15eb7950, nic_wr_credit 2. [D 20:01:02.378473] post_sr: da13:3346 bh 97 len 16 wr_id 15e73498. [D 20:01:02.380901] openib_check_cq: completed rdma wr_id 15ea6170, nic_wr_credit 1. [D 20:01:02.380928] post_sr: da13:3346 bh 98 len 16 wr_id 15e734b0. [D 20:01:02.380954] openib_check_cq: completed send wr_id 15e73348. [D 20:01:02.381297] PINT_client_state_machine_test id 22 [D 20:01:02.383356] openib_check_cq: completed rdma wr_id 15e8c1c8, nic_wr_credit 0. [D 20:01:02.383383] post_sr: da13:3346 bh 99 len 16 wr_id 15e734c8. [D 20:01:02.383410] openib_check_cq: completed send wr_id 15e73360. [D 20:01:02.383435] openib_check_cq: completed send wr_id 15e73378. [D 20:01:02.383460] openib_check_cq: completed send wr_id 15e73390. [D 20:01:02.383485] openib_check_cq: completed send wr_id 15e733a8. [D 20:01:02.383510] openib_check_cq: completed send wr_id 15e733c0. [D 20:01:02.383536] openib_check_cq: completed send wr_id 15e733d8. [D 20:01:02.383561] openib_check_cq: completed send wr_id 15e733f0. [D 20:01:02.383585] openib_check_cq: completed send wr_id 15e73408. [D 20:01:02.383611] openib_check_cq: completed send wr_id 15e73420. [D 20:01:02.383636] openib_check_cq: completed send wr_id 15e73438. [D 20:01:02.383661] openib_check_cq: completed send wr_id 15e73450. [D 20:01:02.383687] openib_check_cq: completed send wr_id 15e73468. [D 20:01:02.383711] openib_check_cq: completed send wr_id 15e73480. [D 20:01:02.383737] openib_check_cq: completed send wr_id 15e73498. [D 20:01:02.383762] openib_check_cq: completed send wr_id 15e734b0. [D 20:01:02.383787] openib_check_cq: completed send wr_id 15e734c8. [D 20:01:02.383819] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:01:02.383845] ========================================================= [D 20:01:02.383870] PINT_process_request [D 20:01:02.383893] tiling 7 copies [D 20:01:02.383917] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.383944] lvl 0 el 0 blk 0 by 199229440 [D 20:01:02.383969] to 199229440 ta 167772160 fi 201326592 [D 20:01:02.383994] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.384018] basic type or contiguous data [D 20:01:02.384042] Request disp [D 20:01:02.384066] contig_offset = 199229440 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.384092] calling distribute [D 20:01:02.384116] PINT_distribute [D 20:01:02.384142] of 199229440 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 199229440 exfl 1 [D 20:01:02.384169] begin iteration loff: 199229440 [D 20:01:02.384197] file being extended [D 20:01:02.384224] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.384253] **********CALL***PROCESS********* [D 20:01:02.384280] segment of 31457280 sz 2097152 [D 20:01:02.384307] PINT_process_request [D 20:01:02.384332] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.384358] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.384383] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.384407] basic type or contiguous data [D 20:01:02.384431] Request disp [D 20:01:02.384455] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.384481] process a segment [D 20:01:02.384505] of 31457280 sz 2097152 [D 20:01:02.384529] add a segment 0 [D 20:01:02.384552] retval = 2097152 [D 20:01:02.384576] return from level -1 [D 20:01:02.384601] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.384627] *****RETURN***FROM***PROCESS***** [D 20:01:02.384652] end iteration [D 20:01:02.384675] done with segments or bytes [D 20:01:02.384699] of 201326592 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.384725] next loff: 201326592 next poff: 201326592 [D 20:01:02.384749] (return value) 2097152 [D 20:01:02.384774] finished [D 20:01:02.384798] retval = 2097152 [D 20:01:02.384821] exiting distribute returned less than expected [D 20:01:02.384845] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 201326592 fo 201326592 eof 0 [D 20:01:02.384872] ========================================================= [D 20:01:02.384897] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 13 [D 20:01:02.384922] element 0: offset: 0xba267000, size: 2097152 [D 20:01:02.384948] post_sr: da13:3346 bh 0 len 32 wr_id 15e72b80. [D 20:01:02.384977] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:01:02.385003] ========================================================= [D 20:01:02.385027] PINT_process_request [D 20:01:02.385051] tiling 8 copies [D 20:01:02.385075] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.385102] lvl 0 el 0 blk 0 by 232783872 [D 20:01:02.385127] to 232783872 ta 201326592 fi 234881024 [D 20:01:02.385151] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.385176] basic type or contiguous data [D 20:01:02.385200] Request disp [D 20:01:02.385224] contig_offset = 232783872 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.385249] calling distribute [D 20:01:02.385273] PINT_distribute [D 20:01:02.385296] of 232783872 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 232783872 exfl 1 [D 20:01:02.385323] begin iteration loff: 232783872 [D 20:01:02.385348] file being extended [D 20:01:02.385373] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.385397] **********CALL***PROCESS********* [D 20:01:02.385421] segment of 31457280 sz 2097152 [D 20:01:02.385445] PINT_process_request [D 20:01:02.385469] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.385495] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.385520] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.385544] basic type or contiguous data [D 20:01:02.385569] Request disp [D 20:01:02.385593] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.385618] process a segment [D 20:01:02.385642] of 31457280 sz 2097152 [D 20:01:02.385665] add a segment 0 [D 20:01:02.385689] retval = 2097152 [D 20:01:02.385713] return from level -1 [D 20:01:02.385738] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.385764] *****RETURN***FROM***PROCESS***** [D 20:01:02.385787] end iteration [D 20:01:02.385811] done with segments or bytes [D 20:01:02.385835] of 234881024 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.385861] next loff: 234881024 next poff: 234881024 [D 20:01:02.385886] (return value) 2097152 [D 20:01:02.385912] finished [D 20:01:02.385936] retval = 2097152 [D 20:01:02.385961] exiting distribute returned less than expected [D 20:01:02.385985] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 234881024 fo 234881024 eof 0 [D 20:01:02.386011] ========================================================= [D 20:01:02.386035] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 14 [D 20:01:02.386060] element 0: offset: 0xb8265000, size: 2097152 [D 20:01:02.386088] post_sr: da13:3346 bh 1 len 32 wr_id 15e72b98. [D 20:01:02.386116] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:01:02.386141] ========================================================= [D 20:01:02.386166] PINT_process_request [D 20:01:02.386193] tiling 9 copies [D 20:01:02.386218] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.386243] lvl 0 el 0 blk 0 by 266338304 [D 20:01:02.386268] to 266338304 ta 234881024 fi 268435456 [D 20:01:02.386293] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.386318] basic type or contiguous data [D 20:01:02.386342] Request disp [D 20:01:02.386365] contig_offset = 266338304 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.386390] calling distribute [D 20:01:02.386414] PINT_distribute [D 20:01:02.386438] of 266338304 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 266338304 exfl 1 [D 20:01:02.386465] begin iteration loff: 266338304 [D 20:01:02.386489] file being extended [D 20:01:02.386513] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.386538] **********CALL***PROCESS********* [D 20:01:02.386562] segment of 31457280 sz 2097152 [D 20:01:02.386586] PINT_process_request [D 20:01:02.386610] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.386635] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.386660] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.386685] basic type or contiguous data [D 20:01:02.386709] Request disp [D 20:01:02.386733] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.386757] process a segment [D 20:01:02.386781] of 31457280 sz 2097152 [D 20:01:02.386806] add a segment 0 [D 20:01:02.386830] retval = 2097152 [D 20:01:02.386854] return from level -1 [D 20:01:02.386878] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.386904] *****RETURN***FROM***PROCESS***** [D 20:01:02.386928] end iteration [D 20:01:02.386952] done with segments or bytes [D 20:01:02.386976] of 268435456 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.387001] next loff: 268435456 next poff: 268435456 [D 20:01:02.387025] (return value) 2097152 [D 20:01:02.387050] finished [D 20:01:02.387073] retval = 2097152 [D 20:01:02.387097] exiting distribute returned less than expected [D 20:01:02.387121] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 268435456 fo 268435456 eof 0 [D 20:01:02.387147] ========================================================= [D 20:01:02.387171] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 15 [D 20:01:02.387197] element 0: offset: 0xb6263000, size: 2097152 [D 20:01:02.387224] post_sr: da13:3346 bh 2 len 32 wr_id 15e72bb0. [D 20:01:02.387251] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:01:02.387277] ========================================================= [D 20:01:02.387302] PINT_process_request [D 20:01:02.387326] tiling 10 copies [D 20:01:02.387350] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.387376] lvl 0 el 0 blk 0 by 299892736 [D 20:01:02.387401] to 299892736 ta 268435456 fi 301989888 [D 20:01:02.387425] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.387450] basic type or contiguous data [D 20:01:02.387474] Request disp [D 20:01:02.387497] contig_offset = 299892736 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.387522] calling distribute [D 20:01:02.387547] PINT_distribute [D 20:01:02.387571] of 299892736 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 299892736 exfl 1 [D 20:01:02.387599] begin iteration loff: 299892736 [D 20:01:02.387627] file being extended [D 20:01:02.387651] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.387676] **********CALL***PROCESS********* [D 20:01:02.387699] segment of 31457280 sz 2097152 [D 20:01:02.387723] PINT_process_request [D 20:01:02.387747] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.387774] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.387799] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.387823] basic type or contiguous data [D 20:01:02.387846] Request disp [D 20:01:02.387870] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.387895] process a segment [D 20:01:02.387919] of 31457280 sz 2097152 [D 20:01:02.387943] add a segment 0 [D 20:01:02.387967] retval = 2097152 [D 20:01:02.387991] return from level -1 [D 20:01:02.388015] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.388041] *****RETURN***FROM***PROCESS***** [D 20:01:02.388066] end iteration [D 20:01:02.388089] done with segments or bytes [D 20:01:02.388113] of 301989888 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.388138] next loff: 301989888 next poff: 301989888 [D 20:01:02.388163] (return value) 2097152 [D 20:01:02.388187] finished [D 20:01:02.388216] retval = 2097152 [D 20:01:02.388244] exiting distribute returned less than expected [D 20:01:02.388272] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 301989888 fo 301989888 eof 0 [D 20:01:02.388301] ========================================================= [D 20:01:02.388326] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 16 [D 20:01:02.388352] element 0: offset: 0xb4261000, size: 2097152 [D 20:01:02.388379] post_sr: da13:3346 bh 3 len 32 wr_id 15e72bc8. [D 20:01:02.388407] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:01:02.388433] ========================================================= [D 20:01:02.388458] PINT_process_request [D 20:01:02.388482] tiling 11 copies [D 20:01:02.388505] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.388532] lvl 0 el 0 blk 0 by 333447168 [D 20:01:02.388557] to 333447168 ta 301989888 fi 335544320 [D 20:01:02.388582] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.388607] basic type or contiguous data [D 20:01:02.388630] Request disp [D 20:01:02.388654] contig_offset = 333447168 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.388679] calling distribute [D 20:01:02.388703] PINT_distribute [D 20:01:02.388727] of 333447168 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 333447168 exfl 1 [D 20:01:02.388753] begin iteration loff: 333447168 [D 20:01:02.388778] file being extended [D 20:01:02.388802] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.388827] **********CALL***PROCESS********* [D 20:01:02.388851] segment of 31457280 sz 2097152 [D 20:01:02.388875] PINT_process_request [D 20:01:02.388899] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.388926] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.388951] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.388976] basic type or contiguous data [D 20:01:02.388999] Request disp [D 20:01:02.389023] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.389048] process a segment [D 20:01:02.389072] of 31457280 sz 2097152 [D 20:01:02.389096] add a segment 0 [D 20:01:02.389121] retval = 2097152 [D 20:01:02.389144] return from level -1 [D 20:01:02.389168] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.389194] *****RETURN***FROM***PROCESS***** [D 20:01:02.389219] end iteration [D 20:01:02.389243] done with segments or bytes [D 20:01:02.389266] of 335544320 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.389291] next loff: 335544320 next poff: 335544320 [D 20:01:02.389316] (return value) 2097152 [D 20:01:02.389341] finished [D 20:01:02.389367] retval = 2097152 [D 20:01:02.389391] exiting distribute returned less than expected [D 20:01:02.389415] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 335544320 fo 335544320 eof 0 [D 20:01:02.389442] ========================================================= [D 20:01:02.389466] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 17 [D 20:01:02.389492] element 0: offset: 0xb225f000, size: 2097152 [D 20:01:02.389518] post_sr: da13:3346 bh 4 len 32 wr_id 15e72be0. [D 20:01:02.389549] openib_check_cq: completed send wr_id 15e72b80. [D 20:01:02.389575] openib_check_cq: completed send wr_id 15e72b98. [D 20:01:02.389600] openib_check_cq: completed send wr_id 15e72bb0. [D 20:01:02.389625] openib_check_cq: completed send wr_id 15e72bc8. [D 20:01:02.389651] openib_check_cq: completed send wr_id 15e72be0. [D 20:01:02.389683] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:01:02.389709] ========================================================= [D 20:01:02.389733] PINT_process_request [D 20:01:02.389758] tiling 12 copies [D 20:01:02.389781] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.389807] lvl 0 el 0 blk 0 by 367001600 [D 20:01:02.389832] to 367001600 ta 335544320 fi 369098752 [D 20:01:02.389857] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.389882] basic type or contiguous data [D 20:01:02.389906] Request disp [D 20:01:02.389930] contig_offset = 367001600 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.389955] calling distribute [D 20:01:02.389979] PINT_distribute [D 20:01:02.390003] of 367001600 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 367001600 exfl 1 [D 20:01:02.390028] begin iteration loff: 367001600 [D 20:01:02.390053] file being extended [D 20:01:02.390077] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.390102] **********CALL***PROCESS********* [D 20:01:02.390126] segment of 31457280 sz 2097152 [D 20:01:02.390151] PINT_process_request [D 20:01:02.390174] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.390205] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.390230] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.390254] basic type or contiguous data [D 20:01:02.390278] Request disp [D 20:01:02.390302] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.390327] process a segment [D 20:01:02.390351] of 31457280 sz 2097152 [D 20:01:02.390375] add a segment 0 [D 20:01:02.390398] retval = 2097152 [D 20:01:02.390423] return from level -1 [D 20:01:02.390447] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.390473] *****RETURN***FROM***PROCESS***** [D 20:01:02.390498] end iteration [D 20:01:02.390522] done with segments or bytes [D 20:01:02.390544] of 369098752 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.390570] next loff: 369098752 next poff: 369098752 [D 20:01:02.390595] (return value) 2097152 [D 20:01:02.390619] finished [D 20:01:02.390643] retval = 2097152 [D 20:01:02.390666] exiting distribute returned less than expected [D 20:01:02.390690] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 369098752 fo 369098752 eof 0 [D 20:01:02.390717] ========================================================= [D 20:01:02.390741] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 18 [D 20:01:02.390766] element 0: offset: 0xb025d000, size: 2097152 [D 20:01:02.390793] post_sr: da13:3346 bh 5 len 32 wr_id 15e72bf8. [D 20:01:02.390821] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:01:02.390847] ========================================================= [D 20:01:02.390872] PINT_process_request [D 20:01:02.390895] tiling 13 copies [D 20:01:02.390919] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.390946] lvl 0 el 0 blk 0 by 400556032 [D 20:01:02.390970] to 400556032 ta 369098752 fi 402653184 [D 20:01:02.390995] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.391019] basic type or contiguous data [D 20:01:02.391046] Request disp [D 20:01:02.391071] contig_offset = 400556032 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.391096] calling distribute [D 20:01:02.391119] PINT_distribute [D 20:01:02.391143] of 400556032 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 400556032 exfl 1 [D 20:01:02.391170] begin iteration loff: 400556032 [D 20:01:02.391195] file being extended [D 20:01:02.391219] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.391243] **********CALL***PROCESS********* [D 20:01:02.391266] segment of 31457280 sz 2097152 [D 20:01:02.391291] PINT_process_request [D 20:01:02.391315] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.391341] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.391366] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.391390] basic type or contiguous data [D 20:01:02.391414] Request disp [D 20:01:02.391438] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.391463] process a segment [D 20:01:02.391487] of 31457280 sz 2097152 [D 20:01:02.391511] add a segment 0 [D 20:01:02.391535] retval = 2097152 [D 20:01:02.391559] return from level -1 [D 20:01:02.391583] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.391609] *****RETURN***FROM***PROCESS***** [D 20:01:02.391633] end iteration [D 20:01:02.391656] done with segments or bytes [D 20:01:02.391680] of 402653184 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.391706] next loff: 402653184 next poff: 402653184 [D 20:01:02.391730] (return value) 2097152 [D 20:01:02.391755] finished [D 20:01:02.391778] retval = 2097152 [D 20:01:02.391802] exiting distribute returned less than expected [D 20:01:02.391826] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 402653184 fo 402653184 eof 0 [D 20:01:02.391852] ========================================================= [D 20:01:02.391877] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 19 [D 20:01:02.391901] element 0: offset: 0xae25b000, size: 2097152 [D 20:01:02.391928] post_sr: da13:3346 bh 6 len 32 wr_id 15e72c10. [D 20:01:02.391957] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:02.391982] ========================================================= [D 20:01:02.392006] PINT_process_request [D 20:01:02.392030] tiling 16 copies [D 20:01:02.392054] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.392081] lvl 0 el 0 blk 0 by 492830720 [D 20:01:02.392105] to 492830720 ta 469762048 fi 503316480 [D 20:01:02.392129] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:02.392154] basic type or contiguous data [D 20:01:02.392178] Request disp [D 20:01:02.392206] contig_offset = 492830720 contig_size = 44040192 lvl_flag = 1 [D 20:01:02.392234] calling distribute [D 20:01:02.392262] PINT_distribute [D 20:01:02.392289] of 492830720 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 492830720 exfl 1 [D 20:01:02.392318] begin iteration loff: 492830720 [D 20:01:02.392343] segment exceeds byte limit [D 20:01:02.392367] file being extended [D 20:01:02.392391] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:02.392416] **********CALL***PROCESS********* [D 20:01:02.392439] segment of 23068672 sz 2097152 [D 20:01:02.392463] PINT_process_request [D 20:01:02.392487] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.392513] lvl 0 el 0 blk 0 by 23068672 [D 20:01:02.392538] to 23068672 ta 23068672 fi 25165824 [D 20:01:02.392563] basic type or contiguous data [D 20:01:02.392587] Request disp [D 20:01:02.392611] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:02.392636] process a segment [D 20:01:02.392660] of 23068672 sz 2097152 [D 20:01:02.392683] add a segment 0 [D 20:01:02.392707] retval = 2097152 [D 20:01:02.392732] exiting distribute returned less than expected [D 20:01:02.392756] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:02.392782] *****RETURN***FROM***PROCESS***** [D 20:01:02.392808] end iteration [D 20:01:02.392832] done with segments or bytes [D 20:01:02.392856] of 494927872 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.392882] next loff: 494927872 next poff: 494927872 [D 20:01:02.392907] return value 2097152 [D 20:01:02.392931] finished [D 20:01:02.392954] retval = 2097152 [D 20:01:02.392978] exiting distribute returned less than expected [D 20:01:02.393002] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 494927872 fo 503316480 eof 0 [D 20:01:02.393029] ========================================================= [D 20:01:02.393053] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:02.393078] element 0: offset: 0xa7a55000, size: 2097152 [D 20:01:02.393105] post_sr: da13:3346 bh 7 len 32 wr_id 15e72c28. [D 20:01:02.393134] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:01:02.393159] ========================================================= [D 20:01:02.393183] PINT_process_request [D 20:01:02.393207] tiling 14 copies [D 20:01:02.393231] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.393258] lvl 0 el 0 blk 0 by 434110464 [D 20:01:02.393282] to 434110464 ta 402653184 fi 436207616 [D 20:01:02.393307] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.393332] basic type or contiguous data [D 20:01:02.393356] Request disp [D 20:01:02.393379] contig_offset = 434110464 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.393404] calling distribute [D 20:01:02.393427] PINT_distribute [D 20:01:02.393451] of 434110464 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 434110464 exfl 1 [D 20:01:02.393477] begin iteration loff: 434110464 [D 20:01:02.393502] file being extended [D 20:01:02.393526] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.393551] **********CALL***PROCESS********* [D 20:01:02.393574] segment of 31457280 sz 2097152 [D 20:01:02.393599] PINT_process_request [D 20:01:02.393623] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.393649] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.393673] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.393697] basic type or contiguous data [D 20:01:02.393721] Request disp [D 20:01:02.393745] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.393771] process a segment [D 20:01:02.393794] of 31457280 sz 2097152 [D 20:01:02.393818] add a segment 0 [D 20:01:02.393842] retval = 2097152 [D 20:01:02.393866] return from level -1 [D 20:01:02.393890] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.393916] *****RETURN***FROM***PROCESS***** [D 20:01:02.393940] end iteration [D 20:01:02.393964] done with segments or bytes [D 20:01:02.393987] of 436207616 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.394013] next loff: 436207616 next poff: 436207616 [D 20:01:02.394038] (return value) 2097152 [D 20:01:02.394062] finished [D 20:01:02.394085] retval = 2097152 [D 20:01:02.394109] exiting distribute returned less than expected [D 20:01:02.394133] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 436207616 fo 436207616 eof 0 [D 20:01:02.394160] ========================================================= [D 20:01:02.394184] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 20 [D 20:01:02.394212] element 0: offset: 0xac259000, size: 2097152 [D 20:01:02.394240] post_sr: da13:3346 bh 8 len 32 wr_id 15e72c40. [D 20:01:02.394268] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:01:02.394293] ========================================================= [D 20:01:02.394318] PINT_process_request [D 20:01:02.394342] tiling 15 copies [D 20:01:02.394366] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.394392] lvl 0 el 0 blk 0 by 467664896 [D 20:01:02.394417] to 467664896 ta 436207616 fi 469762048 [D 20:01:02.394441] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.394469] basic type or contiguous data [D 20:01:02.394494] Request disp [D 20:01:02.394517] contig_offset = 467664896 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.394542] calling distribute [D 20:01:02.394566] PINT_distribute [D 20:01:02.394590] of 467664896 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 467664896 exfl 1 [D 20:01:02.394616] begin iteration loff: 467664896 [D 20:01:02.394640] file being extended [D 20:01:02.394664] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.394689] **********CALL***PROCESS********* [D 20:01:02.394713] segment of 31457280 sz 2097152 [D 20:01:02.394738] PINT_process_request [D 20:01:02.394762] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.394787] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.394812] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.394837] basic type or contiguous data [D 20:01:02.394861] Request disp [D 20:01:02.394885] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.394909] process a segment [D 20:01:02.394933] of 31457280 sz 2097152 [D 20:01:02.394957] add a segment 0 [D 20:01:02.394981] retval = 2097152 [D 20:01:02.395005] return from level -1 [D 20:01:02.395029] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.395055] *****RETURN***FROM***PROCESS***** [D 20:01:02.395079] end iteration [D 20:01:02.395103] done with segments or bytes [D 20:01:02.395126] of 469762048 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.395152] next loff: 469762048 next poff: 469762048 [D 20:01:02.395176] (return value) 2097152 [D 20:01:02.395201] finished [D 20:01:02.395225] retval = 2097152 [D 20:01:02.395249] exiting distribute returned less than expected [D 20:01:02.395273] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 469762048 fo 469762048 eof 0 [D 20:01:02.395298] ========================================================= [D 20:01:02.395323] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 21 [D 20:01:02.395348] element 0: offset: 0xaa257000, size: 2097152 [D 20:01:02.395375] post_sr: da13:3346 bh 9 len 32 wr_id 15e72c58. [D 20:01:02.395404] PINT_client_state_machine_test id 22 [D 20:01:02.395433] openib_check_cq: completed send wr_id 15e72bf8. [D 20:01:02.395459] openib_check_cq: completed send wr_id 15e72c10. [D 20:01:02.395484] openib_check_cq: completed send wr_id 15e72c28. [D 20:01:02.395509] openib_check_cq: completed send wr_id 15e72c40. [D 20:01:02.395534] openib_check_cq: completed send wr_id 15e72c58. [D 20:01:02.395566] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e75278. [D 20:01:02.395592] flowproto completing 0x15e75278 [D 20:01:02.395619] Job flows in progress (callback time): 29 [D 20:01:02.395645] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e76ce0. [D 20:01:02.395670] flowproto completing 0x15e76ce0 [D 20:01:02.395697] Job flows in progress (callback time): 28 [D 20:01:02.395722] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e869b8. [D 20:01:02.395747] flowproto completing 0x15e869b8 [D 20:01:02.395773] Job flows in progress (callback time): 27 [D 20:01:02.395798] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e89d98. [D 20:01:02.395824] flowproto completing 0x15e89d98 [D 20:01:02.395850] Job flows in progress (callback time): 26 [D 20:01:02.395882] sm control op 6 [D 20:01:02.395907] [SM Entering]: (0x15e75b78) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.395935] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.395959] (0x15e62108) io_datafile_complete_operations (tag 2) [D 20:01:02.395992] matched completed flow for context 0x15e74cc8 and reset write_recv timeout [D 20:01:02.396018] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.396043] [SM Exiting]: (0x15e75b78) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.396069] sm control op 6 [D 20:01:02.396094] [SM Entering]: (0x15e75f20) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.396121] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.396147] (0x15e75fb8) io_datafile_complete_operations (tag 2) [D 20:01:02.396177] matched completed flow for context 0x15e76730 and reset write_recv timeout [D 20:01:02.396205] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.396234] [SM Exiting]: (0x15e75f20) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.396265] sm control op 6 [D 20:01:02.396292] [SM Entering]: (0x15e85bb0) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.396320] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.396345] (0x15e85c48) io_datafile_complete_operations (tag 2) [D 20:01:02.396374] matched completed flow for context 0x15e86408 and reset write_recv timeout [D 20:01:02.396399] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.396425] [SM Exiting]: (0x15e85bb0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.396450] sm control op 6 [D 20:01:02.396474] [SM Entering]: (0x15e88fa0) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.396500] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.396525] (0x15e89038) io_datafile_complete_operations (tag 2) [D 20:01:02.396553] matched completed flow for context 0x15e897e8 and reset write_recv timeout [D 20:01:02.396578] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.396603] [SM Exiting]: (0x15e88fa0) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.396629] PINT_client_state_machine_test id 22 [D 20:01:02.406659] PINT_client_state_machine_test id 22 [D 20:01:02.416687] PINT_client_state_machine_test id 22 [D 20:01:02.416716] sm control op 300 [D 20:01:02.416741] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:01:02.416767] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:01:02.416796] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:01:02.416822] PINT_client_state_machine_test id 22 [D 20:01:02.426852] PINT_client_state_machine_test id 22 [D 20:01:02.431415] openib_check_cq: completed recv wr_id 15e73d88. [D 20:01:02.431445] openib_post_rr: da13:3346 bh 92 wr_id 15e73d88. [D 20:01:02.431471] openib_check_cq: completed recv wr_id 15e73da0. [D 20:01:02.431499] openib_post_rr: da13:3346 bh 93 wr_id 15e73da0. [D 20:01:02.431525] openib_check_cq: completed recv wr_id 15e73db8. [D 20:01:02.431552] openib_post_rr: da13:3346 bh 94 wr_id 15e73db8. [D 20:01:02.431579] openib_check_cq: completed recv wr_id 15e73dd0. [D 20:01:02.431607] openib_post_rr: da13:3346 bh 95 wr_id 15e73dd0. [D 20:01:02.431632] openib_check_cq: completed recv wr_id 15e73de8. [D 20:01:02.431660] openib_post_rr: da13:3346 bh 96 wr_id 15e73de8. [D 20:01:02.431686] openib_check_cq: completed recv wr_id 15e73e00. [D 20:01:02.431713] openib_post_rr: da13:3346 bh 97 wr_id 15e73e00. [D 20:01:02.431739] post_sr: da13:3346 bh 10 len 48 wr_id 15e72c70. [D 20:01:02.431766] openib_check_cq: completed recv wr_id 15e73e18. [D 20:01:02.431794] openib_post_rr: da13:3346 bh 98 wr_id 15e73e18. [D 20:01:02.431819] openib_check_cq: completed recv wr_id 15e73e30. [D 20:01:02.431847] openib_post_rr: da13:3346 bh 99 wr_id 15e73e30. [D 20:01:02.431873] openib_check_cq: completed recv wr_id 15e734e8. [D 20:01:02.431900] openib_post_rr: da13:3346 bh 0 wr_id 15e734e8. [D 20:01:02.431927] openib_check_cq: completed recv wr_id 15e73500. [D 20:01:02.431955] openib_post_rr: da13:3346 bh 1 wr_id 15e73500. [D 20:01:02.431982] openib_check_cq: completed send wr_id 15e72c70. [D 20:01:02.436882] PINT_client_state_machine_test id 22 [D 20:01:02.445261] openib_check_cq: completed recv wr_id 15e73518. [D 20:01:02.445289] openib_post_rr: da13:3346 bh 2 wr_id 15e73518. [D 20:01:02.445316] openib_check_cq: completed recv wr_id 15e73530. [D 20:01:02.445344] openib_post_rr: da13:3346 bh 3 wr_id 15e73530. [D 20:01:02.445375] openib_check_cq: completed recv wr_id 15e73548. [D 20:01:02.445404] openib_post_rr: da13:3346 bh 4 wr_id 15e73548. [D 20:01:02.445430] openib_check_cq: completed recv wr_id 15e73560. [D 20:01:02.445458] openib_post_rr: da13:3346 bh 5 wr_id 15e73560. [D 20:01:02.445485] openib_check_cq: completed recv wr_id 15e73578. [D 20:01:02.445511] openib_post_rr: da13:3346 bh 6 wr_id 15e73578. [D 20:01:02.445537] openib_check_cq: completed recv wr_id 15e73590. [D 20:01:02.445565] openib_post_rr: da13:3346 bh 7 wr_id 15e73590. [D 20:01:02.445590] openib_check_cq: completed recv wr_id 15e735a8. [D 20:01:02.445616] openib_post_sr_rdmaw: ibv_post_send wr_id 15f165f8 to da13:3346 remote addr 2aaab9c46010 rkey c8008e00 nic_wr_credit 1. [D 20:01:02.445643] openib_post_rr: da13:3346 bh 8 wr_id 15e735a8. [D 20:01:02.445670] openib_check_cq: completed recv wr_id 15e735c0. [D 20:01:02.445695] openib_post_sr_rdmaw: ibv_post_send wr_id 15f16518 to da13:3346 remote addr 2aaab9e47010 rkey c8008f00 nic_wr_credit 2. [D 20:01:02.445722] openib_post_rr: da13:3346 bh 9 wr_id 15e735c0. [D 20:01:02.445748] openib_check_cq: completed recv wr_id 15e735d8. [D 20:01:02.445775] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaaba048010 rkey 48009000 nic_wr_credit 3. [D 20:01:02.445800] openib_post_rr: da13:3346 bh 10 wr_id 15e735d8. [D 20:01:02.445827] openib_check_cq: completed recv wr_id 15e735f0. [D 20:01:02.445853] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef5290 to da13:3346 remote addr 2aaaba249010 rkey 48009100 nic_wr_credit 4. [D 20:01:02.445879] openib_post_rr: da13:3346 bh 11 wr_id 15e735f0. [D 20:01:02.445905] openib_check_cq: completed recv wr_id 15e73608. [D 20:01:02.445932] openib_post_sr_rdmaw: ibv_post_send wr_id 15eac958 to da13:3346 remote addr 2aaaba44a010 rkey 48009200 nic_wr_credit 5. [D 20:01:02.445959] openib_post_rr: da13:3346 bh 12 wr_id 15e73608. [D 20:01:02.445984] openib_check_cq: completed recv wr_id 15e73620. [D 20:01:02.446010] openib_post_sr_rdmaw: ibv_post_send wr_id 15e9f978 to da13:3346 remote addr 2aaaba64b010 rkey 48009300 nic_wr_credit 6. [D 20:01:02.446037] openib_post_rr: da13:3346 bh 13 wr_id 15e73620. [D 20:01:02.446063] openib_check_cq: completed recv wr_id 15e73638. [D 20:01:02.446089] openib_post_rr: da13:3346 bh 14 wr_id 15e73638. [D 20:01:02.446116] openib_check_cq: completed recv wr_id 15e73650. [D 20:01:02.446142] openib_post_sr_rdmaw: ibv_post_send wr_id 15e75770 to da13:3346 remote addr 2aaaba84c010 rkey 48009400 nic_wr_credit 7. [D 20:01:02.446168] openib_post_rr: da13:3346 bh 15 wr_id 15e73650. [D 20:01:02.446197] openib_check_cq: completed recv wr_id 15e73668. [D 20:01:02.446224] openib_post_sr_rdmaw: ibv_post_send wr_id 15effa68 to da13:3346 remote addr 2aaabaa4d010 rkey 48009500 nic_wr_credit 8. [D 20:01:02.446250] openib_post_rr: da13:3346 bh 16 wr_id 15e73668. [D 20:01:02.446276] openib_check_cq: completed recv wr_id 15e73680. [D 20:01:02.446303] openib_post_sr_rdmaw: ibv_post_send wr_id 15f02500 to da13:3346 remote addr 2aaab9844010 rkey c8008c00 nic_wr_credit 9. [D 20:01:02.446330] openib_post_rr: da13:3346 bh 17 wr_id 15e73680. [D 20:01:02.446355] openib_check_cq: completed recv wr_id 15e73698. [D 20:01:02.446382] openib_post_sr_rdmaw: ibv_post_send wr_id 15f04f98 to da13:3346 remote addr 2aaabac4f010 rkey 48009600 nic_wr_credit 10. [D 20:01:02.446409] openib_post_rr: da13:3346 bh 18 wr_id 15e73698. [D 20:01:02.446443] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.446469] ========================================================= [D 20:01:02.446494] PINT_process_request [D 20:01:02.446517] tiling 4 copies [D 20:01:02.446542] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.446568] lvl 0 el 0 blk 0 by 69206016 [D 20:01:02.446593] to 69206016 ta 67108864 fi 100663296 [D 20:01:02.446618] mto 2097152 mta 0 mfi 2097152 [D 20:01:02.446642] basic type or contiguous data [D 20:01:02.446666] Request disp [D 20:01:02.446690] contig_offset = 69206016 contig_size = 65011712 lvl_flag = 1 [D 20:01:02.446718] calling distribute [D 20:01:02.446743] PINT_distribute [D 20:01:02.446766] of 69206016 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.446793] begin iteration loff: 69206016 [D 20:01:02.446818] segment exceeds byte limit [D 20:01:02.446843] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:02.446867] **********CALL***PROCESS********* [D 20:01:02.446891] segment of 2097152 sz 2097152 [D 20:01:02.446915] PINT_process_request [D 20:01:02.446939] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.446966] lvl 0 el 0 blk 0 by 2097152 [D 20:01:02.446990] to 2097152 ta 2097152 fi 4194304 [D 20:01:02.447014] basic type or contiguous data [D 20:01:02.447038] Request disp [D 20:01:02.447062] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:02.447087] process a segment [D 20:01:02.447111] of 2097152 sz 2097152 [D 20:01:02.447134] add a segment 0 [D 20:01:02.447158] retval = 2097152 [D 20:01:02.447182] exiting distribute returned less than expected [D 20:01:02.447206] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:02.447233] *****RETURN***FROM***PROCESS***** [D 20:01:02.447257] end iteration [D 20:01:02.447280] done with segments or bytes [D 20:01:02.447304] of 71303168 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.447330] next loff: 71303168 next poff: 71303168 [D 20:01:02.447354] return value 2097152 [D 20:01:02.447379] finished [D 20:01:02.447402] retval = 2097152 [D 20:01:02.447426] exiting distribute returned less than expected [D 20:01:02.447450] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 71303168 fo 100663296 eof 0 [D 20:01:02.447477] ========================================================= [D 20:01:02.447502] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.447526] element 0: offset: 0x9e64d000, size: 2097152 [D 20:01:02.447557] post_sr: da13:3346 bh 11 len 48 wr_id 15e72c88. [D 20:01:02.447584] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.447611] ========================================================= [D 20:01:02.447636] PINT_process_request [D 20:01:02.447660] tiling 15 copies [D 20:01:02.447684] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.447710] lvl 0 el 0 blk 0 by 446693376 [D 20:01:02.447735] to 446693376 ta 436207616 fi 469762048 [D 20:01:02.447760] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:02.447785] basic type or contiguous data [D 20:01:02.447809] Request disp [D 20:01:02.447833] contig_offset = 446693376 contig_size = 56623104 lvl_flag = 1 [D 20:01:02.447857] calling distribute [D 20:01:02.447881] PINT_distribute [D 20:01:02.447905] of 446693376 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.447932] begin iteration loff: 446693376 [D 20:01:02.447956] segment exceeds byte limit [D 20:01:02.447979] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:02.448004] **********CALL***PROCESS********* [D 20:01:02.448028] segment of 10485760 sz 2097152 [D 20:01:02.448053] PINT_process_request [D 20:01:02.448076] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.448101] lvl 0 el 0 blk 0 by 10485760 [D 20:01:02.448126] to 10485760 ta 10485760 fi 12582912 [D 20:01:02.448151] basic type or contiguous data [D 20:01:02.448175] Request disp [D 20:01:02.448202] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:02.448231] process a segment [D 20:01:02.448258] of 10485760 sz 2097152 [D 20:01:02.448286] add a segment 0 [D 20:01:02.448313] retval = 2097152 [D 20:01:02.448337] exiting distribute returned less than expected [D 20:01:02.448361] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:02.448388] *****RETURN***FROM***PROCESS***** [D 20:01:02.448411] end iteration [D 20:01:02.448435] done with segments or bytes [D 20:01:02.448459] of 448790528 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.448488] next loff: 448790528 next poff: 448790528 [D 20:01:02.448513] return value 2097152 [D 20:01:02.448536] finished [D 20:01:02.448560] retval = 2097152 [D 20:01:02.448584] exiting distribute returned less than expected [D 20:01:02.448608] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 448790528 fo 469762048 eof 0 [D 20:01:02.448635] ========================================================= [D 20:01:02.448658] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.448684] element 0: offset: 0x88e37000, size: 2097152 [D 20:01:02.448710] openib_check_cq: completed recv wr_id 15e736b0. [D 20:01:02.448737] openib_post_sr_rdmaw: ibv_post_send wr_id 15ef7bb0 to da13:3346 remote addr 2aaabae50010 rkey 48009700 nic_wr_credit 11. [D 20:01:02.448763] openib_post_rr: da13:3346 bh 19 wr_id 15e736b0. [D 20:01:02.448790] openib_check_cq: completed rdma wr_id 15f165f8, nic_wr_credit 10. [D 20:01:02.448817] post_sr: da13:3346 bh 12 len 16 wr_id 15e72ca0. [D 20:01:02.448846] post_sr: da13:3346 bh 13 len 48 wr_id 15e72cb8. [D 20:01:02.448873] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.448899] ========================================================= [D 20:01:02.448923] PINT_process_request [D 20:01:02.448946] tiling 2 copies [D 20:01:02.448971] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.448997] lvl 0 el 0 blk 0 by 8388608 [D 20:01:02.449022] to 8388608 ta 0 fi 33554432 [D 20:01:02.449047] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:02.449070] basic type or contiguous data [D 20:01:02.449094] Request disp [D 20:01:02.449118] contig_offset = 8388608 contig_size = 58720256 lvl_flag = 1 [D 20:01:02.449143] calling distribute [D 20:01:02.449167] PINT_distribute [D 20:01:02.449191] of 8388608 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.449217] begin iteration loff: 8388608 [D 20:01:02.449242] segment exceeds byte limit [D 20:01:02.449266] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:02.449291] **********CALL***PROCESS********* [D 20:01:02.449315] segment of 8388608 sz 2097152 [D 20:01:02.449339] PINT_process_request [D 20:01:02.449363] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.449389] lvl 0 el 0 blk 0 by 8388608 [D 20:01:02.449414] to 8388608 ta 8388608 fi 10485760 [D 20:01:02.449439] basic type or contiguous data [D 20:01:02.449462] Request disp [D 20:01:02.449486] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:02.449511] process a segment [D 20:01:02.449535] of 8388608 sz 2097152 [D 20:01:02.449559] add a segment 0 [D 20:01:02.449583] retval = 2097152 [D 20:01:02.449606] exiting distribute returned less than expected [D 20:01:02.449630] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:02.449656] *****RETURN***FROM***PROCESS***** [D 20:01:02.449681] end iteration [D 20:01:02.449705] done with segments or bytes [D 20:01:02.449728] of 10485760 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.449753] next loff: 10485760 next poff: 10485760 [D 20:01:02.449778] return value 2097152 [D 20:01:02.449803] finished [D 20:01:02.449826] retval = 2097152 [D 20:01:02.449850] exiting distribute returned less than expected [D 20:01:02.449874] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 10485760 fo 33554432 eof 0 [D 20:01:02.449900] ========================================================= [D 20:01:02.449925] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.449950] element 0: offset: 0xa2c51000, size: 2097152 [D 20:01:02.449979] post_sr: da13:3346 bh 14 len 48 wr_id 15e72cd0. [D 20:01:02.450007] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.450033] ========================================================= [D 20:01:02.450057] PINT_process_request [D 20:01:02.450081] tiling 3 copies [D 20:01:02.450108] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.450135] lvl 0 el 0 blk 0 by 39845888 [D 20:01:02.450159] to 39845888 ta 33554432 fi 67108864 [D 20:01:02.450183] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:02.450213] basic type or contiguous data [D 20:01:02.450237] Request disp [D 20:01:02.450261] contig_offset = 39845888 contig_size = 60817408 lvl_flag = 1 [D 20:01:02.450285] calling distribute [D 20:01:02.450309] PINT_distribute [D 20:01:02.450333] of 39845888 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.450360] begin iteration loff: 39845888 [D 20:01:02.450384] segment exceeds byte limit [D 20:01:02.450407] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:02.450432] **********CALL***PROCESS********* [D 20:01:02.450456] segment of 6291456 sz 2097152 [D 20:01:02.450481] PINT_process_request [D 20:01:02.450505] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.450531] lvl 0 el 0 blk 0 by 6291456 [D 20:01:02.450555] to 6291456 ta 6291456 fi 8388608 [D 20:01:02.450579] basic type or contiguous data [D 20:01:02.450603] Request disp [D 20:01:02.450627] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:02.450652] process a segment [D 20:01:02.450675] of 6291456 sz 2097152 [D 20:01:02.450699] add a segment 0 [D 20:01:02.450723] retval = 2097152 [D 20:01:02.450747] exiting distribute returned less than expected [D 20:01:02.450771] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:02.450797] *****RETURN***FROM***PROCESS***** [D 20:01:02.450820] end iteration [D 20:01:02.450844] done with segments or bytes [D 20:01:02.450868] of 41943040 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.450894] next loff: 41943040 next poff: 41943040 [D 20:01:02.450918] return value 2097152 [D 20:01:02.450942] finished [D 20:01:02.450966] retval = 2097152 [D 20:01:02.450990] exiting distribute returned less than expected [D 20:01:02.451014] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 41943040 fo 67108864 eof 0 [D 20:01:02.451040] ========================================================= [D 20:01:02.451065] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.451089] element 0: offset: 0xa0a4f000, size: 2097152 [D 20:01:02.451115] openib_check_cq: completed rdma wr_id 15f16518, nic_wr_credit 9. [D 20:01:02.451142] post_sr: da13:3346 bh 15 len 16 wr_id 15e72ce8. [D 20:01:02.451172] post_sr: da13:3346 bh 16 len 48 wr_id 15e72d00. [D 20:01:02.451200] PINT_client_state_machine_test id 22 [D 20:01:02.453027] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 8. [D 20:01:02.453055] post_sr: da13:3346 bh 17 len 16 wr_id 15e72d18. [D 20:01:02.455487] openib_check_cq: completed rdma wr_id 15ef5290, nic_wr_credit 7. [D 20:01:02.455514] post_sr: da13:3346 bh 18 len 16 wr_id 15e72d30. [D 20:01:02.457946] openib_check_cq: completed rdma wr_id 15eac958, nic_wr_credit 6. [D 20:01:02.457975] post_sr: da13:3346 bh 19 len 16 wr_id 15e72d48. [D 20:01:02.460402] openib_check_cq: completed rdma wr_id 15e9f978, nic_wr_credit 5. [D 20:01:02.460430] post_sr: da13:3346 bh 20 len 16 wr_id 15e72d60. [D 20:01:02.461232] PINT_client_state_machine_test id 22 [D 20:01:02.462863] openib_check_cq: completed rdma wr_id 15e75770, nic_wr_credit 4. [D 20:01:02.462890] post_sr: da13:3346 bh 21 len 16 wr_id 15e72d78. [D 20:01:02.465322] openib_check_cq: completed rdma wr_id 15effa68, nic_wr_credit 3. [D 20:01:02.465350] post_sr: da13:3346 bh 22 len 16 wr_id 15e72d90. [D 20:01:02.467776] openib_check_cq: completed rdma wr_id 15f02500, nic_wr_credit 2. [D 20:01:02.467803] post_sr: da13:3346 bh 23 len 16 wr_id 15e72da8. [D 20:01:02.470232] openib_check_cq: completed rdma wr_id 15f04f98, nic_wr_credit 1. [D 20:01:02.470260] post_sr: da13:3346 bh 24 len 16 wr_id 15e72dc0. [D 20:01:02.470287] openib_check_cq: completed send wr_id 15e72c88. [D 20:01:02.471261] PINT_client_state_machine_test id 22 [D 20:01:02.472698] openib_check_cq: completed rdma wr_id 15ef7bb0, nic_wr_credit 0. [D 20:01:02.472725] post_sr: da13:3346 bh 25 len 16 wr_id 15e72dd8. [D 20:01:02.472751] openib_check_cq: completed send wr_id 15e72ca0. [D 20:01:02.472777] openib_check_cq: completed send wr_id 15e72cb8. [D 20:01:02.472803] openib_check_cq: completed send wr_id 15e72cd0. [D 20:01:02.472827] openib_check_cq: completed send wr_id 15e72ce8. [D 20:01:02.472852] openib_check_cq: completed send wr_id 15e72d00. [D 20:01:02.472878] openib_check_cq: completed send wr_id 15e72d18. [D 20:01:02.472903] openib_check_cq: completed send wr_id 15e72d30. [D 20:01:02.472928] openib_check_cq: completed send wr_id 15e72d48. [D 20:01:02.472953] openib_check_cq: completed send wr_id 15e72d60. [D 20:01:02.472978] openib_check_cq: completed send wr_id 15e72d78. [D 20:01:02.473003] openib_check_cq: completed send wr_id 15e72d90. [D 20:01:02.473029] openib_check_cq: completed send wr_id 15e72da8. [D 20:01:02.473053] openib_check_cq: completed send wr_id 15e72dc0. [D 20:01:02.473078] openib_check_cq: completed send wr_id 15e72dd8. [D 20:01:02.473112] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e8d190. [D 20:01:02.473137] flowproto completing 0x15e8d190 [D 20:01:02.473163] Job flows in progress (callback time): 25 [D 20:01:02.473188] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e90588. [D 20:01:02.473214] flowproto completing 0x15e90588 [D 20:01:02.473239] Job flows in progress (callback time): 24 [D 20:01:02.473265] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e93980. [D 20:01:02.473290] flowproto completing 0x15e93980 [D 20:01:02.473316] Job flows in progress (callback time): 23 [D 20:01:02.473340] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e96d60. [D 20:01:02.473366] flowproto completing 0x15e96d60 [D 20:01:02.473392] Job flows in progress (callback time): 22 [D 20:01:02.473417] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9a158. [D 20:01:02.473442] flowproto completing 0x15e9a158 [D 20:01:02.473467] Job flows in progress (callback time): 21 [D 20:01:02.473500] sm control op 6 [D 20:01:02.473525] [SM Entering]: (0x15e8c340) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.473550] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.473575] (0x15e8c3d8) io_datafile_complete_operations (tag 2) [D 20:01:02.473605] matched completed flow for context 0x15e8cbe0 and reset write_recv timeout [D 20:01:02.473630] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.473655] [SM Exiting]: (0x15e8c340) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.473681] sm control op 6 [D 20:01:02.473706] [SM Entering]: (0x15e8f790) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.473731] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.473756] (0x15e8f828) io_datafile_complete_operations (tag 2) [D 20:01:02.473785] matched completed flow for context 0x15e8ffd8 and reset write_recv timeout [D 20:01:02.473810] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.473834] [SM Exiting]: (0x15e8f790) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.473860] sm control op 6 [D 20:01:02.473885] [SM Entering]: (0x15e92b88) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.473910] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.473936] (0x15e92c20) io_datafile_complete_operations (tag 2) [D 20:01:02.473964] matched completed flow for context 0x15e933d0 and reset write_recv timeout [D 20:01:02.473989] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.474014] [SM Exiting]: (0x15e92b88) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.474040] sm control op 6 [D 20:01:02.474064] [SM Entering]: (0x15e95f68) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.474089] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.474117] (0x15e96000) io_datafile_complete_operations (tag 2) [D 20:01:02.474146] matched completed flow for context 0x15e967b0 and reset write_recv timeout [D 20:01:02.474171] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.474200] [SM Exiting]: (0x15e95f68) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.474225] sm control op 6 [D 20:01:02.474250] [SM Entering]: (0x15e99308) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.474275] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.474301] (0x15e993a0) io_datafile_complete_operations (tag 2) [D 20:01:02.474329] matched completed flow for context 0x15e99ba8 and reset write_recv timeout [D 20:01:02.474355] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.474380] [SM Exiting]: (0x15e99308) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.474406] PINT_client_state_machine_test id 22 [D 20:01:02.474440] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15e9d550. [D 20:01:02.474467] flowproto completing 0x15e9d550 [D 20:01:02.474493] Job flows in progress (callback time): 20 [D 20:01:02.474517] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea0948. [D 20:01:02.474543] flowproto completing 0x15ea0948 [D 20:01:02.474569] Job flows in progress (callback time): 19 [D 20:01:02.474594] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea3d40. [D 20:01:02.474618] flowproto completing 0x15ea3d40 [D 20:01:02.474644] Job flows in progress (callback time): 18 [D 20:01:02.474669] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:02.474695] ========================================================= [D 20:01:02.474719] PINT_process_request [D 20:01:02.474742] tiling 16 copies [D 20:01:02.474766] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.474793] lvl 0 el 0 blk 0 by 494927872 [D 20:01:02.474818] to 494927872 ta 469762048 fi 503316480 [D 20:01:02.474843] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:02.474866] basic type or contiguous data [D 20:01:02.474890] Request disp [D 20:01:02.474914] contig_offset = 494927872 contig_size = 41943040 lvl_flag = 1 [D 20:01:02.474939] calling distribute [D 20:01:02.474963] PINT_distribute [D 20:01:02.474987] of 494927872 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 494927872 exfl 1 [D 20:01:02.475013] begin iteration loff: 494927872 [D 20:01:02.475038] segment exceeds byte limit [D 20:01:02.475062] file being extended [D 20:01:02.475086] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:02.475111] **********CALL***PROCESS********* [D 20:01:02.475133] segment of 25165824 sz 2097152 [D 20:01:02.475158] PINT_process_request [D 20:01:02.475182] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.475209] lvl 0 el 0 blk 0 by 25165824 [D 20:01:02.475233] to 25165824 ta 25165824 fi 27262976 [D 20:01:02.475257] basic type or contiguous data [D 20:01:02.475281] Request disp [D 20:01:02.475305] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:02.475330] process a segment [D 20:01:02.475354] of 25165824 sz 2097152 [D 20:01:02.475378] add a segment 0 [D 20:01:02.475402] retval = 2097152 [D 20:01:02.475426] exiting distribute returned less than expected [D 20:01:02.475451] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:02.475477] *****RETURN***FROM***PROCESS***** [D 20:01:02.475501] end iteration [D 20:01:02.475524] done with segments or bytes [D 20:01:02.475548] of 497025024 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.475574] next loff: 497025024 next poff: 497025024 [D 20:01:02.475599] return value 2097152 [D 20:01:02.475624] finished [D 20:01:02.475646] retval = 2097152 [D 20:01:02.475671] exiting distribute returned less than expected [D 20:01:02.475695] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 497025024 fo 503316480 eof 0 [D 20:01:02.475724] ========================================================= [D 20:01:02.475749] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:02.475773] element 0: offset: 0xa7c55000, size: 2097152 [D 20:01:02.475801] post_sr: da13:3346 bh 26 len 32 wr_id 15e72df0. [D 20:01:02.475830] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15ea7138. [D 20:01:02.475855] flowproto completing 0x15ea7138 [D 20:01:02.475881] Job flows in progress (callback time): 17 [D 20:01:02.475912] sm control op 6 [D 20:01:02.475936] [SM Entering]: (0x15e9c758) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.475962] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.475988] (0x15e9c7f0) io_datafile_complete_operations (tag 2) [D 20:01:02.476016] matched completed flow for context 0x15e9cfa0 and reset write_recv timeout [D 20:01:02.476041] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.476067] [SM Exiting]: (0x15e9c758) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.476093] sm control op 6 [D 20:01:02.476117] [SM Entering]: (0x15e9fb50) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.476142] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.476167] (0x15e9fbe8) io_datafile_complete_operations (tag 2) [D 20:01:02.476199] matched completed flow for context 0x15ea0398 and reset write_recv timeout [D 20:01:02.476228] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.476257] [SM Exiting]: (0x15e9fb50) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.476286] sm control op 6 [D 20:01:02.476314] [SM Entering]: (0x15ea2f48) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.476340] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.476365] (0x15ea2fe0) io_datafile_complete_operations (tag 2) [D 20:01:02.476393] matched completed flow for context 0x15ea3790 and reset write_recv timeout [D 20:01:02.476419] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.476444] [SM Exiting]: (0x15ea2f48) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.476469] sm control op 6 [D 20:01:02.476493] [SM Entering]: (0x15ea6340) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.476519] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.476544] (0x15ea63d8) io_datafile_complete_operations (tag 2) [D 20:01:02.476573] matched completed flow for context 0x15ea6b88 and reset write_recv timeout [D 20:01:02.476598] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.476623] [SM Exiting]: (0x15ea6340) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.476650] PINT_client_state_machine_test id 22 [D 20:01:02.476677] openib_check_cq: completed send wr_id 15e72df0. [D 20:01:02.476708] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaa530. [D 20:01:02.476734] flowproto completing 0x15eaa530 [D 20:01:02.476759] Job flows in progress (callback time): 16 [D 20:01:02.476786] sm control op 6 [D 20:01:02.476811] [SM Entering]: (0x15ea9738) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.476837] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.476861] (0x15ea97d0) io_datafile_complete_operations (tag 2) [D 20:01:02.476890] matched completed flow for context 0x15ea9f80 and reset write_recv timeout [D 20:01:02.476915] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.476941] [SM Exiting]: (0x15ea9738) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.476966] PINT_client_state_machine_test id 22 [D 20:01:02.486995] PINT_client_state_machine_test id 22 [D 20:01:02.497025] PINT_client_state_machine_test id 22 [D 20:01:02.507055] PINT_client_state_machine_test id 22 [D 20:01:02.507928] openib_check_cq: completed recv wr_id 15e736c8. [D 20:01:02.507959] openib_post_rr: da13:3346 bh 20 wr_id 15e736c8. [D 20:01:02.508217] openib_check_cq: completed recv wr_id 15e736e0. [D 20:01:02.508250] openib_post_rr: da13:3346 bh 21 wr_id 15e736e0. [D 20:01:02.508280] openib_check_cq: completed recv wr_id 15e736f8. [D 20:01:02.508311] openib_post_rr: da13:3346 bh 22 wr_id 15e736f8. [D 20:01:02.517086] PINT_client_state_machine_test id 22 [D 20:01:02.519395] openib_check_cq: completed recv wr_id 15e73710. [D 20:01:02.519424] openib_post_rr: da13:3346 bh 23 wr_id 15e73710. [D 20:01:02.519450] openib_check_cq: completed recv wr_id 15e73728. [D 20:01:02.519476] openib_post_rr: da13:3346 bh 24 wr_id 15e73728. [D 20:01:02.519503] openib_check_cq: completed recv wr_id 15e73740. [D 20:01:02.519530] openib_post_rr: da13:3346 bh 25 wr_id 15e73740. [D 20:01:02.519562] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.519588] ========================================================= [D 20:01:02.519613] PINT_process_request [D 20:01:02.519637] tiling 15 copies [D 20:01:02.519660] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.519687] lvl 0 el 0 blk 0 by 448790528 [D 20:01:02.519712] to 448790528 ta 436207616 fi 469762048 [D 20:01:02.519737] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:02.519762] basic type or contiguous data [D 20:01:02.519785] Request disp [D 20:01:02.519808] contig_offset = 448790528 contig_size = 54525952 lvl_flag = 1 [D 20:01:02.519833] calling distribute [D 20:01:02.519857] PINT_distribute [D 20:01:02.519881] of 448790528 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.519907] begin iteration loff: 448790528 [D 20:01:02.519932] segment exceeds byte limit [D 20:01:02.519956] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:02.519981] **********CALL***PROCESS********* [D 20:01:02.520005] segment of 12582912 sz 2097152 [D 20:01:02.520029] PINT_process_request [D 20:01:02.520052] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.520079] lvl 0 el 0 blk 0 by 12582912 [D 20:01:02.520103] to 12582912 ta 12582912 fi 14680064 [D 20:01:02.520128] basic type or contiguous data [D 20:01:02.520152] Request disp [D 20:01:02.520175] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:02.520204] process a segment [D 20:01:02.520232] of 12582912 sz 2097152 [D 20:01:02.520259] add a segment 0 [D 20:01:02.520287] retval = 2097152 [D 20:01:02.520314] exiting distribute returned less than expected [D 20:01:02.520338] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:02.520364] *****RETURN***FROM***PROCESS***** [D 20:01:02.520388] end iteration [D 20:01:02.520412] done with segments or bytes [D 20:01:02.520436] of 450887680 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.520461] next loff: 450887680 next poff: 450887680 [D 20:01:02.520485] return value 2097152 [D 20:01:02.520509] finished [D 20:01:02.520534] retval = 2097152 [D 20:01:02.520557] exiting distribute returned less than expected [D 20:01:02.520581] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 450887680 fo 469762048 eof 0 [D 20:01:02.520607] ========================================================= [D 20:01:02.520631] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.520657] element 0: offset: 0x89037000, size: 2097152 [D 20:01:02.520686] post_sr: da13:3346 bh 27 len 48 wr_id 15e72e08. [D 20:01:02.520713] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.520739] ========================================================= [D 20:01:02.520763] PINT_process_request [D 20:01:02.520787] tiling 2 copies [D 20:01:02.520811] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.520837] lvl 0 el 0 blk 0 by 10485760 [D 20:01:02.520862] to 10485760 ta 0 fi 33554432 [D 20:01:02.520887] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:02.520914] basic type or contiguous data [D 20:01:02.520938] Request disp [D 20:01:02.520962] contig_offset = 10485760 contig_size = 56623104 lvl_flag = 1 [D 20:01:02.520987] calling distribute [D 20:01:02.521012] PINT_distribute [D 20:01:02.521035] of 10485760 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.521061] begin iteration loff: 10485760 [D 20:01:02.521086] segment exceeds byte limit [D 20:01:02.521110] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:02.521135] **********CALL***PROCESS********* [D 20:01:02.521159] segment of 10485760 sz 2097152 [D 20:01:02.521183] PINT_process_request [D 20:01:02.521207] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.521233] lvl 0 el 0 blk 0 by 10485760 [D 20:01:02.521258] to 10485760 ta 10485760 fi 12582912 [D 20:01:02.521282] basic type or contiguous data [D 20:01:02.521306] Request disp [D 20:01:02.521329] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:02.521354] process a segment [D 20:01:02.521378] of 10485760 sz 2097152 [D 20:01:02.521403] add a segment 0 [D 20:01:02.521427] retval = 2097152 [D 20:01:02.521450] exiting distribute returned less than expected [D 20:01:02.521474] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:02.521500] *****RETURN***FROM***PROCESS***** [D 20:01:02.521524] end iteration [D 20:01:02.521548] done with segments or bytes [D 20:01:02.521572] of 12582912 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.521597] next loff: 12582912 next poff: 12582912 [D 20:01:02.521621] return value 2097152 [D 20:01:02.521646] finished [D 20:01:02.521670] retval = 2097152 [D 20:01:02.521694] exiting distribute returned less than expected [D 20:01:02.521718] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 12582912 fo 33554432 eof 0 [D 20:01:02.521743] ========================================================= [D 20:01:02.521767] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.521792] element 0: offset: 0xa2e51000, size: 2097152 [D 20:01:02.521818] openib_check_cq: completed send wr_id 15e72e08. [D 20:01:02.521847] post_sr: da13:3346 bh 28 len 48 wr_id 15e72e20. [D 20:01:02.521874] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.521900] ========================================================= [D 20:01:02.521924] PINT_process_request [D 20:01:02.521948] tiling 4 copies [D 20:01:02.521972] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.521998] lvl 0 el 0 blk 0 by 71303168 [D 20:01:02.522023] to 71303168 ta 67108864 fi 100663296 [D 20:01:02.522047] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:02.522072] basic type or contiguous data [D 20:01:02.522096] Request disp [D 20:01:02.522120] contig_offset = 71303168 contig_size = 62914560 lvl_flag = 1 [D 20:01:02.522145] calling distribute [D 20:01:02.522169] PINT_distribute [D 20:01:02.522195] of 71303168 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.522222] begin iteration loff: 71303168 [D 20:01:02.522248] segment exceeds byte limit [D 20:01:02.522272] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:02.522296] **********CALL***PROCESS********* [D 20:01:02.522320] segment of 4194304 sz 2097152 [D 20:01:02.522345] PINT_process_request [D 20:01:02.522369] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.522395] lvl 0 el 0 blk 0 by 4194304 [D 20:01:02.522419] to 4194304 ta 4194304 fi 6291456 [D 20:01:02.522444] basic type or contiguous data [D 20:01:02.522468] Request disp [D 20:01:02.522492] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:02.522517] process a segment [D 20:01:02.522541] of 4194304 sz 2097152 [D 20:01:02.522564] add a segment 0 [D 20:01:02.522588] retval = 2097152 [D 20:01:02.522612] exiting distribute returned less than expected [D 20:01:02.522637] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:02.522665] *****RETURN***FROM***PROCESS***** [D 20:01:02.522689] end iteration [D 20:01:02.522713] done with segments or bytes [D 20:01:02.522737] of 73400320 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.522763] next loff: 73400320 next poff: 73400320 [D 20:01:02.522788] return value 2097152 [D 20:01:02.522812] finished [D 20:01:02.522835] retval = 2097152 [D 20:01:02.522859] exiting distribute returned less than expected [D 20:01:02.522883] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 73400320 fo 100663296 eof 0 [D 20:01:02.522910] ========================================================= [D 20:01:02.522933] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.522959] element 0: offset: 0x9e84d000, size: 2097152 [D 20:01:02.522985] openib_check_cq: completed send wr_id 15e72e20. [D 20:01:02.523015] post_sr: da13:3346 bh 29 len 48 wr_id 15e72e38. [D 20:01:02.523050] openib_check_cq: completed send wr_id 15e72e38. [D 20:01:02.527116] PINT_client_state_machine_test id 22 [D 20:01:02.532843] openib_check_cq: completed recv wr_id 15e73758. [D 20:01:02.532873] openib_post_rr: da13:3346 bh 26 wr_id 15e73758. [D 20:01:02.532904] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.532930] ========================================================= [D 20:01:02.532954] PINT_process_request [D 20:01:02.532979] tiling 3 copies [D 20:01:02.533002] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.533028] lvl 0 el 0 blk 0 by 41943040 [D 20:01:02.533054] to 41943040 ta 33554432 fi 67108864 [D 20:01:02.533079] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:02.533102] basic type or contiguous data [D 20:01:02.533127] Request disp [D 20:01:02.533151] contig_offset = 41943040 contig_size = 58720256 lvl_flag = 1 [D 20:01:02.533176] calling distribute [D 20:01:02.533200] PINT_distribute [D 20:01:02.533224] of 41943040 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.533250] begin iteration loff: 41943040 [D 20:01:02.533274] segment exceeds byte limit [D 20:01:02.533298] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:02.533323] **********CALL***PROCESS********* [D 20:01:02.533347] segment of 8388608 sz 2097152 [D 20:01:02.533371] PINT_process_request [D 20:01:02.533395] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.533422] lvl 0 el 0 blk 0 by 8388608 [D 20:01:02.533446] to 8388608 ta 8388608 fi 10485760 [D 20:01:02.533471] basic type or contiguous data [D 20:01:02.533494] Request disp [D 20:01:02.533518] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:02.533543] process a segment [D 20:01:02.533567] of 8388608 sz 2097152 [D 20:01:02.533591] add a segment 0 [D 20:01:02.533615] retval = 2097152 [D 20:01:02.533638] exiting distribute returned less than expected [D 20:01:02.533662] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:02.533688] *****RETURN***FROM***PROCESS***** [D 20:01:02.533713] end iteration [D 20:01:02.533737] done with segments or bytes [D 20:01:02.533761] of 44040192 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.533785] next loff: 44040192 next poff: 44040192 [D 20:01:02.533810] return value 2097152 [D 20:01:02.533835] finished [D 20:01:02.533859] retval = 2097152 [D 20:01:02.533883] exiting distribute returned less than expected [D 20:01:02.533906] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 44040192 fo 67108864 eof 0 [D 20:01:02.533933] ========================================================= [D 20:01:02.533957] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.533983] element 0: offset: 0xa0c4f000, size: 2097152 [D 20:01:02.534012] post_sr: da13:3346 bh 30 len 48 wr_id 15e72e50. [D 20:01:02.534084] openib_check_cq: completed send wr_id 15e72e50. [D 20:01:02.537147] PINT_client_state_machine_test id 22 [D 20:01:02.539552] openib_check_cq: completed recv wr_id 15e73770. [D 20:01:02.539580] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaabb051010 rkey c8009800 nic_wr_credit 1. [D 20:01:02.539607] openib_post_rr: da13:3346 bh 27 wr_id 15e73770. [D 20:01:02.542074] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 0. [D 20:01:02.542101] post_sr: da13:3346 bh 31 len 16 wr_id 15e72e68. [D 20:01:02.542140] openib_check_cq: completed send wr_id 15e72e68. [D 20:01:02.542171] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:02.542200] ========================================================= [D 20:01:02.542224] PINT_process_request [D 20:01:02.542248] tiling 16 copies [D 20:01:02.542273] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.542299] lvl 0 el 0 blk 0 by 497025024 [D 20:01:02.542324] to 497025024 ta 469762048 fi 503316480 [D 20:01:02.542348] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:02.542373] basic type or contiguous data [D 20:01:02.542397] Request disp [D 20:01:02.542421] contig_offset = 497025024 contig_size = 39845888 lvl_flag = 1 [D 20:01:02.542446] calling distribute [D 20:01:02.542469] PINT_distribute [D 20:01:02.542493] of 497025024 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 497025024 exfl 1 [D 20:01:02.542519] begin iteration loff: 497025024 [D 20:01:02.542544] segment exceeds byte limit [D 20:01:02.542568] file being extended [D 20:01:02.542592] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:02.542616] **********CALL***PROCESS********* [D 20:01:02.542640] segment of 27262976 sz 2097152 [D 20:01:02.542665] PINT_process_request [D 20:01:02.542689] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.542715] lvl 0 el 0 blk 0 by 27262976 [D 20:01:02.542739] to 27262976 ta 27262976 fi 29360128 [D 20:01:02.542764] basic type or contiguous data [D 20:01:02.542788] Request disp [D 20:01:02.542812] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:02.542837] process a segment [D 20:01:02.542860] of 27262976 sz 2097152 [D 20:01:02.542885] add a segment 0 [D 20:01:02.542909] retval = 2097152 [D 20:01:02.542933] exiting distribute returned less than expected [D 20:01:02.542957] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:02.542983] *****RETURN***FROM***PROCESS***** [D 20:01:02.543007] end iteration [D 20:01:02.543031] done with segments or bytes [D 20:01:02.543054] of 499122176 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.543080] next loff: 499122176 next poff: 499122176 [D 20:01:02.543105] return value 2097152 [D 20:01:02.543128] finished [D 20:01:02.543152] retval = 2097152 [D 20:01:02.543176] exiting distribute returned less than expected [D 20:01:02.543200] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 499122176 fo 503316480 eof 0 [D 20:01:02.543226] ========================================================= [D 20:01:02.543251] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:02.543275] element 0: offset: 0xa7e55000, size: 2097152 [D 20:01:02.543302] post_sr: da13:3346 bh 32 len 32 wr_id 15e72e80. [D 20:01:02.543338] openib_check_cq: completed send wr_id 15e72e80. [D 20:01:02.545725] openib_check_cq: completed recv wr_id 15e73788. [D 20:01:02.545754] openib_post_rr: da13:3346 bh 28 wr_id 15e73788. [D 20:01:02.545780] openib_check_cq: completed recv wr_id 15e737a0. [D 20:01:02.545806] openib_post_rr: da13:3346 bh 29 wr_id 15e737a0. [D 20:01:02.545833] openib_check_cq: completed recv wr_id 15e737b8. [D 20:01:02.545860] openib_post_rr: da13:3346 bh 30 wr_id 15e737b8. [D 20:01:02.545892] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.545918] ========================================================= [D 20:01:02.545942] PINT_process_request [D 20:01:02.545966] tiling 15 copies [D 20:01:02.545989] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.546018] lvl 0 el 0 blk 0 by 450887680 [D 20:01:02.546044] to 450887680 ta 436207616 fi 469762048 [D 20:01:02.546069] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:02.546093] basic type or contiguous data [D 20:01:02.546117] Request disp [D 20:01:02.546141] contig_offset = 450887680 contig_size = 52428800 lvl_flag = 1 [D 20:01:02.546166] calling distribute [D 20:01:02.546193] PINT_distribute [D 20:01:02.546216] of 450887680 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.546243] begin iteration loff: 450887680 [D 20:01:02.546269] segment exceeds byte limit [D 20:01:02.546293] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:02.546318] **********CALL***PROCESS********* [D 20:01:02.546341] segment of 14680064 sz 2097152 [D 20:01:02.546365] PINT_process_request [D 20:01:02.546389] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.546416] lvl 0 el 0 blk 0 by 14680064 [D 20:01:02.546441] to 14680064 ta 14680064 fi 16777216 [D 20:01:02.546464] basic type or contiguous data [D 20:01:02.546488] Request disp [D 20:01:02.546512] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:02.546537] process a segment [D 20:01:02.546561] of 14680064 sz 2097152 [D 20:01:02.546585] add a segment 0 [D 20:01:02.546609] retval = 2097152 [D 20:01:02.546633] exiting distribute returned less than expected [D 20:01:02.546657] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:02.546683] *****RETURN***FROM***PROCESS***** [D 20:01:02.546707] end iteration [D 20:01:02.546730] done with segments or bytes [D 20:01:02.546754] of 452984832 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.546780] next loff: 452984832 next poff: 452984832 [D 20:01:02.546805] return value 2097152 [D 20:01:02.546829] finished [D 20:01:02.546853] retval = 2097152 [D 20:01:02.546876] exiting distribute returned less than expected [D 20:01:02.546900] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 452984832 fo 469762048 eof 0 [D 20:01:02.546927] ========================================================= [D 20:01:02.546951] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.546976] element 0: offset: 0x89237000, size: 2097152 [D 20:01:02.547005] post_sr: da13:3346 bh 33 len 48 wr_id 15e72e98. [D 20:01:02.547032] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.547058] ========================================================= [D 20:01:02.547082] PINT_process_request [D 20:01:02.547106] tiling 2 copies [D 20:01:02.547130] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.547157] lvl 0 el 0 blk 0 by 12582912 [D 20:01:02.547182] to 12582912 ta 0 fi 33554432 [D 20:01:02.547206] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:02.547231] basic type or contiguous data [D 20:01:02.547255] Request disp [D 20:01:02.547279] contig_offset = 12582912 contig_size = 54525952 lvl_flag = 1 [D 20:01:02.547304] calling distribute [D 20:01:02.547327] PINT_distribute [D 20:01:02.547351] of 12582912 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.547378] begin iteration loff: 12582912 [D 20:01:02.547402] segment exceeds byte limit [D 20:01:02.547426] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:02.547451] **********CALL***PROCESS********* [D 20:01:02.547474] segment of 12582912 sz 2097152 [D 20:01:02.547499] PINT_process_request [D 20:01:02.547523] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.547549] lvl 0 el 0 blk 0 by 12582912 [D 20:01:02.547574] to 12582912 ta 12582912 fi 14680064 [D 20:01:02.547600] basic type or contiguous data [D 20:01:02.547624] Request disp [D 20:01:02.547648] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:02.547673] process a segment [D 20:01:02.547696] of 12582912 sz 2097152 [D 20:01:02.547719] add a segment 0 [D 20:01:02.547748] retval = 2097152 [D 20:01:02.547772] exiting distribute returned less than expected [D 20:01:02.547796] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:02.547822] *****RETURN***FROM***PROCESS***** [D 20:01:02.547846] end iteration [D 20:01:02.547870] done with segments or bytes [D 20:01:02.547894] of 14680064 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.547920] next loff: 14680064 next poff: 14680064 [D 20:01:02.547945] return value 2097152 [D 20:01:02.547968] finished [D 20:01:02.547991] retval = 2097152 [D 20:01:02.548015] exiting distribute returned less than expected [D 20:01:02.548039] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 14680064 fo 33554432 eof 0 [D 20:01:02.548066] ========================================================= [D 20:01:02.548089] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.548115] element 0: offset: 0xa3051000, size: 2097152 [D 20:01:02.548141] openib_check_cq: completed send wr_id 15e72e98. [D 20:01:02.548171] post_sr: da13:3346 bh 34 len 48 wr_id 15e72eb0. [D 20:01:02.548200] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.548230] ========================================================= [D 20:01:02.548257] PINT_process_request [D 20:01:02.548285] tiling 4 copies [D 20:01:02.548312] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.548338] lvl 0 el 0 blk 0 by 73400320 [D 20:01:02.548363] to 73400320 ta 67108864 fi 100663296 [D 20:01:02.548388] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:02.548413] basic type or contiguous data [D 20:01:02.548437] Request disp [D 20:01:02.548460] contig_offset = 73400320 contig_size = 60817408 lvl_flag = 1 [D 20:01:02.548486] calling distribute [D 20:01:02.548509] PINT_distribute [D 20:01:02.548533] of 73400320 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.548560] begin iteration loff: 73400320 [D 20:01:02.548584] segment exceeds byte limit [D 20:01:02.548608] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:02.548633] **********CALL***PROCESS********* [D 20:01:02.548657] segment of 6291456 sz 2097152 [D 20:01:02.548682] PINT_process_request [D 20:01:02.548705] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.548731] lvl 0 el 0 blk 0 by 6291456 [D 20:01:02.548756] to 6291456 ta 6291456 fi 8388608 [D 20:01:02.548780] basic type or contiguous data [D 20:01:02.548804] Request disp [D 20:01:02.548828] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:02.548852] process a segment [D 20:01:02.548876] of 6291456 sz 2097152 [D 20:01:02.548900] add a segment 0 [D 20:01:02.548924] retval = 2097152 [D 20:01:02.548949] exiting distribute returned less than expected [D 20:01:02.548972] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:02.548997] *****RETURN***FROM***PROCESS***** [D 20:01:02.549022] end iteration [D 20:01:02.549046] done with segments or bytes [D 20:01:02.549070] of 75497472 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.549095] next loff: 75497472 next poff: 75497472 [D 20:01:02.549119] return value 2097152 [D 20:01:02.549143] finished [D 20:01:02.549167] retval = 2097152 [D 20:01:02.549191] exiting distribute returned less than expected [D 20:01:02.549215] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 75497472 fo 100663296 eof 0 [D 20:01:02.549242] ========================================================= [D 20:01:02.549265] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.549291] element 0: offset: 0x9ea4d000, size: 2097152 [D 20:01:02.549317] openib_check_cq: completed send wr_id 15e72eb0. [D 20:01:02.549346] post_sr: da13:3346 bh 35 len 48 wr_id 15e72ec8. [D 20:01:02.549375] PINT_client_state_machine_test id 22 [D 20:01:02.549403] openib_check_cq: completed send wr_id 15e72ec8. [D 20:01:02.559402] PINT_client_state_machine_test id 22 [D 20:01:02.569437] PINT_client_state_machine_test id 22 [D 20:01:02.579466] PINT_client_state_machine_test id 22 [D 20:01:02.589499] PINT_client_state_machine_test id 22 -[D 20:01:02.598988] openib_check_cq: completed recv wr_id 15e737d0. [D 20:01:02.599022] openib_post_rr: da13:3346 bh 31 wr_id 15e737d0. [D 20:01:02.599055] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.599081] ========================================================= [D 20:01:02.599105] PINT_process_request [D 20:01:02.599129] tiling 3 copies [D 20:01:02.599153] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.599180] lvl 0 el 0 blk 0 by 44040192 [D 20:01:02.599205] to 44040192 ta 33554432 fi 67108864 [D 20:01:02.599229] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:02.599254] basic type or contiguous data [D 20:01:02.599278] Request disp [D 20:01:02.599302] contig_offset = 44040192 contig_size = 56623104 lvl_flag = 1 [D 20:01:02.599328] calling distribute [D 20:01:02.599351] PINT_distribute [D 20:01:02.599375] of 44040192 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.599402] begin iteration loff: 44040192 [D 20:01:02.599427] segment exceeds byte limit [D 20:01:02.599451] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:02.599475] **********CALL***PROCESS********* [D 20:01:02.599498] segment of 10485760 sz 2097152 [D 20:01:02.599523] PINT_process_request [D 20:01:02.599547] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.599573] lvl 0 el 0 blk 0 by 10485760 [D 20:01:02.599597] to 10485760 ta 10485760 fi 12582912 [D 20:01:02.599622] basic type or contiguous data [D 20:01:02.599645] Request disp [D 20:01:02.599669] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:02.599694] process a segment [D 20:01:02.599718] of 10485760 sz 2097152 [D 20:01:02.599741] add a segment 0 [D 20:01:02.599765] retval = 2097152 [D 20:01:02.599789] exiting distribute returned less than expected [D 20:01:02.599814] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:02.599840] *****RETURN***FROM***PROCESS***** [D 20:01:02.599864] end iteration [D 20:01:02.599887] done with segments or bytes [D 20:01:02.599911] of 46137344 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.599937] next loff: 46137344 next poff: 46137344 [D 20:01:02.599961] return value 2097152 [D 20:01:02.599986] finished [D 20:01:02.600009] retval = 2097152 [D 20:01:02.600033] exiting distribute returned less than expected [D 20:01:02.600057] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 46137344 fo 67108864 eof 0 [D 20:01:02.600083] ========================================================= [D 20:01:02.600108] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.600132] element 0: offset: 0xa0e4f000, size: 2097152 [D 20:01:02.600163] post_sr: da13:3346 bh 36 len 48 wr_id 15e72ee0. [D 20:01:02.600192] PINT_client_state_machine_test id 22 [D 20:01:02.600231] openib_check_cq: completed send wr_id 15e72ee0. [D 20:01:02.607906] openib_check_cq: completed recv wr_id 15e737e8. [D 20:01:02.607933] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaabb252010 rkey c8009900 nic_wr_credit 1. [D 20:01:02.607960] openib_post_rr: da13:3346 bh 32 wr_id 15e737e8. [D 20:01:02.607986] openib_check_cq: completed recv wr_id 15e73800. [D 20:01:02.608013] openib_post_rr: da13:3346 bh 33 wr_id 15e73800. [D 20:01:02.608039] openib_check_cq: completed recv wr_id 15e73818. [D 20:01:02.608066] openib_post_rr: da13:3346 bh 34 wr_id 15e73818. [D 20:01:02.608092] openib_check_cq: completed recv wr_id 15e73830. [D 20:01:02.608119] openib_post_rr: da13:3346 bh 35 wr_id 15e73830. [D 20:01:02.608145] openib_check_cq: completed recv wr_id 15e73848. [D 20:01:02.608172] openib_post_rr: da13:3346 bh 36 wr_id 15e73848. [D 20:01:02.608204] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.608233] ========================================================= [D 20:01:02.608258] PINT_process_request [D 20:01:02.608282] tiling 15 copies [D 20:01:02.608305] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.608331] lvl 0 el 0 blk 0 by 452984832 [D 20:01:02.608356] to 452984832 ta 436207616 fi 469762048 [D 20:01:02.608381] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:02.608406] basic type or contiguous data [D 20:01:02.608429] Request disp [D 20:01:02.608453] contig_offset = 452984832 contig_size = 50331648 lvl_flag = 1 [D 20:01:02.608478] calling distribute [D 20:01:02.608502] PINT_distribute [D 20:01:02.608525] of 452984832 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.608551] begin iteration loff: 452984832 [D 20:01:02.608576] segment exceeds byte limit [D 20:01:02.608600] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:02.608625] **********CALL***PROCESS********* [D 20:01:02.608649] segment of 16777216 sz 2097152 [D 20:01:02.608673] PINT_process_request [D 20:01:02.608699] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.608730] lvl 0 el 0 blk 0 by 16777216 [D 20:01:02.608758] to 16777216 ta 16777216 fi 18874368 [D 20:01:02.608787] basic type or contiguous data [D 20:01:02.608813] Request disp [D 20:01:02.608837] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:02.608861] process a segment [D 20:01:02.608885] of 16777216 sz 2097152 [D 20:01:02.608910] add a segment 0 [D 20:01:02.608934] retval = 2097152 [D 20:01:02.608957] exiting distribute returned less than expected [D 20:01:02.608981] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:02.609007] *****RETURN***FROM***PROCESS***** [D 20:01:02.609031] end iteration [D 20:01:02.609055] done with segments or bytes [D 20:01:02.609079] of 455081984 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.609104] next loff: 455081984 next poff: 455081984 [D 20:01:02.609128] return value 2097152 [D 20:01:02.609153] finished [D 20:01:02.609176] retval = 2097152 [D 20:01:02.609200] exiting distribute returned less than expected [D 20:01:02.609224] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 455081984 fo 469762048 eof 0 [D 20:01:02.609250] ========================================================= [D 20:01:02.609274] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.609299] element 0: offset: 0x89437000, size: 2097152 [D 20:01:02.609329] post_sr: da13:3346 bh 37 len 48 wr_id 15e72ef8. [D 20:01:02.609355] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.609380] ========================================================= [D 20:01:02.609405] PINT_process_request [D 20:01:02.609429] tiling 2 copies [D 20:01:02.609453] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.609479] lvl 0 el 0 blk 0 by 14680064 [D 20:01:02.609503] to 14680064 ta 0 fi 33554432 [D 20:01:02.609528] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:02.609553] basic type or contiguous data [D 20:01:02.609577] Request disp [D 20:01:02.609600] contig_offset = 14680064 contig_size = 52428800 lvl_flag = 1 [D 20:01:02.609625] calling distribute [D 20:01:02.609648] PINT_distribute [D 20:01:02.609672] of 14680064 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.609699] begin iteration loff: 14680064 [D 20:01:02.609724] segment exceeds byte limit [D 20:01:02.609747] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:02.609771] **********CALL***PROCESS********* [D 20:01:02.609795] segment of 14680064 sz 2097152 [D 20:01:02.609820] PINT_process_request [D 20:01:02.609844] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.609869] lvl 0 el 0 blk 0 by 14680064 [D 20:01:02.609894] to 14680064 ta 14680064 fi 16777216 [D 20:01:02.609918] basic type or contiguous data [D 20:01:02.609943] Request disp [D 20:01:02.609969] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:02.609993] process a segment [D 20:01:02.610017] of 14680064 sz 2097152 [D 20:01:02.610041] add a segment 0 [D 20:01:02.610065] retval = 2097152 [D 20:01:02.610089] exiting distribute returned less than expected [D 20:01:02.610113] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:02.610138] *****RETURN***FROM***PROCESS***** [D 20:01:02.610163] end iteration [D 20:01:02.610186] done with segments or bytes [D 20:01:02.610210] of 16777216 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.610236] next loff: 16777216 next poff: 16777216 [D 20:01:02.610260] return value 2097152 [D 20:01:02.610284] finished [D 20:01:02.610308] retval = 2097152 [D 20:01:02.610332] exiting distribute returned less than expected [D 20:01:02.610356] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 16777216 fo 33554432 eof 0 [D 20:01:02.610382] ========================================================= [D 20:01:02.610406] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.610431] element 0: offset: 0xa3251000, size: 2097152 [D 20:01:02.610457] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 0. [D 20:01:02.610484] post_sr: da13:3346 bh 38 len 16 wr_id 15e72f10. [D 20:01:02.610509] openib_check_cq: completed send wr_id 15e72ef8. [D 20:01:02.610539] post_sr: da13:3346 bh 39 len 48 wr_id 15e72f28. [D 20:01:02.610566] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.610590] ========================================================= [D 20:01:02.610615] PINT_process_request [D 20:01:02.610638] tiling 3 copies [D 20:01:02.610663] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.610689] lvl 0 el 0 blk 0 by 46137344 [D 20:01:02.610716] to 46137344 ta 33554432 fi 67108864 [D 20:01:02.610741] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:02.610766] basic type or contiguous data [D 20:01:02.610789] Request disp [D 20:01:02.610813] contig_offset = 46137344 contig_size = 54525952 lvl_flag = 1 [D 20:01:02.610837] calling distribute [D 20:01:02.610861] PINT_distribute [D 20:01:02.610885] of 46137344 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.610911] begin iteration loff: 46137344 [D 20:01:02.610936] segment exceeds byte limit [D 20:01:02.610960] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:02.610984] **********CALL***PROCESS********* [D 20:01:02.611008] segment of 12582912 sz 2097152 [D 20:01:02.611033] PINT_process_request [D 20:01:02.611056] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.611082] lvl 0 el 0 blk 0 by 12582912 [D 20:01:02.611106] to 12582912 ta 12582912 fi 14680064 [D 20:01:02.611131] basic type or contiguous data [D 20:01:02.611155] Request disp [D 20:01:02.611179] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:02.611204] process a segment [D 20:01:02.611227] of 12582912 sz 2097152 [D 20:01:02.611251] add a segment 0 [D 20:01:02.611275] retval = 2097152 [D 20:01:02.611299] exiting distribute returned less than expected [D 20:01:02.611323] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:02.611349] *****RETURN***FROM***PROCESS***** [D 20:01:02.611372] end iteration [D 20:01:02.611396] done with segments or bytes [D 20:01:02.611420] of 48234496 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.611446] next loff: 48234496 next poff: 48234496 [D 20:01:02.611470] return value 2097152 [D 20:01:02.611495] finished [D 20:01:02.611517] retval = 2097152 [D 20:01:02.611541] exiting distribute returned less than expected [D 20:01:02.611565] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 48234496 fo 67108864 eof 0 [D 20:01:02.611592] ========================================================= [D 20:01:02.611616] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.611643] element 0: offset: 0xa104f000, size: 2097152 [D 20:01:02.611670] openib_check_cq: completed send wr_id 15e72f10. [D 20:01:02.611696] openib_check_cq: completed send wr_id 15e72f28. [D 20:01:02.611725] post_sr: da13:3346 bh 40 len 48 wr_id 15e72f40. [D 20:01:02.611752] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.611777] ========================================================= [D 20:01:02.611802] PINT_process_request [D 20:01:02.611825] tiling 4 copies [D 20:01:02.611849] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.611875] lvl 0 el 0 blk 0 by 75497472 [D 20:01:02.611900] to 75497472 ta 67108864 fi 100663296 [D 20:01:02.611925] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:02.611949] basic type or contiguous data [D 20:01:02.611973] Request disp [D 20:01:02.611997] contig_offset = 75497472 contig_size = 58720256 lvl_flag = 1 [D 20:01:02.612022] calling distribute [D 20:01:02.612046] PINT_distribute [D 20:01:02.612070] of 75497472 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.612095] begin iteration loff: 75497472 [D 20:01:02.612120] segment exceeds byte limit [D 20:01:02.612144] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:02.612169] **********CALL***PROCESS********* [D 20:01:02.612193] segment of 8388608 sz 2097152 [D 20:01:02.612217] PINT_process_request [D 20:01:02.612241] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.612267] lvl 0 el 0 blk 0 by 8388608 [D 20:01:02.612292] to 8388608 ta 8388608 fi 10485760 [D 20:01:02.612317] basic type or contiguous data [D 20:01:02.612340] Request disp [D 20:01:02.612364] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:02.612389] process a segment [D 20:01:02.612412] of 8388608 sz 2097152 [D 20:01:02.612437] add a segment 0 [D 20:01:02.612461] retval = 2097152 [D 20:01:02.612484] exiting distribute returned less than expected [D 20:01:02.612508] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:02.612534] *****RETURN***FROM***PROCESS***** [D 20:01:02.612558] end iteration [D 20:01:02.612582] done with segments or bytes [D 20:01:02.612606] of 77594624 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.612630] next loff: 77594624 next poff: 77594624 [D 20:01:02.612655] return value 2097152 [D 20:01:02.612679] finished [D 20:01:02.612707] retval = 2097152 [D 20:01:02.612734] exiting distribute returned less than expected [D 20:01:02.612762] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 77594624 fo 100663296 eof 0 [D 20:01:02.612792] ========================================================= [D 20:01:02.612820] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.612846] element 0: offset: 0x9ec4d000, size: 2097152 [D 20:01:02.612872] openib_check_cq: completed send wr_id 15e72f40. [D 20:01:02.612896] openib_check_cq: completed recv wr_id 15e73860. [D 20:01:02.612924] openib_post_rr: da13:3346 bh 37 wr_id 15e73860. [D 20:01:02.612951] openib_check_cq: completed recv wr_id 15e73878. [D 20:01:02.612977] openib_post_rr: da13:3346 bh 38 wr_id 15e73878. [D 20:01:02.613004] openib_check_cq: completed recv wr_id 15e73890. [D 20:01:02.613031] openib_post_rr: da13:3346 bh 39 wr_id 15e73890. [D 20:01:02.613061] post_sr: da13:3346 bh 41 len 48 wr_id 15e72f58. [D 20:01:02.613089] PINT_client_state_machine_test id 22 [D 20:01:02.613118] openib_check_cq: completed send wr_id 15e72f58. [D 20:01:02.613147] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:02.613173] ========================================================= [D 20:01:02.613198] PINT_process_request [D 20:01:02.613222] tiling 16 copies [D 20:01:02.613245] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.613271] lvl 0 el 0 blk 0 by 499122176 [D 20:01:02.613296] to 499122176 ta 469762048 fi 503316480 [D 20:01:02.613321] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.613348] basic type or contiguous data [D 20:01:02.613372] Request disp [D 20:01:02.613396] contig_offset = 499122176 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.613421] calling distribute [D 20:01:02.613445] PINT_distribute [D 20:01:02.613469] of 499122176 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 499122176 exfl 1 [D 20:01:02.613494] begin iteration loff: 499122176 [D 20:01:02.613519] segment exceeds byte limit [D 20:01:02.613543] file being extended [D 20:01:02.613567] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.613592] **********CALL***PROCESS********* [D 20:01:02.613615] segment of 29360128 sz 2097152 [D 20:01:02.613639] PINT_process_request [D 20:01:02.613663] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.613689] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.613714] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.613739] basic type or contiguous data [D 20:01:02.613762] Request disp [D 20:01:02.613786] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.613811] process a segment [D 20:01:02.613835] of 29360128 sz 2097152 [D 20:01:02.613859] add a segment 0 [D 20:01:02.613883] retval = 2097152 [D 20:01:02.613906] exiting distribute returned less than expected [D 20:01:02.613930] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.613957] *****RETURN***FROM***PROCESS***** [D 20:01:02.613981] end iteration [D 20:01:02.614005] done with segments or bytes [D 20:01:02.614027] of 501219328 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.614053] next loff: 501219328 next poff: 501219328 [D 20:01:02.614078] return value 2097152 [D 20:01:02.614102] finished [D 20:01:02.614126] retval = 2097152 [D 20:01:02.614150] exiting distribute returned less than expected [D 20:01:02.614173] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 501219328 fo 503316480 eof 0 [D 20:01:02.614199] ========================================================= [D 20:01:02.614224] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:02.614249] element 0: offset: 0xa8055000, size: 2097152 [D 20:01:02.614275] post_sr: da13:3346 bh 42 len 32 wr_id 15e72f70. [D 20:01:02.614311] openib_check_cq: completed send wr_id 15e72f70. [D 20:01:02.614337] openib_check_cq: completed recv wr_id 15e738a8. [D 20:01:02.614365] openib_post_rr: da13:3346 bh 40 wr_id 15e738a8. [D 20:01:02.622978] openib_check_cq: completed recv wr_id 15e738c0. [D 20:01:02.623006] openib_post_rr: da13:3346 bh 41 wr_id 15e738c0. [D 20:01:02.623037] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.623063] ========================================================= [D 20:01:02.623087] PINT_process_request [D 20:01:02.623111] tiling 15 copies [D 20:01:02.623135] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.623161] lvl 0 el 0 blk 0 by 455081984 [D 20:01:02.623186] to 455081984 ta 436207616 fi 469762048 [D 20:01:02.623210] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:02.623235] basic type or contiguous data [D 20:01:02.623258] Request disp [D 20:01:02.623282] contig_offset = 455081984 contig_size = 48234496 lvl_flag = 1 [D 20:01:02.623307] calling distribute [D 20:01:02.623331] PINT_distribute [D 20:01:02.623354] of 455081984 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.623380] begin iteration loff: 455081984 [D 20:01:02.623406] segment exceeds byte limit [D 20:01:02.623430] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:02.623454] **********CALL***PROCESS********* [D 20:01:02.623477] segment of 18874368 sz 2097152 [D 20:01:02.623502] PINT_process_request [D 20:01:02.623526] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.623552] lvl 0 el 0 blk 0 by 18874368 [D 20:01:02.623577] to 18874368 ta 18874368 fi 20971520 [D 20:01:02.623601] basic type or contiguous data [D 20:01:02.623624] Request disp [D 20:01:02.623651] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:02.623677] process a segment [D 20:01:02.623700] of 18874368 sz 2097152 [D 20:01:02.623724] add a segment 0 [D 20:01:02.623748] retval = 2097152 [D 20:01:02.623772] exiting distribute returned less than expected [D 20:01:02.623796] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:02.623822] *****RETURN***FROM***PROCESS***** [D 20:01:02.623847] end iteration [D 20:01:02.623870] done with segments or bytes [D 20:01:02.623893] of 457179136 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.623919] next loff: 457179136 next poff: 457179136 [D 20:01:02.623944] return value 2097152 [D 20:01:02.623968] finished [D 20:01:02.623991] retval = 2097152 [D 20:01:02.624015] exiting distribute returned less than expected [D 20:01:02.624039] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 457179136 fo 469762048 eof 0 [D 20:01:02.624065] ========================================================= [D 20:01:02.624089] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.624114] element 0: offset: 0x89637000, size: 2097152 [D 20:01:02.624143] post_sr: da13:3346 bh 43 len 48 wr_id 15e72f88. [D 20:01:02.624172] PINT_client_state_machine_test id 22 [D 20:01:02.624215] openib_check_cq: completed send wr_id 15e72f88. [D 20:01:02.625204] openib_check_cq: completed recv wr_id 15e738d8. [D 20:01:02.625230] openib_post_sr_rdmaw: ibv_post_send wr_id 15e88c88 to da13:3346 remote addr 2aaabb453010 rkey c8009a00 nic_wr_credit 1. [D 20:01:02.625257] openib_post_rr: da13:3346 bh 42 wr_id 15e738d8. [D 20:01:02.625283] openib_check_cq: completed recv wr_id 15e738f0. [D 20:01:02.625309] openib_post_rr: da13:3346 bh 43 wr_id 15e738f0. [D 20:01:02.625336] openib_check_cq: completed recv wr_id 15e73908. [D 20:01:02.625363] openib_post_rr: da13:3346 bh 44 wr_id 15e73908. [D 20:01:02.625394] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.625420] ========================================================= [D 20:01:02.625445] PINT_process_request [D 20:01:02.625468] tiling 2 copies [D 20:01:02.625493] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.625518] lvl 0 el 0 blk 0 by 16777216 [D 20:01:02.625543] to 16777216 ta 0 fi 33554432 [D 20:01:02.625568] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:02.625593] basic type or contiguous data [D 20:01:02.625617] Request disp [D 20:01:02.625640] contig_offset = 16777216 contig_size = 50331648 lvl_flag = 1 [D 20:01:02.625665] calling distribute [D 20:01:02.625689] PINT_distribute [D 20:01:02.625713] of 16777216 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.625739] begin iteration loff: 16777216 [D 20:01:02.625763] segment exceeds byte limit [D 20:01:02.625787] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:02.625812] **********CALL***PROCESS********* [D 20:01:02.625836] segment of 16777216 sz 2097152 [D 20:01:02.625861] PINT_process_request [D 20:01:02.625885] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.625910] lvl 0 el 0 blk 0 by 16777216 [D 20:01:02.625935] to 16777216 ta 16777216 fi 18874368 [D 20:01:02.625960] basic type or contiguous data [D 20:01:02.625984] Request disp [D 20:01:02.626008] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:02.626032] process a segment [D 20:01:02.626056] of 16777216 sz 2097152 [D 20:01:02.626080] add a segment 0 [D 20:01:02.626104] retval = 2097152 [D 20:01:02.626128] exiting distribute returned less than expected [D 20:01:02.626152] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:02.626177] *****RETURN***FROM***PROCESS***** [D 20:01:02.626202] end iteration [D 20:01:02.626225] done with segments or bytes [D 20:01:02.626249] of 18874368 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.626275] next loff: 18874368 next poff: 18874368 [D 20:01:02.626301] return value 2097152 [D 20:01:02.626326] finished [D 20:01:02.626350] retval = 2097152 [D 20:01:02.626374] exiting distribute returned less than expected [D 20:01:02.626398] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 18874368 fo 33554432 eof 0 [D 20:01:02.626423] ========================================================= [D 20:01:02.626447] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.626473] element 0: offset: 0xa3451000, size: 2097152 [D 20:01:02.626502] post_sr: da13:3346 bh 44 len 48 wr_id 15e72fa0. [D 20:01:02.626528] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.626554] ========================================================= [D 20:01:02.626578] PINT_process_request [D 20:01:02.626602] tiling 3 copies [D 20:01:02.626626] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.626652] lvl 0 el 0 blk 0 by 48234496 [D 20:01:02.626677] to 48234496 ta 33554432 fi 67108864 [D 20:01:02.626704] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:02.626729] basic type or contiguous data [D 20:01:02.626753] Request disp [D 20:01:02.626777] contig_offset = 48234496 contig_size = 52428800 lvl_flag = 1 [D 20:01:02.626802] calling distribute [D 20:01:02.626826] PINT_distribute [D 20:01:02.626850] of 48234496 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.626875] begin iteration loff: 48234496 [D 20:01:02.626900] segment exceeds byte limit [D 20:01:02.626924] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:02.626950] **********CALL***PROCESS********* [D 20:01:02.626973] segment of 14680064 sz 2097152 [D 20:01:02.626997] PINT_process_request [D 20:01:02.627021] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.627047] lvl 0 el 0 blk 0 by 14680064 [D 20:01:02.627072] to 14680064 ta 14680064 fi 16777216 [D 20:01:02.627097] basic type or contiguous data [D 20:01:02.627120] Request disp [D 20:01:02.627144] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:02.627169] process a segment [D 20:01:02.627193] of 14680064 sz 2097152 [D 20:01:02.627217] add a segment 0 [D 20:01:02.627241] retval = 2097152 [D 20:01:02.627265] exiting distribute returned less than expected [D 20:01:02.627289] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:02.627315] *****RETURN***FROM***PROCESS***** [D 20:01:02.627340] end iteration [D 20:01:02.627364] done with segments or bytes [D 20:01:02.627388] of 50331648 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.627413] next loff: 50331648 next poff: 50331648 [D 20:01:02.627437] return value 2097152 [D 20:01:02.627462] finished [D 20:01:02.627485] retval = 2097152 [D 20:01:02.627510] exiting distribute returned less than expected [D 20:01:02.627533] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 50331648 fo 67108864 eof 0 [D 20:01:02.627559] ========================================================= [D 20:01:02.627584] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.627609] element 0: offset: 0xa124f000, size: 2097152 [D 20:01:02.627638] post_sr: da13:3346 bh 45 len 48 wr_id 15e72fb8. [D 20:01:02.628061] openib_check_cq: completed rdma wr_id 15e88c88, nic_wr_credit 0. [D 20:01:02.628088] post_sr: da13:3346 bh 46 len 16 wr_id 15e72fd0. [D 20:01:02.628114] openib_check_cq: completed send wr_id 15e72fa0. [D 20:01:02.628139] openib_check_cq: completed send wr_id 15e72fb8. [D 20:01:02.628165] openib_check_cq: completed send wr_id 15e72fd0. [D 20:01:02.628194] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:02.628220] ========================================================= [D 20:01:02.628244] PINT_process_request [D 20:01:02.628269] tiling 16 copies [D 20:01:02.628293] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.628318] lvl 0 el 0 blk 0 by 501219328 [D 20:01:02.628343] to 501219328 ta 469762048 fi 503316480 [D 20:01:02.628371] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.628396] basic type or contiguous data [D 20:01:02.628419] Request disp [D 20:01:02.628443] contig_offset = 501219328 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.628468] calling distribute [D 20:01:02.628493] PINT_distribute [D 20:01:02.628516] of 501219328 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 501219328 exfl 1 [D 20:01:02.628542] begin iteration loff: 501219328 [D 20:01:02.628567] file being extended [D 20:01:02.628591] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.628616] **********CALL***PROCESS********* [D 20:01:02.628640] segment of 31457280 sz 2097152 [D 20:01:02.628664] PINT_process_request [D 20:01:02.628688] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.628719] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.628748] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.628776] basic type or contiguous data [D 20:01:02.628805] Request disp [D 20:01:02.628829] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.628853] process a segment [D 20:01:02.628876] of 31457280 sz 2097152 [D 20:01:02.628901] add a segment 0 [D 20:01:02.628925] retval = 2097152 [D 20:01:02.628949] return from level -1 [D 20:01:02.628973] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.628998] *****RETURN***FROM***PROCESS***** [D 20:01:02.629023] end iteration [D 20:01:02.629047] done with segments or bytes [D 20:01:02.629070] of 503316480 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.629096] next loff: 503316480 next poff: 503316480 [D 20:01:02.629120] (return value) 2097152 [D 20:01:02.629145] finished [D 20:01:02.629169] retval = 2097152 [D 20:01:02.629193] exiting distribute returned less than expected [D 20:01:02.629217] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 503316480 fo 503316480 eof 0 [D 20:01:02.629242] ========================================================= [D 20:01:02.629267] BMI_post_send_list: addr: 367403208, count: 1, total_size: 2097152, tag: 22 [D 20:01:02.629292] element 0: offset: 0xa8255000, size: 2097152 [D 20:01:02.629319] post_sr: da13:3346 bh 47 len 32 wr_id 15e72fe8. [D 20:01:02.629349] openib_check_cq: completed recv wr_id 15e73920. [D 20:01:02.629377] openib_post_rr: da13:3346 bh 45 wr_id 15e73920. [D 20:01:02.629403] openib_check_cq: completed recv wr_id 15e73938. [D 20:01:02.629430] openib_post_rr: da13:3346 bh 46 wr_id 15e73938. [D 20:01:02.629456] openib_check_cq: completed send wr_id 15e72fe8. [D 20:01:02.629486] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.629511] ========================================================= [D 20:01:02.629536] PINT_process_request [D 20:01:02.629560] tiling 4 copies [D 20:01:02.629584] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.629610] lvl 0 el 0 blk 0 by 77594624 [D 20:01:02.629634] to 77594624 ta 67108864 fi 100663296 [D 20:01:02.629659] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:02.629684] basic type or contiguous data [D 20:01:02.629708] Request disp [D 20:01:02.629732] contig_offset = 77594624 contig_size = 56623104 lvl_flag = 1 [D 20:01:02.629756] calling distribute [D 20:01:02.629780] PINT_distribute [D 20:01:02.629804] of 77594624 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.629831] begin iteration loff: 77594624 [D 20:01:02.629855] segment exceeds byte limit [D 20:01:02.629879] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:02.629903] **********CALL***PROCESS********* [D 20:01:02.629927] segment of 10485760 sz 2097152 [D 20:01:02.629952] PINT_process_request [D 20:01:02.629976] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.630002] lvl 0 el 0 blk 0 by 10485760 [D 20:01:02.630026] to 10485760 ta 10485760 fi 12582912 [D 20:01:02.630051] basic type or contiguous data [D 20:01:02.630075] Request disp [D 20:01:02.630099] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:02.630126] process a segment [D 20:01:02.630150] of 10485760 sz 2097152 [D 20:01:02.630175] add a segment 0 [D 20:01:02.630199] retval = 2097152 [D 20:01:02.630223] exiting distribute returned less than expected [D 20:01:02.630247] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:02.630273] *****RETURN***FROM***PROCESS***** [D 20:01:02.630297] end iteration [D 20:01:02.630321] done with segments or bytes [D 20:01:02.630345] of 79691776 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.630371] next loff: 79691776 next poff: 79691776 [D 20:01:02.630394] return value 2097152 [D 20:01:02.630419] finished [D 20:01:02.630443] retval = 2097152 [D 20:01:02.630466] exiting distribute returned less than expected [D 20:01:02.630491] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 79691776 fo 100663296 eof 0 [D 20:01:02.630517] ========================================================= [D 20:01:02.630540] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.630566] element 0: offset: 0x9ee4d000, size: 2097152 [D 20:01:02.630596] post_sr: da13:3346 bh 48 len 48 wr_id 15e73000. [D 20:01:02.630630] openib_check_cq: completed send wr_id 15e73000. [D 20:01:02.634202] PINT_client_state_machine_test id 22 [D 20:01:02.640987] openib_check_cq: completed recv wr_id 15e73950. [D 20:01:02.641016] openib_post_rr: da13:3346 bh 47 wr_id 15e73950. [D 20:01:02.641047] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.641073] ========================================================= [D 20:01:02.641097] PINT_process_request [D 20:01:02.641120] tiling 15 copies [D 20:01:02.641144] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.641170] lvl 0 el 0 blk 0 by 457179136 [D 20:01:02.641195] to 457179136 ta 436207616 fi 469762048 [D 20:01:02.641220] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:02.641244] basic type or contiguous data [D 20:01:02.641268] Request disp [D 20:01:02.641292] contig_offset = 457179136 contig_size = 46137344 lvl_flag = 1 [D 20:01:02.641317] calling distribute [D 20:01:02.641341] PINT_distribute [D 20:01:02.641365] of 457179136 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.641390] begin iteration loff: 457179136 [D 20:01:02.641415] segment exceeds byte limit [D 20:01:02.641439] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:02.641464] **********CALL***PROCESS********* [D 20:01:02.641488] segment of 20971520 sz 2097152 [D 20:01:02.641511] PINT_process_request [D 20:01:02.641535] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.641562] lvl 0 el 0 blk 0 by 20971520 [D 20:01:02.641587] to 20971520 ta 20971520 fi 23068672 [D 20:01:02.641612] basic type or contiguous data [D 20:01:02.641634] Request disp [D 20:01:02.641658] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:02.641683] process a segment [D 20:01:02.641707] of 20971520 sz 2097152 [D 20:01:02.641732] add a segment 0 [D 20:01:02.641756] retval = 2097152 [D 20:01:02.641778] exiting distribute returned less than expected [D 20:01:02.641802] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:02.641829] *****RETURN***FROM***PROCESS***** [D 20:01:02.641853] end iteration [D 20:01:02.641877] done with segments or bytes [D 20:01:02.641900] of 459276288 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.641925] next loff: 459276288 next poff: 459276288 [D 20:01:02.641950] return value 2097152 [D 20:01:02.641975] finished [D 20:01:02.641998] retval = 2097152 [D 20:01:02.642023] exiting distribute returned less than expected [D 20:01:02.642046] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 459276288 fo 469762048 eof 0 [D 20:01:02.642072] ========================================================= [D 20:01:02.642096] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.642124] element 0: offset: 0x89837000, size: 2097152 [D 20:01:02.642154] post_sr: da13:3346 bh 49 len 48 wr_id 15e73018. [D 20:01:02.642224] openib_check_cq: completed send wr_id 15e73018. [D 20:01:02.643212] openib_check_cq: completed recv wr_id 15e73968. [D 20:01:02.643240] openib_post_rr: da13:3346 bh 48 wr_id 15e73968. [D 20:01:02.643267] openib_check_cq: completed recv wr_id 15e73980. [D 20:01:02.643295] openib_post_rr: da13:3346 bh 49 wr_id 15e73980. [D 20:01:02.644230] PINT_client_state_machine_test id 22 [D 20:01:02.648244] openib_check_cq: completed recv wr_id 15e73998. [D 20:01:02.648273] openib_post_rr: da13:3346 bh 50 wr_id 15e73998. [D 20:01:02.648300] openib_check_cq: completed recv wr_id 15e739b0. [D 20:01:02.648326] openib_post_rr: da13:3346 bh 51 wr_id 15e739b0. [D 20:01:02.648358] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.648384] ========================================================= [D 20:01:02.648407] PINT_process_request [D 20:01:02.648431] tiling 2 copies [D 20:01:02.648456] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.648482] lvl 0 el 0 blk 0 by 18874368 [D 20:01:02.648507] to 18874368 ta 0 fi 33554432 [D 20:01:02.648531] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:02.648556] basic type or contiguous data [D 20:01:02.648579] Request disp [D 20:01:02.648603] contig_offset = 18874368 contig_size = 48234496 lvl_flag = 1 [D 20:01:02.648629] calling distribute [D 20:01:02.648653] PINT_distribute [D 20:01:02.648676] of 18874368 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.648706] begin iteration loff: 18874368 [D 20:01:02.648735] segment exceeds byte limit [D 20:01:02.648761] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:02.648790] **********CALL***PROCESS********* [D 20:01:02.648817] segment of 18874368 sz 2097152 [D 20:01:02.648841] PINT_process_request [D 20:01:02.648865] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.648891] lvl 0 el 0 blk 0 by 18874368 [D 20:01:02.648916] to 18874368 ta 18874368 fi 20971520 [D 20:01:02.648941] basic type or contiguous data [D 20:01:02.648964] Request disp [D 20:01:02.648988] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:02.649013] process a segment [D 20:01:02.649036] of 18874368 sz 2097152 [D 20:01:02.649061] add a segment 0 [D 20:01:02.649085] retval = 2097152 [D 20:01:02.649108] exiting distribute returned less than expected [D 20:01:02.649132] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:02.649158] *****RETURN***FROM***PROCESS***** [D 20:01:02.649183] end iteration [D 20:01:02.649206] done with segments or bytes [D 20:01:02.649230] of 20971520 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.649255] next loff: 20971520 next poff: 20971520 [D 20:01:02.649280] return value 2097152 [D 20:01:02.649304] finished [D 20:01:02.649328] retval = 2097152 [D 20:01:02.649352] exiting distribute returned less than expected [D 20:01:02.649375] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 20971520 fo 33554432 eof 0 [D 20:01:02.649401] ========================================================= [D 20:01:02.649425] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.649451] element 0: offset: 0xa3651000, size: 2097152 [D 20:01:02.649480] post_sr: da13:3346 bh 50 len 48 wr_id 15e73030. [D 20:01:02.649507] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.649532] ========================================================= [D 20:01:02.649557] PINT_process_request [D 20:01:02.649580] tiling 3 copies [D 20:01:02.649604] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.649630] lvl 0 el 0 blk 0 by 50331648 [D 20:01:02.649655] to 50331648 ta 33554432 fi 67108864 [D 20:01:02.649680] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:02.649707] basic type or contiguous data [D 20:01:02.649731] Request disp [D 20:01:02.649755] contig_offset = 50331648 contig_size = 50331648 lvl_flag = 1 [D 20:01:02.649780] calling distribute [D 20:01:02.649804] PINT_distribute [D 20:01:02.649827] of 50331648 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.649854] begin iteration loff: 50331648 [D 20:01:02.649879] segment exceeds byte limit [D 20:01:02.649903] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:02.649928] **********CALL***PROCESS********* [D 20:01:02.649951] segment of 16777216 sz 2097152 [D 20:01:02.649975] PINT_process_request [D 20:01:02.649999] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.650025] lvl 0 el 0 blk 0 by 16777216 [D 20:01:02.650050] to 16777216 ta 16777216 fi 18874368 [D 20:01:02.650074] basic type or contiguous data [D 20:01:02.650098] Request disp [D 20:01:02.650122] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:02.650147] process a segment [D 20:01:02.650171] of 16777216 sz 2097152 [D 20:01:02.650195] add a segment 0 [D 20:01:02.650218] retval = 2097152 [D 20:01:02.650242] exiting distribute returned less than expected [D 20:01:02.650266] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:02.650293] *****RETURN***FROM***PROCESS***** [D 20:01:02.650317] end iteration [D 20:01:02.650341] done with segments or bytes [D 20:01:02.650364] of 52428800 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.650390] next loff: 52428800 next poff: 52428800 [D 20:01:02.650415] return value 2097152 [D 20:01:02.650439] finished [D 20:01:02.650463] retval = 2097152 [D 20:01:02.650486] exiting distribute returned less than expected [D 20:01:02.650510] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 52428800 fo 67108864 eof 0 [D 20:01:02.650536] ========================================================= [D 20:01:02.650560] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.650586] element 0: offset: 0xa144f000, size: 2097152 [D 20:01:02.650611] openib_check_cq: completed send wr_id 15e73030. [D 20:01:02.650636] openib_check_cq: completed recv wr_id 15e739c8. [D 20:01:02.650664] openib_post_rr: da13:3346 bh 52 wr_id 15e739c8. [D 20:01:02.650691] openib_check_cq: completed recv wr_id 15e739e0. [D 20:01:02.650718] openib_post_sr_rdmaw: ibv_post_send wr_id 15ee24e0 to da13:3346 remote addr 2aaabc85d010 rkey 4800a400 nic_wr_credit 1. [D 20:01:02.650745] openib_post_rr: da13:3346 bh 53 wr_id 15e739e0. [D 20:01:02.650772] openib_check_cq: completed recv wr_id 15e739f8. [D 20:01:02.650798] openib_post_rr: da13:3346 bh 54 wr_id 15e739f8. [D 20:01:02.650828] post_sr: da13:3346 bh 51 len 48 wr_id 15e73048. [D 20:01:02.650860] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.650887] ========================================================= [D 20:01:02.650912] PINT_process_request [D 20:01:02.650936] tiling 4 copies [D 20:01:02.650959] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.650985] lvl 0 el 0 blk 0 by 79691776 [D 20:01:02.651010] to 79691776 ta 67108864 fi 100663296 [D 20:01:02.651036] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:02.651061] basic type or contiguous data [D 20:01:02.651084] Request disp [D 20:01:02.651107] contig_offset = 79691776 contig_size = 54525952 lvl_flag = 1 [D 20:01:02.651133] calling distribute [D 20:01:02.651157] PINT_distribute [D 20:01:02.651181] of 79691776 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.651206] begin iteration loff: 79691776 [D 20:01:02.651231] segment exceeds byte limit [D 20:01:02.651256] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:02.651280] **********CALL***PROCESS********* [D 20:01:02.651304] segment of 12582912 sz 2097152 [D 20:01:02.651329] PINT_process_request [D 20:01:02.651352] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.651381] lvl 0 el 0 blk 0 by 12582912 [D 20:01:02.651406] to 12582912 ta 12582912 fi 14680064 [D 20:01:02.651431] basic type or contiguous data [D 20:01:02.651454] Request disp [D 20:01:02.651478] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:02.651503] process a segment [D 20:01:02.651527] of 12582912 sz 2097152 [D 20:01:02.651552] add a segment 0 [D 20:01:02.651576] retval = 2097152 [D 20:01:02.651599] exiting distribute returned less than expected [D 20:01:02.651623] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:02.651649] *****RETURN***FROM***PROCESS***** [D 20:01:02.651674] end iteration [D 20:01:02.651697] done with segments or bytes [D 20:01:02.651721] of 81788928 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.651746] next loff: 81788928 next poff: 81788928 [D 20:01:02.651771] return value 2097152 [D 20:01:02.651795] finished [D 20:01:02.651819] retval = 2097152 [D 20:01:02.651843] exiting distribute returned less than expected [D 20:01:02.651866] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 81788928 fo 100663296 eof 0 [D 20:01:02.651892] ========================================================= [D 20:01:02.651917] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.651942] element 0: offset: 0x9f04d000, size: 2097152 [D 20:01:02.651971] post_sr: da13:3346 bh 52 len 48 wr_id 15e73060. [D 20:01:02.653149] openib_check_cq: completed recv wr_id 15e73a10. [D 20:01:02.653178] openib_post_rr: da13:3346 bh 55 wr_id 15e73a10. [D 20:01:02.653208] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.653234] ========================================================= [D 20:01:02.653259] PINT_process_request [D 20:01:02.653283] tiling 15 copies [D 20:01:02.653306] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.653333] lvl 0 el 0 blk 0 by 459276288 [D 20:01:02.653358] to 459276288 ta 436207616 fi 469762048 [D 20:01:02.653383] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:02.653408] basic type or contiguous data [D 20:01:02.653431] Request disp [D 20:01:02.653454] contig_offset = 459276288 contig_size = 44040192 lvl_flag = 1 [D 20:01:02.653480] calling distribute [D 20:01:02.653504] PINT_distribute [D 20:01:02.653528] of 459276288 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.653553] begin iteration loff: 459276288 [D 20:01:02.653578] segment exceeds byte limit [D 20:01:02.653602] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:02.653627] **********CALL***PROCESS********* [D 20:01:02.653651] segment of 23068672 sz 2097152 [D 20:01:02.653675] PINT_process_request [D 20:01:02.653698] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.653724] lvl 0 el 0 blk 0 by 23068672 [D 20:01:02.653749] to 23068672 ta 23068672 fi 25165824 [D 20:01:02.653774] basic type or contiguous data [D 20:01:02.653798] Request disp [D 20:01:02.653821] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:02.653846] process a segment [D 20:01:02.653870] of 23068672 sz 2097152 [D 20:01:02.653894] add a segment 0 [D 20:01:02.653918] retval = 2097152 [D 20:01:02.653942] exiting distribute returned less than expected [D 20:01:02.653965] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:02.653991] *****RETURN***FROM***PROCESS***** [D 20:01:02.654016] end iteration [D 20:01:02.654039] done with segments or bytes [D 20:01:02.654063] of 461373440 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.654088] next loff: 461373440 next poff: 461373440 [D 20:01:02.654113] return value 2097152 [D 20:01:02.654137] finished [D 20:01:02.654161] retval = 2097152 [D 20:01:02.654185] exiting distribute returned less than expected [D 20:01:02.654209] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 461373440 fo 469762048 eof 0 [D 20:01:02.654235] ========================================================= [D 20:01:02.654264] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.654290] element 0: offset: 0x89a37000, size: 2097152 [D 20:01:02.654315] openib_check_cq: completed recv wr_id 15e73a28. [D 20:01:02.654343] openib_post_rr: da13:3346 bh 56 wr_id 15e73a28. [D 20:01:02.654370] openib_check_cq: completed rdma wr_id 15ee24e0, nic_wr_credit 0. [D 20:01:02.654396] post_sr: da13:3346 bh 53 len 16 wr_id 15e73078. [D 20:01:02.654422] openib_check_cq: completed send wr_id 15e73048. [D 20:01:02.654447] openib_check_cq: completed send wr_id 15e73060. [D 20:01:02.654473] openib_check_cq: completed send wr_id 15e73078. [D 20:01:02.654501] post_sr: da13:3346 bh 54 len 48 wr_id 15e73090. [D 20:01:02.654529] PINT_client_state_machine_test id 22 [D 20:01:02.654561] flowproto-multiqueue mem_to_bmi_callback_fn, error_code: 0, flow: 0x15eaeb40. [D 20:01:02.654587] flowproto completing 0x15eaeb40 [D 20:01:02.654613] Job flows in progress (callback time): 15 [D 20:01:02.654638] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.654664] ========================================================= [D 20:01:02.654688] PINT_process_request [D 20:01:02.654715] tiling 2 copies [D 20:01:02.654739] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.654765] lvl 0 el 0 blk 0 by 20971520 [D 20:01:02.654790] to 20971520 ta 0 fi 33554432 [D 20:01:02.654814] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:02.654839] basic type or contiguous data [D 20:01:02.654863] Request disp [D 20:01:02.654887] contig_offset = 20971520 contig_size = 46137344 lvl_flag = 1 [D 20:01:02.654911] calling distribute [D 20:01:02.654935] PINT_distribute [D 20:01:02.654959] of 20971520 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.654986] begin iteration loff: 20971520 [D 20:01:02.655010] segment exceeds byte limit [D 20:01:02.655035] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:02.655058] **********CALL***PROCESS********* [D 20:01:02.655082] segment of 20971520 sz 2097152 [D 20:01:02.655107] PINT_process_request [D 20:01:02.655131] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.655157] lvl 0 el 0 blk 0 by 20971520 [D 20:01:02.655181] to 20971520 ta 20971520 fi 23068672 [D 20:01:02.655206] basic type or contiguous data [D 20:01:02.655230] Request disp [D 20:01:02.655254] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:02.655279] process a segment [D 20:01:02.655303] of 20971520 sz 2097152 [D 20:01:02.655326] add a segment 0 [D 20:01:02.655350] retval = 2097152 [D 20:01:02.655374] exiting distribute returned less than expected [D 20:01:02.655398] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:02.655424] *****RETURN***FROM***PROCESS***** [D 20:01:02.655448] end iteration [D 20:01:02.655472] done with segments or bytes [D 20:01:02.655495] of 23068672 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.655521] next loff: 23068672 next poff: 23068672 [D 20:01:02.655546] return value 2097152 [D 20:01:02.655570] finished [D 20:01:02.655593] retval = 2097152 [D 20:01:02.655617] exiting distribute returned less than expected [D 20:01:02.655641] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 23068672 fo 33554432 eof 0 [D 20:01:02.655667] ========================================================= [D 20:01:02.655691] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.655716] element 0: offset: 0xa3851000, size: 2097152 [D 20:01:02.655742] openib_check_cq: completed send wr_id 15e73090. [D 20:01:02.655772] post_sr: da13:3346 bh 55 len 48 wr_id 15e730a8. [D 20:01:02.655800] sm control op 6 [D 20:01:02.655826] [SM Entering]: (0x15eadd18) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:02.655851] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:01:02.655877] (0x15eaddb0) io_datafile_complete_operations (tag 2) [D 20:01:02.655909] matched completed flow for context 0x15eae590 and reset write_recv timeout [D 20:01:02.655935] 0 flows pending, 1 write acks pending, 0 msgpair [D 20:01:02.655960] [SM Exiting]: (0x15eadd18) client_io_sm:io_datafile_complete_operations (error code: 0), (action: DEFERRED) [D 20:01:02.655985] PINT_client_state_machine_test id 22 [D 20:01:02.656014] openib_check_cq: completed send wr_id 15e730a8. [D 20:01:02.657908] openib_check_cq: completed recv wr_id 15e73a40. [D 20:01:02.657936] openib_post_rr: da13:3346 bh 57 wr_id 15e73a40. [D 20:01:02.657962] post_sr: da13:3346 bh 56 len 48 wr_id 15e730c0. [D 20:01:02.657999] openib_check_cq: completed send wr_id 15e730c0. [D 20:01:02.663028] openib_check_cq: completed recv wr_id 15e73a58. [D 20:01:02.663057] openib_post_rr: da13:3346 bh 58 wr_id 15e73a58. [D 20:01:02.663087] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.663113] ========================================================= [D 20:01:02.663138] PINT_process_request [D 20:01:02.663162] tiling 3 copies [D 20:01:02.663186] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.663212] lvl 0 el 0 blk 0 by 52428800 [D 20:01:02.663237] to 52428800 ta 33554432 fi 67108864 [D 20:01:02.663262] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:02.663287] basic type or contiguous data [D 20:01:02.663311] Request disp [D 20:01:02.663334] contig_offset = 52428800 contig_size = 48234496 lvl_flag = 1 [D 20:01:02.663359] calling distribute [D 20:01:02.663383] PINT_distribute [D 20:01:02.663407] of 52428800 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.663434] begin iteration loff: 52428800 [D 20:01:02.663458] segment exceeds byte limit [D 20:01:02.663482] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:02.663507] **********CALL***PROCESS********* [D 20:01:02.663531] segment of 18874368 sz 2097152 [D 20:01:02.663556] PINT_process_request [D 20:01:02.663579] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.663605] lvl 0 el 0 blk 0 by 18874368 [D 20:01:02.663630] to 18874368 ta 18874368 fi 20971520 [D 20:01:02.663655] basic type or contiguous data [D 20:01:02.663679] Request disp [D 20:01:02.663702] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:02.663727] process a segment [D 20:01:02.663751] of 18874368 sz 2097152 [D 20:01:02.663775] add a segment 0 [D 20:01:02.663799] retval = 2097152 [D 20:01:02.663823] exiting distribute returned less than expected [D 20:01:02.663847] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:02.663873] *****RETURN***FROM***PROCESS***** [D 20:01:02.663897] end iteration [D 20:01:02.663921] done with segments or bytes [D 20:01:02.663945] of 54525952 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.663970] next loff: 54525952 next poff: 54525952 [D 20:01:02.663995] return value 2097152 [D 20:01:02.664019] finished [D 20:01:02.664043] retval = 2097152 [D 20:01:02.664067] exiting distribute returned less than expected [D 20:01:02.664091] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 54525952 fo 67108864 eof 0 [D 20:01:02.664117] ========================================================= [D 20:01:02.664141] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.664167] element 0: offset: 0xa164f000, size: 2097152 [D 20:01:02.664196] post_sr: da13:3346 bh 57 len 48 wr_id 15e730d8. [D 20:01:02.664231] openib_check_cq: completed send wr_id 15e730d8. [D 20:01:02.666013] PINT_client_state_machine_test id 22 [D 20:01:02.676042] PINT_client_state_machine_test id 22 [D 20:01:02.686074] PINT_client_state_machine_test id 22 [D 20:01:02.696102] PINT_client_state_machine_test id 22 [D 20:01:02.714698] PINT_client_state_machine_test id 22 [D 20:01:02.726697] PINT_client_state_machine_test id 22 -[D 20:01:02.738415] openib_check_cq: completed recv wr_id 15e73a70. [D 20:01:02.738449] openib_post_rr: da13:3346 bh 59 wr_id 15e73a70. [D 20:01:02.738482] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.738509] ========================================================= [D 20:01:02.738533] PINT_process_request [D 20:01:02.738556] tiling 4 copies [D 20:01:02.738581] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.738607] lvl 0 el 0 blk 0 by 81788928 [D 20:01:02.738632] to 81788928 ta 67108864 fi 100663296 [D 20:01:02.738656] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:02.738681] basic type or contiguous data [D 20:01:02.738708] Request disp [D 20:01:02.738732] contig_offset = 81788928 contig_size = 52428800 lvl_flag = 1 [D 20:01:02.738757] calling distribute [D 20:01:02.738780] PINT_distribute [D 20:01:02.738804] of 81788928 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.738831] begin iteration loff: 81788928 [D 20:01:02.738856] segment exceeds byte limit [D 20:01:02.738880] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:02.738904] **********CALL***PROCESS********* [D 20:01:02.738928] segment of 14680064 sz 2097152 [D 20:01:02.738952] PINT_process_request [D 20:01:02.738976] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.739003] lvl 0 el 0 blk 0 by 14680064 [D 20:01:02.739027] to 14680064 ta 14680064 fi 16777216 [D 20:01:02.739052] basic type or contiguous data [D 20:01:02.739076] Request disp [D 20:01:02.739099] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:02.739124] process a segment [D 20:01:02.739148] of 14680064 sz 2097152 [D 20:01:02.739171] add a segment 0 [D 20:01:02.739196] retval = 2097152 [D 20:01:02.739220] exiting distribute returned less than expected [D 20:01:02.739244] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:02.739270] *****RETURN***FROM***PROCESS***** [D 20:01:02.739294] end iteration [D 20:01:02.739317] done with segments or bytes [D 20:01:02.739341] of 83886080 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.739367] next loff: 83886080 next poff: 83886080 [D 20:01:02.739391] return value 2097152 [D 20:01:02.739416] finished [D 20:01:02.739439] retval = 2097152 [D 20:01:02.739463] exiting distribute returned less than expected [D 20:01:02.739487] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 83886080 fo 100663296 eof 0 [D 20:01:02.739513] ========================================================= [D 20:01:02.739538] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.739562] element 0: offset: 0x9f24d000, size: 2097152 [D 20:01:02.739593] post_sr: da13:3346 bh 58 len 48 wr_id 15e730f0. [D 20:01:02.739622] PINT_client_state_machine_test id 22 [D 20:01:02.739663] openib_check_cq: completed send wr_id 15e730f0. [D 20:01:02.740625] openib_check_cq: completed recv wr_id 15e73a88. [D 20:01:02.740653] openib_post_rr: da13:3346 bh 60 wr_id 15e73a88. [D 20:01:02.740683] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.740713] ========================================================= [D 20:01:02.740740] PINT_process_request [D 20:01:02.740768] tiling 15 copies [D 20:01:02.740797] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.740823] lvl 0 el 0 blk 0 by 461373440 [D 20:01:02.740848] to 461373440 ta 436207616 fi 469762048 [D 20:01:02.740873] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:02.740898] basic type or contiguous data [D 20:01:02.740921] Request disp [D 20:01:02.740945] contig_offset = 461373440 contig_size = 41943040 lvl_flag = 1 [D 20:01:02.740971] calling distribute [D 20:01:02.740995] PINT_distribute [D 20:01:02.741019] of 461373440 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.741044] begin iteration loff: 461373440 [D 20:01:02.741069] segment exceeds byte limit [D 20:01:02.741093] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:02.741118] **********CALL***PROCESS********* [D 20:01:02.741145] segment of 25165824 sz 2097152 [D 20:01:02.741169] PINT_process_request [D 20:01:02.741193] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.741219] lvl 0 el 0 blk 0 by 25165824 [D 20:01:02.741244] to 25165824 ta 25165824 fi 27262976 [D 20:01:02.741269] basic type or contiguous data [D 20:01:02.741292] Request disp [D 20:01:02.741316] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:02.741341] process a segment [D 20:01:02.741365] of 25165824 sz 2097152 [D 20:01:02.741389] add a segment 0 [D 20:01:02.741413] retval = 2097152 [D 20:01:02.741436] exiting distribute returned less than expected [D 20:01:02.741461] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:02.741487] *****RETURN***FROM***PROCESS***** [D 20:01:02.741511] end iteration [D 20:01:02.741535] done with segments or bytes [D 20:01:02.741559] of 463470592 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.741584] next loff: 463470592 next poff: 463470592 [D 20:01:02.741609] return value 2097152 [D 20:01:02.741633] finished [D 20:01:02.741657] retval = 2097152 [D 20:01:02.741681] exiting distribute returned less than expected [D 20:01:02.741704] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 463470592 fo 469762048 eof 0 [D 20:01:02.741730] ========================================================= [D 20:01:02.741755] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.741781] element 0: offset: 0x89c37000, size: 2097152 [D 20:01:02.741809] post_sr: da13:3346 bh 59 len 48 wr_id 15e73108. [D 20:01:02.741878] openib_check_cq: completed send wr_id 15e73108. [D 20:01:02.742852] openib_check_cq: completed recv wr_id 15e73aa0. [D 20:01:02.742879] openib_post_rr: da13:3346 bh 61 wr_id 15e73aa0. [D 20:01:02.742911] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.742937] ========================================================= [D 20:01:02.742961] PINT_process_request [D 20:01:02.742985] tiling 2 copies [D 20:01:02.743009] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.743036] lvl 0 el 0 blk 0 by 23068672 [D 20:01:02.743061] to 23068672 ta 0 fi 33554432 [D 20:01:02.743085] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:02.743110] basic type or contiguous data [D 20:01:02.743133] Request disp [D 20:01:02.743157] contig_offset = 23068672 contig_size = 44040192 lvl_flag = 1 [D 20:01:02.743182] calling distribute [D 20:01:02.743206] PINT_distribute [D 20:01:02.743229] of 23068672 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.743256] begin iteration loff: 23068672 [D 20:01:02.743281] segment exceeds byte limit [D 20:01:02.743305] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:02.743330] **********CALL***PROCESS********* [D 20:01:02.743352] segment of 23068672 sz 2097152 [D 20:01:02.743377] PINT_process_request [D 20:01:02.743401] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.743427] lvl 0 el 0 blk 0 by 23068672 [D 20:01:02.743452] to 23068672 ta 23068672 fi 25165824 [D 20:01:02.743476] basic type or contiguous data [D 20:01:02.743500] Request disp [D 20:01:02.743524] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:02.743549] process a segment [D 20:01:02.743573] of 23068672 sz 2097152 [D 20:01:02.743597] add a segment 0 [D 20:01:02.743620] retval = 2097152 [D 20:01:02.743644] exiting distribute returned less than expected [D 20:01:02.743668] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:02.743694] *****RETURN***FROM***PROCESS***** [D 20:01:02.743718] end iteration [D 20:01:02.743741] done with segments or bytes [D 20:01:02.743765] of 25165824 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.743791] next loff: 25165824 next poff: 25165824 [D 20:01:02.743815] return value 2097152 [D 20:01:02.743840] finished [D 20:01:02.743865] retval = 2097152 [D 20:01:02.743890] exiting distribute returned less than expected [D 20:01:02.743914] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 25165824 fo 33554432 eof 0 [D 20:01:02.743940] ========================================================= [D 20:01:02.743964] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.743989] element 0: offset: 0xa3a51000, size: 2097152 [D 20:01:02.744019] post_sr: da13:3346 bh 60 len 48 wr_id 15e73120. [D 20:01:02.744090] openib_check_cq: completed send wr_id 15e73120. [D 20:01:02.745075] openib_check_cq: completed recv wr_id 15e73ab8. [D 20:01:02.745102] openib_post_rr: da13:3346 bh 62 wr_id 15e73ab8. [D 20:01:02.745133] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:02.745159] ========================================================= [D 20:01:02.745183] PINT_process_request [D 20:01:02.745207] tiling 5 copies [D 20:01:02.745231] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.745257] lvl 0 el 0 blk 0 by 102760448 [D 20:01:02.745283] to 102760448 ta 100663296 fi 134217728 [D 20:01:02.745307] mto 2097152 mta 0 mfi 2097152 [D 20:01:02.745331] basic type or contiguous data [D 20:01:02.745356] Request disp [D 20:01:02.745380] contig_offset = 102760448 contig_size = 65011712 lvl_flag = 1 [D 20:01:02.745405] calling distribute [D 20:01:02.745429] PINT_distribute [D 20:01:02.745452] of 102760448 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:02.745478] begin iteration loff: 102760448 [D 20:01:02.745504] segment exceeds byte limit [D 20:01:02.745528] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:02.745553] **********CALL***PROCESS********* [D 20:01:02.745575] segment of 2097152 sz 2097152 [D 20:01:02.745600] PINT_process_request [D 20:01:02.745624] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.745651] lvl 0 el 0 blk 0 by 2097152 [D 20:01:02.745675] to 2097152 ta 2097152 fi 4194304 [D 20:01:02.745699] basic type or contiguous data [D 20:01:02.745723] Request disp [D 20:01:02.745747] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:02.745772] process a segment [D 20:01:02.745796] of 2097152 sz 2097152 [D 20:01:02.745820] add a segment 0 [D 20:01:02.745843] retval = 2097152 [D 20:01:02.745868] exiting distribute returned less than expected [D 20:01:02.745892] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:02.745918] *****RETURN***FROM***PROCESS***** [D 20:01:02.745942] end iteration [D 20:01:02.745965] done with segments or bytes [D 20:01:02.745989] of 104857600 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.746015] next loff: 104857600 next poff: 104857600 [D 20:01:02.746040] return value 2097152 [D 20:01:02.746064] finished [D 20:01:02.746088] retval = 2097152 [D 20:01:02.746111] exiting distribute returned less than expected [D 20:01:02.746135] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 104857600 fo 134217728 eof 0 [D 20:01:02.746161] ========================================================= [D 20:01:02.746186] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:02.746211] element 0: offset: 0x9c64b000, size: 2097152 [D 20:01:02.747299] openib_check_cq: completed recv wr_id 15e73ad0. [D 20:01:02.747326] openib_post_rr: da13:3346 bh 63 wr_id 15e73ad0. [D 20:01:02.747357] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.747383] ========================================================= [D 20:01:02.747406] PINT_process_request [D 20:01:02.747431] tiling 3 copies [D 20:01:02.747455] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.747481] lvl 0 el 0 blk 0 by 54525952 [D 20:01:02.747506] to 54525952 ta 33554432 fi 67108864 [D 20:01:02.747530] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:02.747555] basic type or contiguous data [D 20:01:02.747579] Request disp [D 20:01:02.747606] contig_offset = 54525952 contig_size = 46137344 lvl_flag = 1 [D 20:01:02.747631] calling distribute [D 20:01:02.747655] PINT_distribute [D 20:01:02.747678] of 54525952 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.747705] begin iteration loff: 54525952 [D 20:01:02.747730] segment exceeds byte limit [D 20:01:02.747754] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:02.747778] **********CALL***PROCESS********* [D 20:01:02.747802] segment of 20971520 sz 2097152 [D 20:01:02.747827] PINT_process_request [D 20:01:02.747851] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.747877] lvl 0 el 0 blk 0 by 20971520 [D 20:01:02.747901] to 20971520 ta 20971520 fi 23068672 [D 20:01:02.747926] basic type or contiguous data [D 20:01:02.747950] Request disp [D 20:01:02.747974] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:02.747999] process a segment [D 20:01:02.748023] of 20971520 sz 2097152 [D 20:01:02.748046] add a segment 0 [D 20:01:02.748070] retval = 2097152 [D 20:01:02.748094] exiting distribute returned less than expected [D 20:01:02.748118] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:02.748145] *****RETURN***FROM***PROCESS***** [D 20:01:02.748168] end iteration [D 20:01:02.748192] done with segments or bytes [D 20:01:02.748215] of 56623104 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.748241] next loff: 56623104 next poff: 56623104 [D 20:01:02.748266] return value 2097152 [D 20:01:02.748290] finished [D 20:01:02.748313] retval = 2097152 [D 20:01:02.748337] exiting distribute returned less than expected [D 20:01:02.748361] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 56623104 fo 67108864 eof 0 [D 20:01:02.748387] ========================================================= [D 20:01:02.748412] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.748436] element 0: offset: 0xa184f000, size: 2097152 [D 20:01:02.748466] post_sr: da13:3346 bh 61 len 48 wr_id 15e73138. [D 20:01:02.748537] openib_check_cq: completed send wr_id 15e73138. [D 20:01:02.749524] openib_check_cq: completed recv wr_id 15e73ae8. [D 20:01:02.749552] openib_post_rr: da13:3346 bh 64 wr_id 15e73ae8. [D 20:01:02.749578] openib_check_cq: completed recv wr_id 15e73b00. [D 20:01:02.749605] openib_post_rr: da13:3346 bh 65 wr_id 15e73b00. [D 20:01:02.749635] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.749661] ========================================================= [D 20:01:02.749686] PINT_process_request [D 20:01:02.749710] tiling 4 copies [D 20:01:02.749733] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.749760] lvl 0 el 0 blk 0 by 83886080 [D 20:01:02.749785] to 83886080 ta 67108864 fi 100663296 [D 20:01:02.749810] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:02.749835] basic type or contiguous data [D 20:01:02.749858] Request disp [D 20:01:02.749882] contig_offset = 83886080 contig_size = 50331648 lvl_flag = 1 [D 20:01:02.749907] calling distribute [D 20:01:02.749931] PINT_distribute [D 20:01:02.749955] of 83886080 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.749980] begin iteration loff: 83886080 [D 20:01:02.750005] segment exceeds byte limit [D 20:01:02.750029] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:02.750054] **********CALL***PROCESS********* [D 20:01:02.750078] segment of 16777216 sz 2097152 [D 20:01:02.750102] PINT_process_request [D 20:01:02.750126] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.750152] lvl 0 el 0 blk 0 by 16777216 [D 20:01:02.750177] to 16777216 ta 16777216 fi 18874368 [D 20:01:02.750202] basic type or contiguous data [D 20:01:02.750226] Request disp [D 20:01:02.750249] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:02.750274] process a segment [D 20:01:02.750298] of 16777216 sz 2097152 [D 20:01:02.750325] add a segment 0 [D 20:01:02.750349] retval = 2097152 [D 20:01:02.750373] exiting distribute returned less than expected [D 20:01:02.750397] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:02.750423] *****RETURN***FROM***PROCESS***** [D 20:01:02.750448] end iteration [D 20:01:02.750471] done with segments or bytes [D 20:01:02.750494] of 85983232 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.750520] next loff: 85983232 next poff: 85983232 [D 20:01:02.750545] return value 2097152 [D 20:01:02.750569] finished [D 20:01:02.750593] retval = 2097152 [D 20:01:02.750617] exiting distribute returned less than expected [D 20:01:02.750641] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 85983232 fo 100663296 eof 0 [D 20:01:02.750667] ========================================================= [D 20:01:02.750694] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.750720] element 0: offset: 0x9f44d000, size: 2097152 [D 20:01:02.750749] post_sr: da13:3346 bh 62 len 48 wr_id 15e73150. [D 20:01:02.750776] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.750802] ========================================================= [D 20:01:02.750826] PINT_process_request [D 20:01:02.750850] tiling 15 copies [D 20:01:02.750874] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.750900] lvl 0 el 0 blk 0 by 463470592 [D 20:01:02.750925] to 463470592 ta 436207616 fi 469762048 [D 20:01:02.750949] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:02.750974] basic type or contiguous data [D 20:01:02.750999] Request disp [D 20:01:02.751023] contig_offset = 463470592 contig_size = 39845888 lvl_flag = 1 [D 20:01:02.751048] calling distribute [D 20:01:02.751071] PINT_distribute [D 20:01:02.751095] of 463470592 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.751121] begin iteration loff: 463470592 [D 20:01:02.751146] segment exceeds byte limit [D 20:01:02.751170] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:02.751194] **********CALL***PROCESS********* [D 20:01:02.751218] segment of 27262976 sz 2097152 [D 20:01:02.751243] PINT_process_request [D 20:01:02.751267] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.751293] lvl 0 el 0 blk 0 by 27262976 [D 20:01:02.751317] to 27262976 ta 27262976 fi 29360128 [D 20:01:02.751341] basic type or contiguous data [D 20:01:02.751365] Request disp [D 20:01:02.751389] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:02.751414] process a segment [D 20:01:02.751438] of 27262976 sz 2097152 [D 20:01:02.751461] add a segment 0 [D 20:01:02.751485] retval = 2097152 [D 20:01:02.751509] exiting distribute returned less than expected [D 20:01:02.751534] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:02.751560] *****RETURN***FROM***PROCESS***** [D 20:01:02.751584] end iteration [D 20:01:02.751607] done with segments or bytes [D 20:01:02.751631] of 465567744 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.751657] next loff: 465567744 next poff: 465567744 [D 20:01:02.751682] return value 2097152 [D 20:01:02.751706] finished [D 20:01:02.751729] retval = 2097152 [D 20:01:02.751753] exiting distribute returned less than expected [D 20:01:02.751777] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 465567744 fo 469762048 eof 0 [D 20:01:02.751803] ========================================================= [D 20:01:02.751828] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.751852] element 0: offset: 0x89e37000, size: 2097152 [D 20:01:02.751878] openib_check_cq: completed send wr_id 15e73150. [D 20:01:02.751904] openib_check_cq: completed recv wr_id 15e73b18. [D 20:01:02.751931] openib_post_rr: da13:3346 bh 66 wr_id 15e73b18. [D 20:01:02.751961] PINT_client_state_machine_test id 22 [D 20:01:02.751993] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.752021] ========================================================= [D 20:01:02.752046] PINT_process_request [D 20:01:02.752070] tiling 2 copies [D 20:01:02.752094] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.752120] lvl 0 el 0 blk 0 by 25165824 [D 20:01:02.752145] to 25165824 ta 0 fi 33554432 [D 20:01:02.752170] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:02.752195] basic type or contiguous data [D 20:01:02.752219] Request disp [D 20:01:02.752243] contig_offset = 25165824 contig_size = 41943040 lvl_flag = 1 [D 20:01:02.752267] calling distribute [D 20:01:02.752291] PINT_distribute [D 20:01:02.752315] of 25165824 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.752342] begin iteration loff: 25165824 [D 20:01:02.752367] segment exceeds byte limit [D 20:01:02.752390] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:02.752414] **********CALL***PROCESS********* [D 20:01:02.752438] segment of 25165824 sz 2097152 [D 20:01:02.752463] PINT_process_request [D 20:01:02.752487] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.752512] lvl 0 el 0 blk 0 by 25165824 [D 20:01:02.752537] to 25165824 ta 25165824 fi 27262976 [D 20:01:02.752562] basic type or contiguous data [D 20:01:02.752586] Request disp [D 20:01:02.752610] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:02.752635] process a segment [D 20:01:02.752657] of 25165824 sz 2097152 [D 20:01:02.752682] add a segment 0 [D 20:01:02.752709] retval = 2097152 [D 20:01:02.752737] exiting distribute returned less than expected [D 20:01:02.752765] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:02.752796] *****RETURN***FROM***PROCESS***** [D 20:01:02.752820] end iteration [D 20:01:02.752844] done with segments or bytes [D 20:01:02.752868] of 27262976 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.752896] next loff: 27262976 next poff: 27262976 [D 20:01:02.752921] return value 2097152 [D 20:01:02.752944] finished [D 20:01:02.752968] retval = 2097152 [D 20:01:02.752993] exiting distribute returned less than expected [D 20:01:02.753017] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 27262976 fo 33554432 eof 0 [D 20:01:02.753043] ========================================================= [D 20:01:02.753067] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.753092] element 0: offset: 0xa3c51000, size: 2097152 [D 20:01:02.753975] openib_check_cq: completed recv wr_id 15e73b30. [D 20:01:02.754003] openib_post_rr: da13:3346 bh 67 wr_id 15e73b30. [D 20:01:02.754033] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.754058] ========================================================= [D 20:01:02.754083] PINT_process_request [D 20:01:02.754107] tiling 3 copies [D 20:01:02.754131] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.754157] lvl 0 el 0 blk 0 by 56623104 [D 20:01:02.754181] to 56623104 ta 33554432 fi 67108864 [D 20:01:02.754206] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:02.754231] basic type or contiguous data [D 20:01:02.754255] Request disp [D 20:01:02.754279] contig_offset = 56623104 contig_size = 44040192 lvl_flag = 1 [D 20:01:02.754304] calling distribute [D 20:01:02.754328] PINT_distribute [D 20:01:02.754352] of 56623104 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.754378] begin iteration loff: 56623104 [D 20:01:02.754403] segment exceeds byte limit [D 20:01:02.754426] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:02.754451] **********CALL***PROCESS********* [D 20:01:02.754474] segment of 23068672 sz 2097152 [D 20:01:02.754499] PINT_process_request [D 20:01:02.754523] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.754549] lvl 0 el 0 blk 0 by 23068672 [D 20:01:02.754573] to 23068672 ta 23068672 fi 25165824 [D 20:01:02.754600] basic type or contiguous data [D 20:01:02.754625] Request disp [D 20:01:02.754649] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:02.754673] process a segment [D 20:01:02.754699] of 23068672 sz 2097152 [D 20:01:02.754724] add a segment 0 [D 20:01:02.754748] retval = 2097152 [D 20:01:02.754772] exiting distribute returned less than expected [D 20:01:02.754796] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:02.754822] *****RETURN***FROM***PROCESS***** [D 20:01:02.754846] end iteration [D 20:01:02.754870] done with segments or bytes [D 20:01:02.754894] of 58720256 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.754920] next loff: 58720256 next poff: 58720256 [D 20:01:02.754944] return value 2097152 [D 20:01:02.754968] finished [D 20:01:02.754992] retval = 2097152 [D 20:01:02.755016] exiting distribute returned less than expected [D 20:01:02.755040] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 58720256 fo 67108864 eof 0 [D 20:01:02.755065] ========================================================= [D 20:01:02.755090] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.755115] element 0: offset: 0xa1a4f000, size: 2097152 [D 20:01:02.755142] openib_check_cq: completed recv wr_id 15e73b48. [D 20:01:02.755168] openib_post_rr: da13:3346 bh 68 wr_id 15e73b48. [D 20:01:02.755204] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.755229] ========================================================= [D 20:01:02.755254] PINT_process_request [D 20:01:02.755278] tiling 4 copies [D 20:01:02.755302] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.755328] lvl 0 el 0 blk 0 by 85983232 [D 20:01:02.755352] to 85983232 ta 67108864 fi 100663296 [D 20:01:02.755377] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:02.755402] basic type or contiguous data [D 20:01:02.755426] Request disp [D 20:01:02.755450] contig_offset = 85983232 contig_size = 48234496 lvl_flag = 1 [D 20:01:02.755475] calling distribute [D 20:01:02.755498] PINT_distribute [D 20:01:02.755522] of 85983232 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.755549] begin iteration loff: 85983232 [D 20:01:02.755573] segment exceeds byte limit [D 20:01:02.755597] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:02.755621] **********CALL***PROCESS********* [D 20:01:02.755645] segment of 18874368 sz 2097152 [D 20:01:02.755670] PINT_process_request [D 20:01:02.755693] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.755719] lvl 0 el 0 blk 0 by 18874368 [D 20:01:02.755744] to 18874368 ta 18874368 fi 20971520 [D 20:01:02.755768] basic type or contiguous data [D 20:01:02.755792] Request disp [D 20:01:02.755816] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:02.755841] process a segment [D 20:01:02.755865] of 18874368 sz 2097152 [D 20:01:02.755888] add a segment 0 [D 20:01:02.755912] retval = 2097152 [D 20:01:02.755936] exiting distribute returned less than expected [D 20:01:02.755960] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:02.755986] *****RETURN***FROM***PROCESS***** [D 20:01:02.756011] end iteration [D 20:01:02.756033] done with segments or bytes [D 20:01:02.756057] of 88080384 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.756083] next loff: 88080384 next poff: 88080384 [D 20:01:02.756108] return value 2097152 [D 20:01:02.756132] finished [D 20:01:02.756155] retval = 2097152 [D 20:01:02.756179] exiting distribute returned less than expected [D 20:01:02.756203] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 88080384 fo 100663296 eof 0 [D 20:01:02.756229] ========================================================= [D 20:01:02.756254] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.756278] element 0: offset: 0x9f64d000, size: 2097152 [D 20:01:02.759158] openib_check_cq: completed recv wr_id 15e73b60. [D 20:01:02.759185] openib_post_rr: da13:3346 bh 69 wr_id 15e73b60. [D 20:01:02.759213] post_sr: da13:3346 bh 63 len 48 wr_id 15e73168. [D 20:01:02.759240] openib_check_cq: completed recv wr_id 15e73b78. [D 20:01:02.759266] openib_post_rr: da13:3346 bh 70 wr_id 15e73b78. [D 20:01:02.759293] post_sr: da13:3346 bh 64 len 48 wr_id 15e73180. [D 20:01:02.759319] openib_check_cq: completed send wr_id 15e73168. [D 20:01:02.759345] openib_check_cq: completed send wr_id 15e73180. [D 20:01:02.761991] PINT_client_state_machine_test id 22 [D 20:01:02.772022] PINT_client_state_machine_test id 22 [D 20:01:02.782050] PINT_client_state_machine_test id 22 [D 20:01:02.792078] PINT_client_state_machine_test id 22 [D 20:01:02.806698] PINT_client_state_machine_test id 22 [D 20:01:02.818698] PINT_client_state_machine_test id 22 [D 20:01:02.830698] PINT_client_state_machine_test id 22 [D 20:01:02.842699] PINT_client_state_machine_test id 22 -[D 20:01:02.844387] openib_check_cq: completed recv wr_id 15e73b90. [D 20:01:02.844420] openib_post_rr: da13:3346 bh 71 wr_id 15e73b90. [D 20:01:02.844450] openib_check_cq: completed recv wr_id 15e73ba8. [D 20:01:02.844478] openib_post_rr: da13:3346 bh 72 wr_id 15e73ba8. [D 20:01:02.844509] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.844534] ========================================================= [D 20:01:02.844559] PINT_process_request [D 20:01:02.844583] tiling 2 copies [D 20:01:02.844608] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.844634] lvl 0 el 0 blk 0 by 27262976 [D 20:01:02.844658] to 27262976 ta 0 fi 33554432 [D 20:01:02.844683] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:02.844712] basic type or contiguous data [D 20:01:02.844736] Request disp [D 20:01:02.844759] contig_offset = 27262976 contig_size = 39845888 lvl_flag = 1 [D 20:01:02.844784] calling distribute [D 20:01:02.844809] PINT_distribute [D 20:01:02.844832] of 27262976 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.844859] begin iteration loff: 27262976 [D 20:01:02.844883] segment exceeds byte limit [D 20:01:02.844908] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:02.844933] **********CALL***PROCESS********* [D 20:01:02.844957] segment of 27262976 sz 2097152 [D 20:01:02.844982] PINT_process_request [D 20:01:02.845005] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.845031] lvl 0 el 0 blk 0 by 27262976 [D 20:01:02.845056] to 27262976 ta 27262976 fi 29360128 [D 20:01:02.845081] basic type or contiguous data [D 20:01:02.845105] Request disp [D 20:01:02.845129] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:02.845153] process a segment [D 20:01:02.845177] of 27262976 sz 2097152 [D 20:01:02.845201] add a segment 0 [D 20:01:02.845226] retval = 2097152 [D 20:01:02.845250] exiting distribute returned less than expected [D 20:01:02.845273] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:02.845300] *****RETURN***FROM***PROCESS***** [D 20:01:02.845324] end iteration [D 20:01:02.845348] done with segments or bytes [D 20:01:02.845372] of 29360128 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.845397] next loff: 29360128 next poff: 29360128 [D 20:01:02.845422] return value 2097152 [D 20:01:02.845447] finished [D 20:01:02.845470] retval = 2097152 [D 20:01:02.845494] exiting distribute returned less than expected [D 20:01:02.845518] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 29360128 fo 33554432 eof 0 [D 20:01:02.845544] ========================================================= [D 20:01:02.845569] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.845594] element 0: offset: 0xa3e51000, size: 2097152 [D 20:01:02.845623] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.845649] ========================================================= [D 20:01:02.845677] PINT_process_request [D 20:01:02.845701] tiling 4 copies [D 20:01:02.845726] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.845751] lvl 0 el 0 blk 0 by 88080384 [D 20:01:02.845776] to 88080384 ta 67108864 fi 100663296 [D 20:01:02.845801] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:02.845826] basic type or contiguous data [D 20:01:02.845849] Request disp [D 20:01:02.845873] contig_offset = 88080384 contig_size = 46137344 lvl_flag = 1 [D 20:01:02.845898] calling distribute [D 20:01:02.845923] PINT_distribute [D 20:01:02.845946] of 88080384 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.845973] begin iteration loff: 88080384 [D 20:01:02.845997] segment exceeds byte limit [D 20:01:02.846022] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:02.846047] **********CALL***PROCESS********* [D 20:01:02.846071] segment of 20971520 sz 2097152 [D 20:01:02.846096] PINT_process_request [D 20:01:02.846119] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.846145] lvl 0 el 0 blk 0 by 20971520 [D 20:01:02.846170] to 20971520 ta 20971520 fi 23068672 [D 20:01:02.846195] basic type or contiguous data [D 20:01:02.846219] Request disp [D 20:01:02.846242] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:02.846267] process a segment [D 20:01:02.846291] of 20971520 sz 2097152 [D 20:01:02.846315] add a segment 0 [D 20:01:02.846339] retval = 2097152 [D 20:01:02.846364] exiting distribute returned less than expected [D 20:01:02.846387] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:02.846413] *****RETURN***FROM***PROCESS***** [D 20:01:02.846438] end iteration [D 20:01:02.846461] done with segments or bytes [D 20:01:02.846486] of 90177536 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.846511] next loff: 90177536 next poff: 90177536 [D 20:01:02.846535] return value 2097152 [D 20:01:02.846560] finished [D 20:01:02.846584] retval = 2097152 [D 20:01:02.846608] exiting distribute returned less than expected [D 20:01:02.846632] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 90177536 fo 100663296 eof 0 [D 20:01:02.846658] ========================================================= [D 20:01:02.846683] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.846713] element 0: offset: 0x9f84d000, size: 2097152 [D 20:01:02.848730] openib_check_cq: completed recv wr_id 15e73bc0. [D 20:01:02.848757] openib_post_rr: da13:3346 bh 73 wr_id 15e73bc0. [D 20:01:02.848785] post_sr: da13:3346 bh 65 len 48 wr_id 15e73198. [D 20:01:02.848811] openib_check_cq: completed recv wr_id 15e73bd8. [D 20:01:02.848838] openib_post_rr: da13:3346 bh 74 wr_id 15e73bd8. [D 20:01:02.848865] openib_check_cq: completed recv wr_id 15e73bf0. [D 20:01:02.848892] openib_post_rr: da13:3346 bh 75 wr_id 15e73bf0. [D 20:01:02.848918] openib_check_cq: completed send wr_id 15e73198. [D 20:01:02.848943] openib_check_cq: completed recv wr_id 15e73c08. [D 20:01:02.848971] openib_post_rr: da13:3346 bh 76 wr_id 15e73c08. [D 20:01:02.848998] openib_check_cq: completed recv wr_id 15e73c20. [D 20:01:02.849025] openib_post_rr: da13:3346 bh 77 wr_id 15e73c20. [D 20:01:02.851407] openib_check_cq: completed recv wr_id 15e73c38. [D 20:01:02.851435] openib_post_rr: da13:3346 bh 78 wr_id 15e73c38. [D 20:01:02.851465] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:02.851490] ========================================================= [D 20:01:02.851515] PINT_process_request [D 20:01:02.851540] tiling 5 copies [D 20:01:02.851564] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.851590] lvl 0 el 0 blk 0 by 104857600 [D 20:01:02.851614] to 104857600 ta 100663296 fi 134217728 [D 20:01:02.851639] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:02.851665] basic type or contiguous data [D 20:01:02.851689] Request disp [D 20:01:02.851713] contig_offset = 104857600 contig_size = 62914560 lvl_flag = 1 [D 20:01:02.851743] calling distribute [D 20:01:02.851768] PINT_distribute [D 20:01:02.851792] of 104857600 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:02.851819] begin iteration loff: 104857600 [D 20:01:02.851843] segment exceeds byte limit [D 20:01:02.851867] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:02.851892] **********CALL***PROCESS********* [D 20:01:02.851916] segment of 4194304 sz 2097152 [D 20:01:02.851941] PINT_process_request [D 20:01:02.851964] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.851990] lvl 0 el 0 blk 0 by 4194304 [D 20:01:02.852015] to 4194304 ta 4194304 fi 6291456 [D 20:01:02.852040] basic type or contiguous data [D 20:01:02.852064] Request disp [D 20:01:02.852087] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:02.852112] process a segment [D 20:01:02.852136] of 4194304 sz 2097152 [D 20:01:02.852161] add a segment 0 [D 20:01:02.852185] retval = 2097152 [D 20:01:02.852209] exiting distribute returned less than expected [D 20:01:02.852232] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:02.852259] *****RETURN***FROM***PROCESS***** [D 20:01:02.852283] end iteration [D 20:01:02.852307] done with segments or bytes [D 20:01:02.852331] of 106954752 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.852356] next loff: 106954752 next poff: 106954752 [D 20:01:02.852381] return value 2097152 [D 20:01:02.852406] finished [D 20:01:02.852429] retval = 2097152 [D 20:01:02.852454] exiting distribute returned less than expected [D 20:01:02.852478] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 106954752 fo 134217728 eof 0 [D 20:01:02.852504] ========================================================= [D 20:01:02.852528] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:02.852554] element 0: offset: 0x9c84b000, size: 2097152 [D 20:01:02.852580] openib_check_cq: completed recv wr_id 15e73c50. [D 20:01:02.852606] openib_post_rr: da13:3346 bh 79 wr_id 15e73c50. [D 20:01:02.852633] post_sr: da13:3346 bh 66 len 48 wr_id 15e731b0. [D 20:01:02.852660] openib_check_cq: completed recv wr_id 15e73c68. [D 20:01:02.852686] openib_post_rr: da13:3346 bh 80 wr_id 15e73c68. [D 20:01:02.852715] post_sr: da13:3346 bh 67 len 48 wr_id 15e731c8. [D 20:01:02.852742] openib_check_cq: completed recv wr_id 15e73c80. [D 20:01:02.852768] openib_post_rr: da13:3346 bh 81 wr_id 15e73c80. [D 20:01:02.852795] openib_check_cq: completed send wr_id 15e731b0. [D 20:01:02.852820] openib_check_cq: completed send wr_id 15e731c8. [D 20:01:02.852849] post_sr: da13:3346 bh 68 len 48 wr_id 15e731e0. [D 20:01:02.852877] PINT_client_state_machine_test id 22 [D 20:01:02.852915] openib_check_cq: completed send wr_id 15e731e0. [D 20:01:02.859495] openib_check_cq: completed recv wr_id 15e73c98. [D 20:01:02.859523] openib_post_rr: da13:3346 bh 82 wr_id 15e73c98. [D 20:01:02.859550] openib_check_cq: completed recv wr_id 15e73cb0. [D 20:01:02.859577] openib_post_rr: da13:3346 bh 83 wr_id 15e73cb0. [D 20:01:02.859603] openib_check_cq: completed recv wr_id 15e73cc8. [D 20:01:02.859630] openib_post_rr: da13:3346 bh 84 wr_id 15e73cc8. [D 20:01:02.859656] openib_check_cq: completed recv wr_id 15e73ce0. [D 20:01:02.859683] openib_post_rr: da13:3346 bh 85 wr_id 15e73ce0. [D 20:01:02.859714] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.859740] ========================================================= [D 20:01:02.859765] PINT_process_request [D 20:01:02.859788] tiling 15 copies [D 20:01:02.859812] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.859839] lvl 0 el 0 blk 0 by 465567744 [D 20:01:02.859864] to 465567744 ta 436207616 fi 469762048 [D 20:01:02.859889] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.859913] basic type or contiguous data [D 20:01:02.859937] Request disp [D 20:01:02.859961] contig_offset = 465567744 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.859989] calling distribute [D 20:01:02.860013] PINT_distribute [D 20:01:02.860037] of 465567744 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.860064] begin iteration loff: 465567744 [D 20:01:02.860089] segment exceeds byte limit [D 20:01:02.860113] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.860137] **********CALL***PROCESS********* [D 20:01:02.860161] segment of 29360128 sz 2097152 [D 20:01:02.860186] PINT_process_request [D 20:01:02.860210] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.860236] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.860260] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.860285] basic type or contiguous data [D 20:01:02.860309] Request disp [D 20:01:02.860333] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.860359] process a segment [D 20:01:02.860383] of 29360128 sz 2097152 [D 20:01:02.860406] add a segment 0 [D 20:01:02.860430] retval = 2097152 [D 20:01:02.860455] exiting distribute returned less than expected [D 20:01:02.860479] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.860506] *****RETURN***FROM***PROCESS***** [D 20:01:02.860529] end iteration [D 20:01:02.860553] done with segments or bytes [D 20:01:02.860577] of 467664896 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.860603] next loff: 467664896 next poff: 467664896 [D 20:01:02.860628] return value 2097152 [D 20:01:02.860652] finished [D 20:01:02.860676] retval = 2097152 [D 20:01:02.860703] exiting distribute returned less than expected [D 20:01:02.860727] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 467664896 fo 469762048 eof 0 [D 20:01:02.860754] ========================================================= [D 20:01:02.860778] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.860803] element 0: offset: 0x8a037000, size: 2097152 [D 20:01:02.860830] openib_check_cq: completed recv wr_id 15e73cf8. [D 20:01:02.860857] openib_post_rr: da13:3346 bh 86 wr_id 15e73cf8. [D 20:01:02.860883] post_sr: da13:3346 bh 69 len 48 wr_id 15e731f8. [D 20:01:02.860910] openib_check_cq: completed recv wr_id 15e73d10. [D 20:01:02.860938] openib_post_rr: da13:3346 bh 87 wr_id 15e73d10. [D 20:01:02.860963] openib_check_cq: completed recv wr_id 15e73d28. [D 20:01:02.860991] openib_post_rr: da13:3346 bh 88 wr_id 15e73d28. [D 20:01:02.861017] openib_check_cq: completed send wr_id 15e731f8. [D 20:01:02.861046] post_sr: da13:3346 bh 70 len 48 wr_id 15e73210. [D 20:01:02.861073] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.861099] ========================================================= [D 20:01:02.861123] PINT_process_request [D 20:01:02.861147] tiling 3 copies [D 20:01:02.861171] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.861198] lvl 0 el 0 blk 0 by 58720256 [D 20:01:02.861223] to 58720256 ta 33554432 fi 67108864 [D 20:01:02.861248] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:02.861272] basic type or contiguous data [D 20:01:02.861296] Request disp [D 20:01:02.861320] contig_offset = 58720256 contig_size = 41943040 lvl_flag = 1 [D 20:01:02.861346] calling distribute [D 20:01:02.861370] PINT_distribute [D 20:01:02.861393] of 58720256 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.861420] begin iteration loff: 58720256 [D 20:01:02.861445] segment exceeds byte limit [D 20:01:02.861469] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:02.861494] **********CALL***PROCESS********* [D 20:01:02.861517] segment of 25165824 sz 2097152 [D 20:01:02.861542] PINT_process_request [D 20:01:02.861566] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.861593] lvl 0 el 0 blk 0 by 25165824 [D 20:01:02.861618] to 25165824 ta 25165824 fi 27262976 [D 20:01:02.861642] basic type or contiguous data [D 20:01:02.861666] Request disp [D 20:01:02.861693] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:02.861719] process a segment [D 20:01:02.861742] of 25165824 sz 2097152 [D 20:01:02.861766] add a segment 0 [D 20:01:02.861790] retval = 2097152 [D 20:01:02.861815] exiting distribute returned less than expected [D 20:01:02.861839] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:02.861865] *****RETURN***FROM***PROCESS***** [D 20:01:02.861889] end iteration [D 20:01:02.861913] done with segments or bytes [D 20:01:02.861937] of 60817408 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.861963] next loff: 60817408 next poff: 60817408 [D 20:01:02.861988] return value 2097152 [D 20:01:02.862011] finished [D 20:01:02.862035] retval = 2097152 [D 20:01:02.862059] exiting distribute returned less than expected [D 20:01:02.862084] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 60817408 fo 67108864 eof 0 [D 20:01:02.862110] ========================================================= [D 20:01:02.862134] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.862160] element 0: offset: 0xa1c4f000, size: 2097152 [D 20:01:02.862186] openib_check_cq: completed send wr_id 15e73210. [D 20:01:02.862216] post_sr: da13:3346 bh 71 len 48 wr_id 15e73228. [D 20:01:02.862243] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:02.862269] ========================================================= [D 20:01:02.862293] PINT_process_request [D 20:01:02.862317] tiling 5 copies [D 20:01:02.862341] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.862367] lvl 0 el 0 blk 0 by 106954752 [D 20:01:02.862393] to 106954752 ta 100663296 fi 134217728 [D 20:01:02.862418] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:02.862442] basic type or contiguous data [D 20:01:02.862466] Request disp [D 20:01:02.862490] contig_offset = 106954752 contig_size = 60817408 lvl_flag = 1 [D 20:01:02.862515] calling distribute [D 20:01:02.862539] PINT_distribute [D 20:01:02.862563] of 106954752 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:02.862589] begin iteration loff: 106954752 [D 20:01:02.862614] segment exceeds byte limit [D 20:01:02.862638] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:02.862663] **********CALL***PROCESS********* [D 20:01:02.862687] segment of 6291456 sz 2097152 [D 20:01:02.862715] PINT_process_request [D 20:01:02.862743] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.862773] lvl 0 el 0 blk 0 by 6291456 [D 20:01:02.862802] to 6291456 ta 6291456 fi 8388608 [D 20:01:02.862827] basic type or contiguous data [D 20:01:02.862851] Request disp [D 20:01:02.862875] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:02.862902] process a segment [D 20:01:02.862926] of 6291456 sz 2097152 [D 20:01:02.862951] add a segment 0 [D 20:01:02.862975] retval = 2097152 [D 20:01:02.862998] exiting distribute returned less than expected [D 20:01:02.863023] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:02.863049] *****RETURN***FROM***PROCESS***** [D 20:01:02.863074] end iteration [D 20:01:02.863098] done with segments or bytes [D 20:01:02.863121] of 109051904 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.863147] next loff: 109051904 next poff: 109051904 [D 20:01:02.863171] return value 2097152 [D 20:01:02.863196] finished [D 20:01:02.863220] retval = 2097152 [D 20:01:02.863244] exiting distribute returned less than expected [D 20:01:02.863267] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 109051904 fo 134217728 eof 0 [D 20:01:02.863294] ========================================================= [D 20:01:02.863318] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:02.863344] element 0: offset: 0x9ca4b000, size: 2097152 [D 20:01:02.863369] openib_check_cq: completed send wr_id 15e73228. [D 20:01:02.863402] post_sr: da13:3346 bh 72 len 48 wr_id 15e73240. [D 20:01:02.863430] PINT_client_state_machine_test id 22 [D 20:01:02.863459] openib_check_cq: completed send wr_id 15e73240. [D 20:01:02.868588] openib_check_cq: completed recv wr_id 15e73d40. [D 20:01:02.868616] openib_post_rr: da13:3346 bh 89 wr_id 15e73d40. [D 20:01:02.868642] post_sr: da13:3346 bh 73 len 48 wr_id 15e73258. [D 20:01:02.868679] openib_check_cq: completed send wr_id 15e73258. [D 20:01:02.868854] openib_check_cq: completed recv wr_id 15e73d58. [D 20:01:02.868882] openib_post_rr: da13:3346 bh 90 wr_id 15e73d58. [D 20:01:02.869048] openib_check_cq: completed recv wr_id 15e73d70. [D 20:01:02.869075] openib_post_rr: da13:3346 bh 91 wr_id 15e73d70. [D 20:01:02.869219] openib_check_cq: completed recv wr_id 15e73d88. [D 20:01:02.869246] openib_post_rr: da13:3346 bh 92 wr_id 15e73d88. [D 20:01:02.869273] post_sr: da13:3346 bh 74 len 48 wr_id 15e73270. [D 20:01:02.869309] openib_check_cq: completed send wr_id 15e73270. [D 20:01:02.870066] openib_check_cq: completed recv wr_id 15e73da0. [D 20:01:02.870094] openib_post_rr: da13:3346 bh 93 wr_id 15e73da0. [D 20:01:02.870120] openib_check_cq: completed recv wr_id 15e73db8. [D 20:01:02.870147] openib_post_rr: da13:3346 bh 94 wr_id 15e73db8. [D 20:01:02.870293] openib_check_cq: completed recv wr_id 15e73dd0. [D 20:01:02.870321] openib_post_rr: da13:3346 bh 95 wr_id 15e73dd0. [D 20:01:02.873460] PINT_client_state_machine_test id 22 [D 20:01:02.883491] PINT_client_state_machine_test id 22 [D 20:01:02.884294] openib_check_cq: completed recv wr_id 15e73de8. [D 20:01:02.884322] openib_post_rr: da13:3346 bh 96 wr_id 15e73de8. [D 20:01:02.884348] openib_check_cq: completed recv wr_id 15e73e00. [D 20:01:02.884376] openib_post_rr: da13:3346 bh 97 wr_id 15e73e00. [D 20:01:02.884402] openib_check_cq: completed recv wr_id 15e73e18. [D 20:01:02.884429] openib_post_rr: da13:3346 bh 98 wr_id 15e73e18. [D 20:01:02.884455] openib_check_cq: completed recv wr_id 15e73e30. [D 20:01:02.884483] openib_post_rr: da13:3346 bh 99 wr_id 15e73e30. [D 20:01:02.884509] openib_check_cq: completed recv wr_id 15e734e8. [D 20:01:02.884536] openib_post_rr: da13:3346 bh 0 wr_id 15e734e8. [D 20:01:02.884563] openib_check_cq: completed recv wr_id 15e73500. [D 20:01:02.884590] openib_post_rr: da13:3346 bh 1 wr_id 15e73500. [D 20:01:02.884616] openib_check_cq: completed recv wr_id 15e73518. [D 20:01:02.884643] openib_post_rr: da13:3346 bh 2 wr_id 15e73518. [D 20:01:02.884670] openib_check_cq: completed recv wr_id 15e73530. [D 20:01:02.884698] openib_post_rr: da13:3346 bh 3 wr_id 15e73530. [D 20:01:02.884731] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:02.884757] ========================================================= [D 20:01:02.884781] PINT_process_request [D 20:01:02.884805] tiling 6 copies [D 20:01:02.884830] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.884856] lvl 0 el 0 blk 0 by 136314880 [D 20:01:02.884881] to 136314880 ta 134217728 fi 167772160 [D 20:01:02.884905] mto 2097152 mta 0 mfi 2097152 [D 20:01:02.884930] basic type or contiguous data [D 20:01:02.884954] Request disp [D 20:01:02.884978] contig_offset = 136314880 contig_size = 65011712 lvl_flag = 1 [D 20:01:02.885004] calling distribute [D 20:01:02.885027] PINT_distribute [D 20:01:02.885051] of 136314880 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:02.885078] begin iteration loff: 136314880 [D 20:01:02.885104] segment exceeds byte limit [D 20:01:02.885128] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:02.885152] **********CALL***PROCESS********* [D 20:01:02.885176] segment of 2097152 sz 2097152 [D 20:01:02.885201] PINT_process_request [D 20:01:02.885225] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.885251] lvl 0 el 0 blk 0 by 2097152 [D 20:01:02.885275] to 2097152 ta 2097152 fi 4194304 [D 20:01:02.885300] basic type or contiguous data [D 20:01:02.885325] Request disp [D 20:01:02.885349] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:02.885376] process a segment [D 20:01:02.885400] of 2097152 sz 2097152 [D 20:01:02.885425] add a segment 0 [D 20:01:02.885449] retval = 2097152 [D 20:01:02.885473] exiting distribute returned less than expected [D 20:01:02.885497] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:02.885523] *****RETURN***FROM***PROCESS***** [D 20:01:02.885547] end iteration [D 20:01:02.885571] done with segments or bytes [D 20:01:02.885595] of 138412032 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.885621] next loff: 138412032 next poff: 138412032 [D 20:01:02.885645] return value 2097152 [D 20:01:02.885670] finished [D 20:01:02.885694] retval = 2097152 [D 20:01:02.885718] exiting distribute returned less than expected [D 20:01:02.885742] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 138412032 fo 167772160 eof 0 [D 20:01:02.885767] ========================================================= [D 20:01:02.885792] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:02.885817] element 0: offset: 0x9a649000, size: 2097152 [D 20:01:02.885844] openib_check_cq: completed recv wr_id 15e73548. [D 20:01:02.885870] openib_post_rr: da13:3346 bh 4 wr_id 15e73548. [D 20:01:02.885897] openib_check_cq: completed recv wr_id 15e73560. [D 20:01:02.885925] openib_post_rr: da13:3346 bh 5 wr_id 15e73560. [D 20:01:02.885955] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:02.885981] ========================================================= [D 20:01:02.886006] PINT_process_request [D 20:01:02.886030] tiling 2 copies [D 20:01:02.886054] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.886081] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.886106] to 29360128 ta 0 fi 33554432 [D 20:01:02.886131] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:02.886156] basic type or contiguous data [D 20:01:02.886179] Request disp [D 20:01:02.886203] contig_offset = 29360128 contig_size = 37748736 lvl_flag = 1 [D 20:01:02.886228] calling distribute [D 20:01:02.886252] PINT_distribute [D 20:01:02.886276] of 29360128 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.886302] begin iteration loff: 29360128 [D 20:01:02.886327] segment exceeds byte limit [D 20:01:02.886351] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:02.886376] **********CALL***PROCESS********* [D 20:01:02.886400] segment of 29360128 sz 2097152 [D 20:01:02.886425] PINT_process_request [D 20:01:02.886448] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.886474] lvl 0 el 0 blk 0 by 29360128 [D 20:01:02.886499] to 29360128 ta 29360128 fi 31457280 [D 20:01:02.886524] basic type or contiguous data [D 20:01:02.886548] Request disp [D 20:01:02.886571] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:02.886597] process a segment [D 20:01:02.886620] of 29360128 sz 2097152 [D 20:01:02.886645] add a segment 0 [D 20:01:02.886669] retval = 2097152 [D 20:01:02.886694] exiting distribute returned less than expected [D 20:01:02.886723] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:02.886754] *****RETURN***FROM***PROCESS***** [D 20:01:02.886781] end iteration [D 20:01:02.886808] done with segments or bytes [D 20:01:02.886832] of 31457280 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.886858] next loff: 31457280 next poff: 31457280 [D 20:01:02.886882] return value 2097152 [D 20:01:02.886907] finished [D 20:01:02.886931] retval = 2097152 [D 20:01:02.886955] exiting distribute returned less than expected [D 20:01:02.886979] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 31457280 fo 33554432 eof 0 [D 20:01:02.887005] ========================================================= [D 20:01:02.887029] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:02.887055] element 0: offset: 0xa4051000, size: 2097152 [D 20:01:02.887087] post_sr: da13:3346 bh 75 len 48 wr_id 15e73288. [D 20:01:02.887114] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:02.887140] ========================================================= [D 20:01:02.887165] PINT_process_request [D 20:01:02.887188] tiling 4 copies [D 20:01:02.887212] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.887239] lvl 0 el 0 blk 0 by 90177536 [D 20:01:02.887264] to 90177536 ta 67108864 fi 100663296 [D 20:01:02.887289] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:02.887313] basic type or contiguous data [D 20:01:02.887337] Request disp [D 20:01:02.887362] contig_offset = 90177536 contig_size = 44040192 lvl_flag = 1 [D 20:01:02.887387] calling distribute [D 20:01:02.887411] PINT_distribute [D 20:01:02.887434] of 90177536 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.887461] begin iteration loff: 90177536 [D 20:01:02.887486] segment exceeds byte limit [D 20:01:02.887511] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:02.887536] **********CALL***PROCESS********* [D 20:01:02.887559] segment of 23068672 sz 2097152 [D 20:01:02.887583] PINT_process_request [D 20:01:02.887607] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.887634] lvl 0 el 0 blk 0 by 23068672 [D 20:01:02.887659] to 23068672 ta 23068672 fi 25165824 [D 20:01:02.887683] basic type or contiguous data [D 20:01:02.887707] Request disp [D 20:01:02.887731] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:02.887756] process a segment [D 20:01:02.887780] of 23068672 sz 2097152 [D 20:01:02.887804] add a segment 0 [D 20:01:02.887828] retval = 2097152 [D 20:01:02.887853] exiting distribute returned less than expected [D 20:01:02.887877] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:02.887903] *****RETURN***FROM***PROCESS***** [D 20:01:02.887928] end iteration [D 20:01:02.887951] done with segments or bytes [D 20:01:02.887975] of 92274688 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.888001] next loff: 92274688 next poff: 92274688 [D 20:01:02.888026] return value 2097152 [D 20:01:02.888050] finished [D 20:01:02.888073] retval = 2097152 [D 20:01:02.888097] exiting distribute returned less than expected [D 20:01:02.888122] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 92274688 fo 100663296 eof 0 [D 20:01:02.888148] ========================================================= [D 20:01:02.888172] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:02.888197] element 0: offset: 0x9fa4d000, size: 2097152 [D 20:01:02.888224] openib_check_cq: completed send wr_id 15e73288. [D 20:01:02.888253] post_sr: da13:3346 bh 76 len 48 wr_id 15e732a0. [D 20:01:02.888279] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:02.888305] ========================================================= [D 20:01:02.888330] PINT_process_request [D 20:01:02.888354] tiling 5 copies [D 20:01:02.888379] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.888404] lvl 0 el 0 blk 0 by 109051904 [D 20:01:02.888429] to 109051904 ta 100663296 fi 134217728 [D 20:01:02.888454] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:02.888479] basic type or contiguous data [D 20:01:02.888504] Request disp [D 20:01:02.888527] contig_offset = 109051904 contig_size = 58720256 lvl_flag = 1 [D 20:01:02.888552] calling distribute [D 20:01:02.888576] PINT_distribute [D 20:01:02.888600] of 109051904 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:02.888627] begin iteration loff: 109051904 [D 20:01:02.888651] segment exceeds byte limit [D 20:01:02.888676] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:02.888703] **********CALL***PROCESS********* [D 20:01:02.888727] segment of 8388608 sz 2097152 [D 20:01:02.888751] PINT_process_request [D 20:01:02.888775] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.888804] lvl 0 el 0 blk 0 by 8388608 [D 20:01:02.888830] to 8388608 ta 8388608 fi 10485760 [D 20:01:02.888854] basic type or contiguous data [D 20:01:02.888878] Request disp [D 20:01:02.888902] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:02.888928] process a segment [D 20:01:02.888952] of 8388608 sz 2097152 [D 20:01:02.888976] add a segment 0 [D 20:01:02.888999] retval = 2097152 [D 20:01:02.889023] exiting distribute returned less than expected [D 20:01:02.889047] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:02.889074] *****RETURN***FROM***PROCESS***** [D 20:01:02.889098] end iteration [D 20:01:02.889122] done with segments or bytes [D 20:01:02.889145] of 111149056 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.889171] next loff: 111149056 next poff: 111149056 [D 20:01:02.889196] return value 2097152 [D 20:01:02.889221] finished [D 20:01:02.889245] retval = 2097152 [D 20:01:02.889268] exiting distribute returned less than expected [D 20:01:02.889292] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 111149056 fo 134217728 eof 0 [D 20:01:02.889319] ========================================================= [D 20:01:02.889343] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:02.889369] element 0: offset: 0x9cc4b000, size: 2097152 [D 20:01:02.889394] openib_check_cq: completed send wr_id 15e732a0. [D 20:01:02.889420] openib_check_cq: completed recv wr_id 15e73578. [D 20:01:02.889447] openib_post_rr: da13:3346 bh 6 wr_id 15e73578. [D 20:01:02.889473] post_sr: da13:3346 bh 77 len 48 wr_id 15e732b8. [D 20:01:02.889500] openib_check_cq: completed recv wr_id 15e73590. [D 20:01:02.889528] openib_post_rr: da13:3346 bh 7 wr_id 15e73590. [D 20:01:02.889554] openib_check_cq: completed send wr_id 15e732b8. [D 20:01:02.889583] post_sr: da13:3346 bh 78 len 48 wr_id 15e732d0. [D 20:01:02.889611] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:02.889636] ========================================================= [D 20:01:02.889660] PINT_process_request [D 20:01:02.889684] tiling 15 copies [D 20:01:02.889709] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.889736] lvl 0 el 0 blk 0 by 467664896 [D 20:01:02.889760] to 467664896 ta 436207616 fi 469762048 [D 20:01:02.889785] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:02.889810] basic type or contiguous data [D 20:01:02.889834] Request disp [D 20:01:02.889858] contig_offset = 467664896 contig_size = 35651584 lvl_flag = 1 [D 20:01:02.889883] calling distribute [D 20:01:02.889906] PINT_distribute [D 20:01:02.889930] of 467664896 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:02.889957] begin iteration loff: 467664896 [D 20:01:02.889982] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:02.890007] **********CALL***PROCESS********* [D 20:01:02.890031] segment of 31457280 sz 2097152 [D 20:01:02.890056] PINT_process_request [D 20:01:02.890080] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.890107] lvl 0 el 0 blk 0 by 31457280 [D 20:01:02.890131] to 31457280 ta 31457280 fi 33554432 [D 20:01:02.890156] basic type or contiguous data [D 20:01:02.890180] Request disp [D 20:01:02.890205] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:02.890230] process a segment [D 20:01:02.890254] of 31457280 sz 2097152 [D 20:01:02.890277] add a segment 0 [D 20:01:02.890302] retval = 2097152 [D 20:01:02.890326] return from level -1 [D 20:01:02.890350] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:02.890377] *****RETURN***FROM***PROCESS***** [D 20:01:02.890401] end iteration [D 20:01:02.890425] done with segments or bytes [D 20:01:02.890449] of 469762048 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.890475] next loff: 469762048 next poff: 469762048 [D 20:01:02.890500] (return value) 2097152 [D 20:01:02.890526] finished [D 20:01:02.890550] retval = 2097152 [D 20:01:02.890575] exiting distribute returned less than expected [D 20:01:02.890599] done sg 1 sm 64 by 2097152 bm 2097152 ta 436207616 to 469762048 fo 469762048 eof 0 [D 20:01:02.890625] ========================================================= [D 20:01:02.890649] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 36 [D 20:01:02.890675] element 0: offset: 0x8a237000, size: 2097152 [D 20:01:02.890705] openib_check_cq: completed send wr_id 15e732d0. [D 20:01:02.890738] post_sr: da13:3346 bh 79 len 48 wr_id 15e732e8. [D 20:01:02.890774] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:02.890804] ========================================================= [D 20:01:02.890830] PINT_process_request [D 20:01:02.890853] tiling 3 copies [D 20:01:02.890877] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.890904] lvl 0 el 0 blk 0 by 60817408 [D 20:01:02.890929] to 60817408 ta 33554432 fi 67108864 [D 20:01:02.890954] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:02.890978] basic type or contiguous data [D 20:01:02.891002] Request disp [D 20:01:02.891027] contig_offset = 60817408 contig_size = 39845888 lvl_flag = 1 [D 20:01:02.891052] calling distribute [D 20:01:02.891076] PINT_distribute [D 20:01:02.891100] of 60817408 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:02.891126] begin iteration loff: 60817408 [D 20:01:02.891152] segment exceeds byte limit [D 20:01:02.891176] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:02.891201] **********CALL***PROCESS********* [D 20:01:02.891224] segment of 27262976 sz 2097152 [D 20:01:02.891249] PINT_process_request [D 20:01:02.891273] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:02.891300] lvl 0 el 0 blk 0 by 27262976 [D 20:01:02.891324] to 27262976 ta 27262976 fi 29360128 [D 20:01:02.891349] basic type or contiguous data [D 20:01:02.891373] Request disp [D 20:01:02.891397] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:02.891422] process a segment [D 20:01:02.891447] of 27262976 sz 2097152 [D 20:01:02.891470] add a segment 0 [D 20:01:02.891494] retval = 2097152 [D 20:01:02.891519] exiting distribute returned less than expected [D 20:01:02.891543] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:02.891569] *****RETURN***FROM***PROCESS***** [D 20:01:02.891593] end iteration [D 20:01:02.891617] done with segments or bytes [D 20:01:02.891641] of 62914560 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:02.891667] next loff: 62914560 next poff: 62914560 [D 20:01:02.891692] return value 2097152 [D 20:01:02.891716] finished [D 20:01:02.891740] retval = 2097152 [D 20:01:02.891764] exiting distribute returned less than expected [D 20:01:02.891788] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 62914560 fo 67108864 eof 0 [D 20:01:02.891815] ========================================================= [D 20:01:02.891839] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:02.891864] element 0: offset: 0xa1e4f000, size: 2097152 [D 20:01:02.891890] openib_check_cq: completed send wr_id 15e732e8. [D 20:01:02.891920] post_sr: da13:3346 bh 80 len 48 wr_id 15e73300. [D 20:01:02.891954] openib_check_cq: completed send wr_id 15e73300. [D 20:01:02.893523] PINT_client_state_machine_test id 22 [D 20:01:02.903553] PINT_client_state_machine_test id 22 [D 20:01:02.913585] PINT_client_state_machine_test id 22 [D 20:01:02.923615] PINT_client_state_machine_test id 22 [D 20:01:02.933647] PINT_client_state_machine_test id 22 [D 20:01:02.948698] PINT_client_state_machine_test id 22 [D 20:01:02.960699] PINT_client_state_machine_test id 22 [D 20:01:02.972700] PINT_client_state_machine_test id 22 [D 20:01:02.984700] PINT_client_state_machine_test id 22 [D 20:01:02.996699] PINT_client_state_machine_test id 22 [D 20:01:03.008699] PINT_client_state_machine_test id 22 [D 20:01:03.020699] PINT_client_state_machine_test id 22 [D 20:01:03.032699] PINT_client_state_machine_test id 22 [D 20:01:03.044699] PINT_client_state_machine_test id 22 [D 20:01:03.056699] PINT_client_state_machine_test id 22 [D 20:01:03.068699] PINT_client_state_machine_test id 22 [D 20:01:03.080700] PINT_client_state_machine_test id 22 [D 20:01:03.092700] PINT_client_state_machine_test id 22 [D 20:01:03.104700] PINT_client_state_machine_test id 22 [D 20:01:03.116701] PINT_client_state_machine_test id 22 [D 20:01:03.128700] PINT_client_state_machine_test id 22 [D 20:01:03.140700] PINT_client_state_machine_test id 22 [D 20:01:03.152700] PINT_client_state_machine_test id 22 [D 20:01:03.164700] PINT_client_state_machine_test id 22 [D 20:01:03.176701] PINT_client_state_machine_test id 22 [D 20:01:03.188700] PINT_client_state_machine_test id 22 [D 20:01:03.200700] PINT_client_state_machine_test id 22 [D 20:01:03.212700] PINT_client_state_machine_test id 22 [D 20:01:03.224700] PINT_client_state_machine_test id 22 [D 20:01:03.236701] PINT_client_state_machine_test id 22 [D 20:01:03.248701] PINT_client_state_machine_test id 22 [D 20:01:03.260701] PINT_client_state_machine_test id 22 [D 20:01:03.272701] PINT_client_state_machine_test id 22 [D 20:01:03.284701] PINT_client_state_machine_test id 22 [D 20:01:03.296701] PINT_client_state_machine_test id 22 [D 20:01:03.308700] PINT_client_state_machine_test id 22 [D 20:01:03.320700] PINT_client_state_machine_test id 22 [D 20:01:03.332700] PINT_client_state_machine_test id 22 [D 20:01:03.344700] PINT_client_state_machine_test id 22 [D 20:01:03.356700] PINT_client_state_machine_test id 22 [D 20:01:03.368702] PINT_client_state_machine_test id 22 [D 20:01:03.380702] PINT_client_state_machine_test id 22 [D 20:01:03.392701] PINT_client_state_machine_test id 22 [D 20:01:03.404704] PINT_client_state_machine_test id 22 [D 20:01:03.416703] PINT_client_state_machine_test id 22 [D 20:01:03.428702] PINT_client_state_machine_test id 22 [D 20:01:03.428735] sm control op 300 [D 20:01:03.428763] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:01:03.428789] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:01:03.428818] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:01:03.428845] PINT_client_state_machine_test id 22 [D 20:01:03.440701] PINT_client_state_machine_test id 22 [D 20:01:03.452702] PINT_client_state_machine_test id 22 [D 20:01:03.464702] PINT_client_state_machine_test id 22 [D 20:01:03.476701] PINT_client_state_machine_test id 22 [D 20:01:03.488704] PINT_client_state_machine_test id 22 [D 20:01:03.500701] PINT_client_state_machine_test id 22 [D 20:01:03.512701] PINT_client_state_machine_test id 22 [D 20:01:03.524702] PINT_client_state_machine_test id 22 [D 20:01:03.536702] PINT_client_state_machine_test id 22 [D 20:01:03.548702] PINT_client_state_machine_test id 22 [D 20:01:03.560703] PINT_client_state_machine_test id 22 [D 20:01:03.572719] PINT_client_state_machine_test id 22 [D 20:01:03.584702] PINT_client_state_machine_test id 22 [D 20:01:03.596702] PINT_client_state_machine_test id 22 [D 20:01:03.608704] PINT_client_state_machine_test id 22 [D 20:01:03.620703] PINT_client_state_machine_test id 22 [D 20:01:03.632702] PINT_client_state_machine_test id 22 [D 20:01:03.644702] PINT_client_state_machine_test id 22 [D 20:01:03.656703] PINT_client_state_machine_test id 22 [D 20:01:03.668703] PINT_client_state_machine_test id 22 [D 20:01:03.680703] PINT_client_state_machine_test id 22 [D 20:01:03.692703] PINT_client_state_machine_test id 22 [D 20:01:03.704703] PINT_client_state_machine_test id 22 [D 20:01:03.716703] PINT_client_state_machine_test id 22 [D 20:01:03.728703] PINT_client_state_machine_test id 22 [D 20:01:03.740702] PINT_client_state_machine_test id 22 [D 20:01:03.752704] PINT_client_state_machine_test id 22 [D 20:01:03.764703] PINT_client_state_machine_test id 22 [D 20:01:03.776703] PINT_client_state_machine_test id 22 [D 20:01:03.788703] PINT_client_state_machine_test id 22 [D 20:01:03.800704] PINT_client_state_machine_test id 22 [D 20:01:03.812703] PINT_client_state_machine_test id 22 [D 20:01:03.824704] PINT_client_state_machine_test id 22 [D 20:01:03.836703] PINT_client_state_machine_test id 22 [D 20:01:03.848704] PINT_client_state_machine_test id 22 [D 20:01:03.860703] PINT_client_state_machine_test id 22 [D 20:01:03.872703] PINT_client_state_machine_test id 22 [D 20:01:03.884703] PINT_client_state_machine_test id 22 [D 20:01:03.896704] PINT_client_state_machine_test id 22 [D 20:01:03.908706] PINT_client_state_machine_test id 22 [D 20:01:03.920704] PINT_client_state_machine_test id 22 [D 20:01:03.932703] PINT_client_state_machine_test id 22 [D 20:01:03.944705] PINT_client_state_machine_test id 22 [D 20:01:03.956704] PINT_client_state_machine_test id 22 [D 20:01:03.968704] PINT_client_state_machine_test id 22 [D 20:01:03.980705] PINT_client_state_machine_test id 22 [D 20:01:03.992706] PINT_client_state_machine_test id 22 [D 20:01:04.004704] PINT_client_state_machine_test id 22 [D 20:01:04.016705] PINT_client_state_machine_test id 22 [D 20:01:04.028704] PINT_client_state_machine_test id 22 [D 20:01:04.040705] PINT_client_state_machine_test id 22 [D 20:01:04.052704] PINT_client_state_machine_test id 22 [D 20:01:04.064704] PINT_client_state_machine_test id 22 [D 20:01:04.076704] PINT_client_state_machine_test id 22 [D 20:01:04.088705] PINT_client_state_machine_test id 22 [D 20:01:04.100705] PINT_client_state_machine_test id 22 [D 20:01:04.112707] PINT_client_state_machine_test id 22 [D 20:01:04.124706] PINT_client_state_machine_test id 22 [D 20:01:04.136706] PINT_client_state_machine_test id 22 [D 20:01:04.148705] PINT_client_state_machine_test id 22 [D 20:01:04.160705] PINT_client_state_machine_test id 22 [D 20:01:04.172705] PINT_client_state_machine_test id 22 [D 20:01:04.184705] PINT_client_state_machine_test id 22 [D 20:01:04.196705] PINT_client_state_machine_test id 22 [D 20:01:04.208705] PINT_client_state_machine_test id 22 -[D 20:01:04.209762] openib_check_cq: completed recv wr_id 15e735a8. [D 20:01:04.209813] openib_post_rr: da13:3346 bh 8 wr_id 15e735a8. [D 20:01:04.209972] openib_check_cq: completed recv wr_id 15e735c0. [D 20:01:04.210000] openib_post_rr: da13:3346 bh 9 wr_id 15e735c0. [D 20:01:04.210027] openib_check_cq: completed recv wr_id 15e735d8. [D 20:01:04.210056] openib_post_rr: da13:3346 bh 10 wr_id 15e735d8. [D 20:01:04.218740] PINT_client_state_machine_test id 22 [D 20:01:04.223626] openib_check_cq: completed recv wr_id 15e735f0. [D 20:01:04.223654] openib_post_rr: da13:3346 bh 11 wr_id 15e735f0. [D 20:01:04.223682] post_sr: da13:3346 bh 81 len 48 wr_id 15e73318. [D 20:01:04.223709] openib_check_cq: completed recv wr_id 15e73608. [D 20:01:04.223736] openib_post_rr: da13:3346 bh 12 wr_id 15e73608. [D 20:01:04.223763] openib_check_cq: completed recv wr_id 15e73620. [D 20:01:04.223791] openib_post_rr: da13:3346 bh 13 wr_id 15e73620. [D 20:01:04.223817] openib_check_cq: completed recv wr_id 15e73638. [D 20:01:04.223845] openib_post_rr: da13:3346 bh 14 wr_id 15e73638. [D 20:01:04.223872] openib_check_cq: completed recv wr_id 15e73650. [D 20:01:04.223899] openib_post_rr: da13:3346 bh 15 wr_id 15e73650. [D 20:01:04.223925] openib_check_cq: completed recv wr_id 15e73668. [D 20:01:04.223953] openib_post_rr: da13:3346 bh 16 wr_id 15e73668. [D 20:01:04.223979] openib_check_cq: completed recv wr_id 15e73680. [D 20:01:04.224006] openib_post_rr: da13:3346 bh 17 wr_id 15e73680. [D 20:01:04.224033] openib_check_cq: completed send wr_id 15e73318. [D 20:01:04.224064] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:04.224091] ========================================================= [D 20:01:04.224116] PINT_process_request [D 20:01:04.224141] tiling 2 copies [D 20:01:04.224164] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.224191] lvl 0 el 0 blk 0 by 31457280 [D 20:01:04.224216] to 31457280 ta 0 fi 33554432 [D 20:01:04.224242] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:04.224266] basic type or contiguous data [D 20:01:04.224295] Request disp [D 20:01:04.224319] contig_offset = 31457280 contig_size = 35651584 lvl_flag = 1 [D 20:01:04.224345] calling distribute [D 20:01:04.224370] PINT_distribute [D 20:01:04.224393] of 31457280 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:04.224420] begin iteration loff: 31457280 [D 20:01:04.224446] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:04.224471] **********CALL***PROCESS********* [D 20:01:04.224494] segment of 31457280 sz 2097152 [D 20:01:04.224519] PINT_process_request [D 20:01:04.224543] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.224570] lvl 0 el 0 blk 0 by 31457280 [D 20:01:04.224595] to 31457280 ta 31457280 fi 33554432 [D 20:01:04.224618] basic type or contiguous data [D 20:01:04.224642] Request disp [D 20:01:04.224666] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:04.224691] process a segment [D 20:01:04.224715] of 31457280 sz 2097152 [D 20:01:04.224739] add a segment 0 [D 20:01:04.224762] retval = 2097152 [D 20:01:04.224786] return from level -1 [D 20:01:04.224810] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:04.224837] *****RETURN***FROM***PROCESS***** [D 20:01:04.224861] end iteration [D 20:01:04.224884] done with segments or bytes [D 20:01:04.224908] of 33554432 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.224934] next loff: 33554432 next poff: 33554432 [D 20:01:04.224959] (return value) 2097152 [D 20:01:04.224983] finished [D 20:01:04.225007] retval = 2097152 [D 20:01:04.225031] exiting distribute returned less than expected [D 20:01:04.225055] done sg 1 sm 64 by 2097152 bm 2097152 ta 0 to 33554432 fo 33554432 eof 0 [D 20:01:04.225081] ========================================================= [D 20:01:04.225106] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 23 [D 20:01:04.225131] element 0: offset: 0xa4251000, size: 2097152 [D 20:01:04.225163] post_sr: da13:3346 bh 82 len 48 wr_id 15e73330. [D 20:01:04.225192] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:04.225220] ========================================================= [D 20:01:04.225245] PINT_process_request [D 20:01:04.225269] tiling 4 copies [D 20:01:04.225293] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.225319] lvl 0 el 0 blk 0 by 92274688 [D 20:01:04.225344] to 92274688 ta 67108864 fi 100663296 [D 20:01:04.225369] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:04.225394] basic type or contiguous data [D 20:01:04.225418] Request disp [D 20:01:04.225441] contig_offset = 92274688 contig_size = 41943040 lvl_flag = 1 [D 20:01:04.225466] calling distribute [D 20:01:04.225490] PINT_distribute [D 20:01:04.225514] of 92274688 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:04.225541] begin iteration loff: 92274688 [D 20:01:04.225565] segment exceeds byte limit [D 20:01:04.225590] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:04.225615] **********CALL***PROCESS********* [D 20:01:04.225638] segment of 25165824 sz 2097152 [D 20:01:04.225663] PINT_process_request [D 20:01:04.225687] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.225713] lvl 0 el 0 blk 0 by 25165824 [D 20:01:04.225738] to 25165824 ta 25165824 fi 27262976 [D 20:01:04.225765] basic type or contiguous data [D 20:01:04.225789] Request disp [D 20:01:04.225812] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:04.225837] process a segment [D 20:01:04.225861] of 25165824 sz 2097152 [D 20:01:04.225885] add a segment 0 [D 20:01:04.225909] retval = 2097152 [D 20:01:04.225933] exiting distribute returned less than expected [D 20:01:04.225956] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:04.225983] *****RETURN***FROM***PROCESS***** [D 20:01:04.226007] end iteration [D 20:01:04.226031] done with segments or bytes [D 20:01:04.226058] of 94371840 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.226083] next loff: 94371840 next poff: 94371840 [D 20:01:04.226108] return value 2097152 [D 20:01:04.226133] finished [D 20:01:04.226156] retval = 2097152 [D 20:01:04.226180] exiting distribute returned less than expected [D 20:01:04.226204] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 94371840 fo 100663296 eof 0 [D 20:01:04.226230] ========================================================= [D 20:01:04.226255] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:04.226280] element 0: offset: 0x9fc4d000, size: 2097152 [D 20:01:04.226306] openib_check_cq: completed send wr_id 15e73330. [D 20:01:04.226336] post_sr: da13:3346 bh 83 len 48 wr_id 15e73348. [D 20:01:04.226370] openib_check_cq: completed send wr_id 15e73348. [D 20:01:04.228774] PINT_client_state_machine_test id 22 [D 20:01:04.238803] PINT_client_state_machine_test id 22 [D 20:01:04.248833] PINT_client_state_machine_test id 22 [D 20:01:04.258862] PINT_client_state_machine_test id 22 [D 20:01:04.268892] PINT_client_state_machine_test id 22 [D 20:01:04.289208] PINT_client_state_machine_test id 22 [D 20:01:04.301206] PINT_client_state_machine_test id 22 [D 20:01:04.313206] PINT_client_state_machine_test id 22 [D 20:01:04.325206] PINT_client_state_machine_test id 22 [D 20:01:04.337206] PINT_client_state_machine_test id 22 -[D 20:01:04.342262] openib_check_cq: completed recv wr_id 15e73698. [D 20:01:04.342298] openib_post_rr: da13:3346 bh 18 wr_id 15e73698. [D 20:01:04.342326] openib_check_cq: completed recv wr_id 15e736b0. [D 20:01:04.342352] openib_post_rr: da13:3346 bh 19 wr_id 15e736b0. [D 20:01:04.342379] openib_check_cq: completed recv wr_id 15e736c8. [D 20:01:04.342406] openib_post_rr: da13:3346 bh 20 wr_id 15e736c8. [D 20:01:04.342432] openib_check_cq: completed recv wr_id 15e736e0. [D 20:01:04.342459] openib_post_rr: da13:3346 bh 21 wr_id 15e736e0. [D 20:01:04.342493] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:04.342519] ========================================================= [D 20:01:04.342544] PINT_process_request [D 20:01:04.342568] tiling 5 copies [D 20:01:04.342592] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.342617] lvl 0 el 0 blk 0 by 111149056 [D 20:01:04.342642] to 111149056 ta 100663296 fi 134217728 [D 20:01:04.342667] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:04.342692] basic type or contiguous data [D 20:01:04.342716] Request disp [D 20:01:04.342741] contig_offset = 111149056 contig_size = 56623104 lvl_flag = 1 [D 20:01:04.342765] calling distribute [D 20:01:04.342789] PINT_distribute [D 20:01:04.342813] of 111149056 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:04.342840] begin iteration loff: 111149056 [D 20:01:04.342864] segment exceeds byte limit [D 20:01:04.342888] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:04.342913] **********CALL***PROCESS********* [D 20:01:04.342937] segment of 10485760 sz 2097152 [D 20:01:04.342962] PINT_process_request [D 20:01:04.342986] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.343011] lvl 0 el 0 blk 0 by 10485760 [D 20:01:04.343036] to 10485760 ta 10485760 fi 12582912 [D 20:01:04.343061] basic type or contiguous data [D 20:01:04.343085] Request disp [D 20:01:04.343109] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:04.343133] process a segment [D 20:01:04.343157] of 10485760 sz 2097152 [D 20:01:04.343181] add a segment 0 [D 20:01:04.343205] retval = 2097152 [D 20:01:04.343229] exiting distribute returned less than expected [D 20:01:04.343253] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:04.343279] *****RETURN***FROM***PROCESS***** [D 20:01:04.343303] end iteration [D 20:01:04.343327] done with segments or bytes [D 20:01:04.343351] of 113246208 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.343377] next loff: 113246208 next poff: 113246208 [D 20:01:04.343403] return value 2097152 [D 20:01:04.343428] finished [D 20:01:04.343452] retval = 2097152 [D 20:01:04.343476] exiting distribute returned less than expected [D 20:01:04.343501] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 113246208 fo 134217728 eof 0 [D 20:01:04.343526] ========================================================= [D 20:01:04.343551] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:04.343576] element 0: offset: 0x9ce4b000, size: 2097152 [D 20:01:04.343606] post_sr: da13:3346 bh 84 len 48 wr_id 15e73360. [D 20:01:04.343634] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15edcd38. [D 20:01:04.343660] flowproto completing 0x15edcd38 [D 20:01:04.343687] Job flows in progress (callback time): 14 [D 20:01:04.343715] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:04.343741] ========================================================= [D 20:01:04.343766] PINT_process_request [D 20:01:04.343790] tiling 3 copies [D 20:01:04.343813] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.343839] lvl 0 el 0 blk 0 by 62914560 [D 20:01:04.343865] to 62914560 ta 33554432 fi 67108864 [D 20:01:04.343889] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:04.343914] basic type or contiguous data [D 20:01:04.343937] Request disp [D 20:01:04.343961] contig_offset = 62914560 contig_size = 37748736 lvl_flag = 1 [D 20:01:04.343986] calling distribute [D 20:01:04.344011] PINT_distribute [D 20:01:04.344034] of 62914560 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:04.344060] begin iteration loff: 62914560 [D 20:01:04.344085] segment exceeds byte limit [D 20:01:04.344109] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:04.344134] **********CALL***PROCESS********* [D 20:01:04.344158] segment of 29360128 sz 2097152 [D 20:01:04.344182] PINT_process_request [D 20:01:04.344206] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.344232] lvl 0 el 0 blk 0 by 29360128 [D 20:01:04.344257] to 29360128 ta 29360128 fi 31457280 [D 20:01:04.344282] basic type or contiguous data [D 20:01:04.344305] Request disp [D 20:01:04.344329] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:04.344354] process a segment [D 20:01:04.344378] of 29360128 sz 2097152 [D 20:01:04.344403] add a segment 0 [D 20:01:04.344427] retval = 2097152 [D 20:01:04.344450] exiting distribute returned less than expected [D 20:01:04.344474] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:04.344500] *****RETURN***FROM***PROCESS***** [D 20:01:04.344525] end iteration [D 20:01:04.344549] done with segments or bytes [D 20:01:04.344572] of 65011712 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.344597] next loff: 65011712 next poff: 65011712 [D 20:01:04.344622] return value 2097152 [D 20:01:04.344646] finished [D 20:01:04.344670] retval = 2097152 [D 20:01:04.344694] exiting distribute returned less than expected [D 20:01:04.344718] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 65011712 fo 67108864 eof 0 [D 20:01:04.344744] ========================================================= [D 20:01:04.344768] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:04.344794] element 0: offset: 0xa204f000, size: 2097152 [D 20:01:04.344820] openib_check_cq: completed send wr_id 15e73360. [D 20:01:04.344849] post_sr: da13:3346 bh 85 len 48 wr_id 15e73378. [D 20:01:04.344877] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:04.344902] ========================================================= [D 20:01:04.344927] PINT_process_request [D 20:01:04.344951] tiling 6 copies [D 20:01:04.344975] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.345002] lvl 0 el 0 blk 0 by 138412032 [D 20:01:04.345026] to 138412032 ta 134217728 fi 167772160 [D 20:01:04.345054] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:04.345079] basic type or contiguous data [D 20:01:04.345103] Request disp [D 20:01:04.345127] contig_offset = 138412032 contig_size = 62914560 lvl_flag = 1 [D 20:01:04.345151] calling distribute [D 20:01:04.345175] PINT_distribute [D 20:01:04.345199] of 138412032 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:04.345225] begin iteration loff: 138412032 [D 20:01:04.345250] segment exceeds byte limit [D 20:01:04.345274] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:04.345299] **********CALL***PROCESS********* [D 20:01:04.345322] segment of 4194304 sz 2097152 [D 20:01:04.345347] PINT_process_request [D 20:01:04.345371] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.345396] lvl 0 el 0 blk 0 by 4194304 [D 20:01:04.345421] to 4194304 ta 4194304 fi 6291456 [D 20:01:04.345446] basic type or contiguous data [D 20:01:04.345470] Request disp [D 20:01:04.345494] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:04.345519] process a segment [D 20:01:04.345542] of 4194304 sz 2097152 [D 20:01:04.345566] add a segment 0 [D 20:01:04.345590] retval = 2097152 [D 20:01:04.345614] exiting distribute returned less than expected [D 20:01:04.345638] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:04.345664] *****RETURN***FROM***PROCESS***** [D 20:01:04.345687] end iteration [D 20:01:04.345715] done with segments or bytes [D 20:01:04.345743] of 140509184 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.345772] next loff: 140509184 next poff: 140509184 [D 20:01:04.345800] return value 2097152 [D 20:01:04.345827] finished [D 20:01:04.345850] retval = 2097152 [D 20:01:04.345874] exiting distribute returned less than expected [D 20:01:04.345898] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 140509184 fo 167772160 eof 0 [D 20:01:04.345925] ========================================================= [D 20:01:04.345949] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:04.345973] element 0: offset: 0x9a849000, size: 2097152 [D 20:01:04.346000] openib_check_cq: completed send wr_id 15e73378. [D 20:01:04.346030] post_sr: da13:3346 bh 86 len 48 wr_id 15e73390. [D 20:01:04.346060] sm control op 6 [D 20:01:04.346085] [SM Entering]: (0x15edbf20) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:04.346112] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.346138] (0x15edbfb8) io_datafile_complete_operations (tag 2) [D 20:01:04.346162] matched completed flow for context 0x15edc788 [D 20:01:04.346188] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.346213] [SM Exiting]: (0x15edbf20) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.346239] [SM Entering]: (0x15edbf20) client_io_sm:io_analyze_results (status: 2) [D 20:01:04.346264] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.346289] (0x15edbfb8) io state: io_analyze_results [D 20:01:04.346315] - io_check_context_status called [D 20:01:04.346340] 33554432 bytes read from context 0x15edc788 [D 20:01:04.346364] [1/1] running size is 33554432 [D 20:01:04.346389] [2/1] running size is 33554432 [D 20:01:04.346421] no context errors found; using: Success [D 20:01:04.346446] total bytes transferred is 33554432 [D 20:01:04.346470] [SM Exiting]: (0x15edbf20) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.346497] [SM Entering]: (0x15edbf20) client_io_sm:io_cleanup (status: 2) [D 20:01:04.346522] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.346548] (0x15edbfb8) io state: io_cleanup [D 20:01:04.346575] [SM Exiting]: (0x15edbf20) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.346602] client_state_machine_terminate smcb 0x15edbf20 [D 20:01:04.346627] add smcb 0x15edbf20 to completion list [D 20:01:04.346652] PINT_client_state_machine_test id 22 [D 20:01:04.346683] openib_check_cq: completed send wr_id 15e73390. [D 20:01:04.356686] PINT_client_state_machine_test id 22 [D 20:01:04.366715] PINT_client_state_machine_test id 22 [D 20:01:04.376747] PINT_client_state_machine_test id 22 [D 20:01:04.386775] PINT_client_state_machine_test id 22 [D 20:01:04.403706] PINT_client_state_machine_test id 22 [D 20:01:04.415706] PINT_client_state_machine_test id 22 [D 20:01:04.427706] PINT_client_state_machine_test id 22 [D 20:01:04.439706] PINT_client_state_machine_test id 22 [D 20:01:04.439740] sm control op 300 [D 20:01:04.439764] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:01:04.439790] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:01:04.439819] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:01:04.439846] PINT_client_state_machine_test id 22 -[D 20:01:04.442501] openib_check_cq: completed recv wr_id 15e736f8. [D 20:01:04.442530] openib_post_rr: da13:3346 bh 22 wr_id 15e736f8. [D 20:01:04.442563] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:04.442588] ========================================================= [D 20:01:04.442613] PINT_process_request [D 20:01:04.442638] tiling 7 copies [D 20:01:04.442662] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.442688] lvl 0 el 0 blk 0 by 169869312 [D 20:01:04.442713] to 169869312 ta 167772160 fi 201326592 [D 20:01:04.442738] mto 2097152 mta 0 mfi 2097152 [D 20:01:04.442763] basic type or contiguous data [D 20:01:04.442787] Request disp [D 20:01:04.442811] contig_offset = 169869312 contig_size = 65011712 lvl_flag = 1 [D 20:01:04.442835] calling distribute [D 20:01:04.442859] PINT_distribute [D 20:01:04.442883] of 169869312 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:04.442910] begin iteration loff: 169869312 [D 20:01:04.442934] segment exceeds byte limit [D 20:01:04.442958] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:04.442983] **********CALL***PROCESS********* [D 20:01:04.443007] segment of 2097152 sz 2097152 [D 20:01:04.443032] PINT_process_request [D 20:01:04.443056] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.443082] lvl 0 el 0 blk 0 by 2097152 [D 20:01:04.443107] to 2097152 ta 2097152 fi 4194304 [D 20:01:04.443132] basic type or contiguous data [D 20:01:04.443156] Request disp [D 20:01:04.443180] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:04.443204] process a segment [D 20:01:04.443228] of 2097152 sz 2097152 [D 20:01:04.443252] add a segment 0 [D 20:01:04.443277] retval = 2097152 [D 20:01:04.443301] exiting distribute returned less than expected [D 20:01:04.443325] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:04.443350] *****RETURN***FROM***PROCESS***** [D 20:01:04.443375] end iteration [D 20:01:04.443399] done with segments or bytes [D 20:01:04.443422] of 171966464 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.443449] next loff: 171966464 next poff: 171966464 [D 20:01:04.443472] return value 2097152 [D 20:01:04.443497] finished [D 20:01:04.443520] retval = 2097152 [D 20:01:04.443545] exiting distribute returned less than expected [D 20:01:04.443569] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 171966464 fo 201326592 eof 0 [D 20:01:04.443594] ========================================================= [D 20:01:04.443619] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:04.443644] element 0: offset: 0x98647000, size: 2097152 [D 20:01:04.443675] post_sr: da13:3346 bh 87 len 48 wr_id 15e733a8. [D 20:01:04.443744] openib_check_cq: completed send wr_id 15e733a8. [D 20:01:04.446934] openib_check_cq: completed recv wr_id 15e73710. [D 20:01:04.446963] openib_post_rr: da13:3346 bh 23 wr_id 15e73710. [D 20:01:04.446990] openib_check_cq: completed recv wr_id 15e73728. [D 20:01:04.447016] openib_post_rr: da13:3346 bh 24 wr_id 15e73728. [D 20:01:04.447046] openib_check_cq: completed recv wr_id 15e73740. [D 20:01:04.447073] openib_post_rr: da13:3346 bh 25 wr_id 15e73740. [D 20:01:04.447099] openib_check_cq: completed recv wr_id 15e73758. [D 20:01:04.447127] openib_post_rr: da13:3346 bh 26 wr_id 15e73758. [D 20:01:04.447153] openib_check_cq: completed recv wr_id 15e73770. [D 20:01:04.447180] openib_post_rr: da13:3346 bh 27 wr_id 15e73770. [D 20:01:04.447207] openib_check_cq: completed recv wr_id 15e73788. [D 20:01:04.447233] openib_post_rr: da13:3346 bh 28 wr_id 15e73788. [D 20:01:04.447259] openib_check_cq: completed recv wr_id 15e737a0. [D 20:01:04.447287] openib_post_rr: da13:3346 bh 29 wr_id 15e737a0. [D 20:01:04.447313] openib_check_cq: completed recv wr_id 15e737b8. [D 20:01:04.447340] openib_post_rr: da13:3346 bh 30 wr_id 15e737b8. [D 20:01:04.447366] openib_check_cq: completed recv wr_id 15e737d0. [D 20:01:04.447393] openib_post_rr: da13:3346 bh 31 wr_id 15e737d0. [D 20:01:04.447421] post_sr: da13:3346 bh 88 len 48 wr_id 15e733c0. [D 20:01:04.447447] openib_check_cq: completed recv wr_id 15e737e8. [D 20:01:04.447474] openib_post_rr: da13:3346 bh 32 wr_id 15e737e8. [D 20:01:04.447500] openib_check_cq: completed recv wr_id 15e73800. [D 20:01:04.447527] openib_post_rr: da13:3346 bh 33 wr_id 15e73800. [D 20:01:04.447553] openib_check_cq: completed recv wr_id 15e73818. [D 20:01:04.447580] openib_post_rr: da13:3346 bh 34 wr_id 15e73818. [D 20:01:04.447606] openib_check_cq: completed send wr_id 15e733c0. [D 20:01:04.447638] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:04.447664] ========================================================= [D 20:01:04.447688] PINT_process_request [D 20:01:04.447715] tiling 5 copies [D 20:01:04.447743] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.447773] lvl 0 el 0 blk 0 by 113246208 [D 20:01:04.447802] to 113246208 ta 100663296 fi 134217728 [D 20:01:04.447829] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:04.447854] basic type or contiguous data [D 20:01:04.447877] Request disp [D 20:01:04.447901] contig_offset = 113246208 contig_size = 54525952 lvl_flag = 1 [D 20:01:04.447927] calling distribute [D 20:01:04.447951] PINT_distribute [D 20:01:04.447975] of 113246208 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:04.448001] begin iteration loff: 113246208 [D 20:01:04.448026] segment exceeds byte limit [D 20:01:04.448050] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:04.448074] **********CALL***PROCESS********* [D 20:01:04.448099] segment of 12582912 sz 2097152 [D 20:01:04.448123] PINT_process_request [D 20:01:04.448147] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.448173] lvl 0 el 0 blk 0 by 12582912 [D 20:01:04.448198] to 12582912 ta 12582912 fi 14680064 [D 20:01:04.448223] basic type or contiguous data [D 20:01:04.448246] Request disp [D 20:01:04.448270] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:04.448295] process a segment [D 20:01:04.448319] of 12582912 sz 2097152 [D 20:01:04.448343] add a segment 0 [D 20:01:04.448368] retval = 2097152 [D 20:01:04.448391] exiting distribute returned less than expected [D 20:01:04.448414] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:04.448441] *****RETURN***FROM***PROCESS***** [D 20:01:04.448465] end iteration [D 20:01:04.448489] done with segments or bytes [D 20:01:04.448513] of 115343360 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.448538] next loff: 115343360 next poff: 115343360 [D 20:01:04.448563] return value 2097152 [D 20:01:04.448587] finished [D 20:01:04.448611] retval = 2097152 [D 20:01:04.448635] exiting distribute returned less than expected [D 20:01:04.448659] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 115343360 fo 134217728 eof 0 [D 20:01:04.448685] ========================================================= [D 20:01:04.448709] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:04.448738] element 0: offset: 0x9d04b000, size: 2097152 [D 20:01:04.448767] post_sr: da13:3346 bh 89 len 48 wr_id 15e733d8. [D 20:01:04.448794] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:04.448820] ========================================================= [D 20:01:04.448844] PINT_process_request [D 20:01:04.448868] tiling 4 copies [D 20:01:04.448893] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.448919] lvl 0 el 0 blk 0 by 94371840 [D 20:01:04.448945] to 94371840 ta 67108864 fi 100663296 [D 20:01:04.448969] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:04.448994] basic type or contiguous data [D 20:01:04.449018] Request disp [D 20:01:04.449042] contig_offset = 94371840 contig_size = 39845888 lvl_flag = 1 [D 20:01:04.449067] calling distribute [D 20:01:04.449090] PINT_distribute [D 20:01:04.449114] of 94371840 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:04.449141] begin iteration loff: 94371840 [D 20:01:04.449166] segment exceeds byte limit [D 20:01:04.449190] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:04.449214] **********CALL***PROCESS********* [D 20:01:04.449238] segment of 27262976 sz 2097152 [D 20:01:04.449263] PINT_process_request [D 20:01:04.449287] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.449314] lvl 0 el 0 blk 0 by 27262976 [D 20:01:04.449338] to 27262976 ta 27262976 fi 29360128 [D 20:01:04.449363] basic type or contiguous data [D 20:01:04.449387] Request disp [D 20:01:04.449411] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:04.449436] process a segment [D 20:01:04.449460] of 27262976 sz 2097152 [D 20:01:04.449484] add a segment 0 [D 20:01:04.449508] retval = 2097152 [D 20:01:04.449532] exiting distribute returned less than expected [D 20:01:04.449556] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:04.449583] *****RETURN***FROM***PROCESS***** [D 20:01:04.449607] end iteration [D 20:01:04.449630] done with segments or bytes [D 20:01:04.449654] of 96468992 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.449680] next loff: 96468992 next poff: 96468992 [D 20:01:04.449707] return value 2097152 [D 20:01:04.449731] finished [D 20:01:04.449756] retval = 2097152 [D 20:01:04.449780] exiting distribute returned less than expected [D 20:01:04.449804] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 96468992 fo 100663296 eof 0 [D 20:01:04.449830] ========================================================= [D 20:01:04.449854] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:04.449880] element 0: offset: 0x9fe4d000, size: 2097152 [D 20:01:04.449906] openib_check_cq: completed send wr_id 15e733d8. [D 20:01:04.449932] openib_check_cq: completed recv wr_id 15e73830. [D 20:01:04.449957] openib_post_rr: da13:3346 bh 35 wr_id 15e73830. [D 20:01:04.449984] post_sr: da13:3346 bh 90 len 48 wr_id 15e733f0. [D 20:01:04.450011] openib_check_cq: completed recv wr_id 15e73848. [D 20:01:04.450038] openib_post_rr: da13:3346 bh 36 wr_id 15e73848. [D 20:01:04.450065] openib_check_cq: completed recv wr_id 15e73860. [D 20:01:04.450093] openib_post_rr: da13:3346 bh 37 wr_id 15e73860. [D 20:01:04.450119] openib_check_cq: completed recv wr_id 15e73878. [D 20:01:04.450146] openib_post_rr: da13:3346 bh 38 wr_id 15e73878. [D 20:01:04.450173] openib_check_cq: completed recv wr_id 15e73890. [D 20:01:04.450200] openib_post_rr: da13:3346 bh 39 wr_id 15e73890. [D 20:01:04.450226] openib_check_cq: completed recv wr_id 15e738a8. [D 20:01:04.450252] openib_post_rr: da13:3346 bh 40 wr_id 15e738a8. [D 20:01:04.450279] openib_check_cq: completed recv wr_id 15e738c0. [D 20:01:04.450304] openib_post_rr: da13:3346 bh 41 wr_id 15e738c0. [D 20:01:04.450330] openib_check_cq: completed recv wr_id 15e738d8. [D 20:01:04.450356] openib_post_rr: da13:3346 bh 42 wr_id 15e738d8. [D 20:01:04.450382] openib_check_cq: completed recv wr_id 15e738f0. [D 20:01:04.450411] openib_post_rr: da13:3346 bh 43 wr_id 15e738f0. [D 20:01:04.450437] openib_check_cq: completed recv wr_id 15e73908. [D 20:01:04.450464] openib_post_rr: da13:3346 bh 44 wr_id 15e73908. [D 20:01:04.450489] openib_check_cq: completed recv wr_id 15e73920. [D 20:01:04.450515] openib_post_rr: da13:3346 bh 45 wr_id 15e73920. [D 20:01:04.450542] openib_check_cq: completed recv wr_id 15e73938. [D 20:01:04.450568] openib_post_rr: da13:3346 bh 46 wr_id 15e73938. [D 20:01:04.450593] openib_check_cq: completed recv wr_id 15e73950. [D 20:01:04.450619] openib_post_rr: da13:3346 bh 47 wr_id 15e73950. [D 20:01:04.450646] openib_check_cq: completed recv wr_id 15e73968. [D 20:01:04.450672] openib_post_rr: da13:3346 bh 48 wr_id 15e73968. [D 20:01:04.450698] openib_check_cq: completed recv wr_id 15e73980. [D 20:01:04.450724] openib_post_rr: da13:3346 bh 49 wr_id 15e73980. [D 20:01:04.450751] openib_check_cq: completed recv wr_id 15e73998. [D 20:01:04.450777] openib_post_rr: da13:3346 bh 50 wr_id 15e73998. [D 20:01:04.450803] openib_check_cq: completed recv wr_id 15e739b0. [D 20:01:04.450829] openib_post_rr: da13:3346 bh 51 wr_id 15e739b0. [D 20:01:04.450856] openib_check_cq: completed recv wr_id 15e739c8. [D 20:01:04.450881] openib_post_rr: da13:3346 bh 52 wr_id 15e739c8. [D 20:01:04.450908] openib_check_cq: completed recv wr_id 15e739e0. [D 20:01:04.450934] openib_post_rr: da13:3346 bh 53 wr_id 15e739e0. [D 20:01:04.450961] openib_check_cq: completed recv wr_id 15e739f8. [D 20:01:04.450988] openib_post_rr: da13:3346 bh 54 wr_id 15e739f8. [D 20:01:04.451014] openib_check_cq: completed recv wr_id 15e73a10. [D 20:01:04.451041] openib_post_rr: da13:3346 bh 55 wr_id 15e73a10. [D 20:01:04.451066] openib_check_cq: completed recv wr_id 15e73a28. [D 20:01:04.451094] openib_post_rr: da13:3346 bh 56 wr_id 15e73a28. [D 20:01:04.451120] openib_check_cq: completed recv wr_id 15e73a40. [D 20:01:04.451148] openib_post_rr: da13:3346 bh 57 wr_id 15e73a40. [D 20:01:04.451174] openib_check_cq: completed recv wr_id 15e73a58. [D 20:01:04.451202] openib_post_rr: da13:3346 bh 58 wr_id 15e73a58. [D 20:01:04.451228] openib_check_cq: completed recv wr_id 15e73a70. [D 20:01:04.451255] openib_post_rr: da13:3346 bh 59 wr_id 15e73a70. [D 20:01:04.451282] openib_check_cq: completed recv wr_id 15e73a88. [D 20:01:04.451310] openib_post_rr: da13:3346 bh 60 wr_id 15e73a88. [D 20:01:04.451335] openib_check_cq: completed recv wr_id 15e73aa0. [D 20:01:04.451363] openib_post_rr: da13:3346 bh 61 wr_id 15e73aa0. [D 20:01:04.451390] openib_check_cq: completed recv wr_id 15e73ab8. [D 20:01:04.451417] openib_post_rr: da13:3346 bh 62 wr_id 15e73ab8. [D 20:01:04.451444] openib_check_cq: completed recv wr_id 15e73ad0. [D 20:01:04.451470] openib_post_rr: da13:3346 bh 63 wr_id 15e73ad0. [D 20:01:04.451497] post_sr: da13:3346 bh 91 len 48 wr_id 15e73408. [D 20:01:04.451523] openib_check_cq: completed recv wr_id 15e73ae8. [D 20:01:04.451551] openib_post_rr: da13:3346 bh 64 wr_id 15e73ae8. [D 20:01:04.451577] openib_check_cq: completed recv wr_id 15e73b00. [D 20:01:04.451604] openib_post_rr: da13:3346 bh 65 wr_id 15e73b00. [D 20:01:04.451631] openib_check_cq: completed recv wr_id 15e73b18. [D 20:01:04.451658] openib_post_rr: da13:3346 bh 66 wr_id 15e73b18. [D 20:01:04.451684] openib_check_cq: completed recv wr_id 15e73b30. [D 20:01:04.451715] openib_post_rr: da13:3346 bh 67 wr_id 15e73b30. [D 20:01:04.451745] openib_check_cq: completed recv wr_id 15e73b48. [D 20:01:04.451776] openib_post_rr: da13:3346 bh 68 wr_id 15e73b48. [D 20:01:04.451807] openib_check_cq: completed send wr_id 15e733f0. [D 20:01:04.451833] openib_check_cq: completed send wr_id 15e73408. [D 20:01:04.451862] post_sr: da13:3346 bh 92 len 48 wr_id 15e73420. [D 20:01:04.451889] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb1f38. [D 20:01:04.451915] flowproto completing 0x15eb1f38 [D 20:01:04.451941] Job flows in progress (callback time): 13 [D 20:01:04.451970] sm control op 6 [D 20:01:04.451993] [SM Entering]: (0x15e75b78) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.452022] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.452047] (0x15e62108) io_datafile_complete_operations (tag 3) [D 20:01:04.452073] matched completed ack for context 0x15e74cc8 [D 20:01:04.452097] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.452122] [SM Exiting]: (0x15e75b78) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.452148] [SM Entering]: (0x15e75b78) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.452173] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.452198] (0x15e62108) io state: io_analyze_results [D 20:01:04.452222] - io_check_context_status called [D 20:01:04.452247] PINT_decode [D 20:01:04.452272] lebf_decode_resp [D 20:01:04.452304] 33554432 bytes written to context 0x15e74cc8 [D 20:01:04.452330] PINT_decode_release [D 20:01:04.452354] lebf_decode_rel [D 20:01:04.452381] [1/1] running size is 33554432 [D 20:01:04.452405] [2/1] running size is 33554432 [D 20:01:04.452432] no context errors found; using: Success [D 20:01:04.452457] total bytes transferred is 33554432 [D 20:01:04.452482] acache: invalidate_size(): H=146 [D 20:01:04.452510] [SM Exiting]: (0x15e75b78) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.452537] [SM Entering]: (0x15e75b78) client_io_sm:io_cleanup (status: 3) [D 20:01:04.452562] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.452588] (0x15e62108) io state: io_cleanup [D 20:01:04.452613] [SM Exiting]: (0x15e75b78) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.452639] client_state_machine_terminate smcb 0x15e75b78 [D 20:01:04.452664] add smcb 0x15e75b78 to completion list [D 20:01:04.452689] sm control op 6 [D 20:01:04.452713] [SM Entering]: (0x15eb1140) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:04.452738] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.452763] (0x15eb11d8) io_datafile_complete_operations (tag 2) [D 20:01:04.452789] matched completed flow for context 0x15eb1988 [D 20:01:04.452814] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.452838] [SM Exiting]: (0x15eb1140) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.452863] [SM Entering]: (0x15eb1140) client_io_sm:io_analyze_results (status: 2) [D 20:01:04.452889] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.452914] (0x15eb11d8) io state: io_analyze_results [D 20:01:04.452939] - io_check_context_status called [D 20:01:04.452962] 33554432 bytes read from context 0x15eb1988 [D 20:01:04.452987] [1/1] running size is 33554432 [D 20:01:04.453012] [2/1] running size is 33554432 [D 20:01:04.453039] no context errors found; using: Success [D 20:01:04.453063] total bytes transferred is 33554432 [D 20:01:04.453087] [SM Exiting]: (0x15eb1140) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.453113] [SM Entering]: (0x15eb1140) client_io_sm:io_cleanup (status: 2) [D 20:01:04.453138] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.453163] (0x15eb11d8) io state: io_cleanup [D 20:01:04.453188] [SM Exiting]: (0x15eb1140) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.453214] client_state_machine_terminate smcb 0x15eb1140 [D 20:01:04.453239] add smcb 0x15eb1140 to completion list [D 20:01:04.453264] [SM frame get]: (0x15e75b78) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32550 curbuf 17 buffer_ahead_offset 16 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 2 aio_req 0x15e74490 b->offset 33554432 b->size 33554432 [D 20:01:04.453328] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.453354] PINT_client_state_machine_test id 25 [D 20:01:04.453382] openib_check_cq: completed send wr_id 15e73420. [D 20:01:04.453430] sm control op 6 [D 20:01:04.453454] [SM Entering]: (0x15e75f20) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.453485] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.453512] (0x15e75fb8) io_datafile_complete_operations (tag 3) [D 20:01:04.453536] matched completed ack for context 0x15e76730 [D 20:01:04.453561] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.453585] [SM Exiting]: (0x15e75f20) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.453611] [SM Entering]: (0x15e75f20) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.453637] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.453661] (0x15e75fb8) io state: io_analyze_results [D 20:01:04.453686] - io_check_context_status called [D 20:01:04.453713] PINT_decode [D 20:01:04.453738] lebf_decode_resp [D 20:01:04.453762] 33554432 bytes written to context 0x15e76730 [D 20:01:04.453786] PINT_decode_release [D 20:01:04.453810] lebf_decode_rel [D 20:01:04.453836] [1/1] running size is 33554432 [D 20:01:04.453862] [2/1] running size is 33554432 [D 20:01:04.453887] no context errors found; using: Success [D 20:01:04.453912] total bytes transferred is 33554432 [D 20:01:04.453937] acache: invalidate_size(): H=146 [D 20:01:04.453963] [SM Exiting]: (0x15e75f20) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.453989] [SM Entering]: (0x15e75f20) client_io_sm:io_cleanup (status: 3) [D 20:01:04.454014] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.454039] (0x15e75fb8) io state: io_cleanup [D 20:01:04.454065] [SM Exiting]: (0x15e75f20) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.454091] client_state_machine_terminate smcb 0x15e75f20 [D 20:01:04.454115] add smcb 0x15e75f20 to completion list [D 20:01:04.454140] sm control op 6 [D 20:01:04.454164] [SM Entering]: (0x15e85bb0) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.454190] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.454216] (0x15e85c48) io_datafile_complete_operations (tag 3) [D 20:01:04.454240] matched completed ack for context 0x15e86408 [D 20:01:04.454264] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.454289] [SM Exiting]: (0x15e85bb0) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.454315] [SM Entering]: (0x15e85bb0) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.454340] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.454365] (0x15e85c48) io state: io_analyze_results [D 20:01:04.454390] - io_check_context_status called [D 20:01:04.454414] PINT_decode [D 20:01:04.454438] lebf_decode_resp [D 20:01:04.454463] 33554432 bytes written to context 0x15e86408 [D 20:01:04.454486] PINT_decode_release [D 20:01:04.454510] lebf_decode_rel [D 20:01:04.454537] [1/1] running size is 33554432 [D 20:01:04.454562] [2/1] running size is 33554432 [D 20:01:04.454588] no context errors found; using: Success [D 20:01:04.454613] total bytes transferred is 33554432 [D 20:01:04.454637] acache: invalidate_size(): H=146 [D 20:01:04.454663] [SM Exiting]: (0x15e85bb0) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.454689] [SM Entering]: (0x15e85bb0) client_io_sm:io_cleanup (status: 3) [D 20:01:04.454714] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.454739] (0x15e85c48) io state: io_cleanup [D 20:01:04.454765] [SM Exiting]: (0x15e85bb0) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.454791] client_state_machine_terminate smcb 0x15e85bb0 [D 20:01:04.454815] add smcb 0x15e85bb0 to completion list [D 20:01:04.454840] sm control op 6 [D 20:01:04.454865] [SM Entering]: (0x15e88fa0) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.454890] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.454916] (0x15e89038) io_datafile_complete_operations (tag 3) [D 20:01:04.454940] matched completed ack for context 0x15e897e8 [D 20:01:04.454967] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.454992] [SM Exiting]: (0x15e88fa0) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.455018] [SM Entering]: (0x15e88fa0) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.455043] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.455068] (0x15e89038) io state: io_analyze_results [D 20:01:04.455093] - io_check_context_status called [D 20:01:04.455118] PINT_decode [D 20:01:04.455142] lebf_decode_resp [D 20:01:04.455166] 33554432 bytes written to context 0x15e897e8 [D 20:01:04.455191] PINT_decode_release [D 20:01:04.455215] lebf_decode_rel [D 20:01:04.455241] [1/1] running size is 33554432 [D 20:01:04.455265] [2/1] running size is 33554432 [D 20:01:04.455292] no context errors found; using: Success [D 20:01:04.455317] total bytes transferred is 33554432 [D 20:01:04.455341] acache: invalidate_size(): H=146 [D 20:01:04.455367] [SM Exiting]: (0x15e88fa0) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.455392] [SM Entering]: (0x15e88fa0) client_io_sm:io_cleanup (status: 3) [D 20:01:04.455418] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.455443] (0x15e89038) io state: io_cleanup [D 20:01:04.455470] [SM Exiting]: (0x15e88fa0) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.455495] client_state_machine_terminate smcb 0x15e88fa0 [D 20:01:04.455520] add smcb 0x15e88fa0 to completion list [D 20:01:04.455545] sm control op 6 [D 20:01:04.455570] [SM Entering]: (0x15e8c340) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.455595] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.455620] (0x15e8c3d8) io_datafile_complete_operations (tag 3) [D 20:01:04.455645] matched completed ack for context 0x15e8cbe0 [D 20:01:04.455670] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.455695] [SM Exiting]: (0x15e8c340) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.455725] [SM Entering]: (0x15e8c340) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.455754] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.455783] (0x15e8c3d8) io state: io_analyze_results [D 20:01:04.455811] - io_check_context_status called [D 20:01:04.455836] PINT_decode [D 20:01:04.455860] lebf_decode_resp [D 20:01:04.455884] 33554432 bytes written to context 0x15e8cbe0 [D 20:01:04.455909] PINT_decode_release [D 20:01:04.455933] lebf_decode_rel [D 20:01:04.455959] [1/1] running size is 33554432 [D 20:01:04.455984] [2/1] running size is 33554432 [D 20:01:04.456010] no context errors found; using: Success [D 20:01:04.456035] total bytes transferred is 33554432 [D 20:01:04.456060] acache: invalidate_size(): H=146 [D 20:01:04.456086] [SM Exiting]: (0x15e8c340) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.456111] [SM Entering]: (0x15e8c340) client_io_sm:io_cleanup (status: 3) [D 20:01:04.456136] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.456162] (0x15e8c3d8) io state: io_cleanup [D 20:01:04.456188] [SM Exiting]: (0x15e8c340) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.456213] client_state_machine_terminate smcb 0x15e8c340 [D 20:01:04.456238] add smcb 0x15e8c340 to completion list [D 20:01:04.456263] sm control op 6 [D 20:01:04.456287] [SM Entering]: (0x15e8f790) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.456313] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.456337] (0x15e8f828) io_datafile_complete_operations (tag 3) [D 20:01:04.456363] matched completed ack for context 0x15e8ffd8 [D 20:01:04.456388] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.456413] [SM Exiting]: (0x15e8f790) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.456439] [SM Entering]: (0x15e8f790) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.456466] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.456492] (0x15e8f828) io state: io_analyze_results [D 20:01:04.456517] - io_check_context_status called [D 20:01:04.456541] PINT_decode [D 20:01:04.456565] lebf_decode_resp [D 20:01:04.456589] 33554432 bytes written to context 0x15e8ffd8 [D 20:01:04.456614] PINT_decode_release [D 20:01:04.456638] lebf_decode_rel [D 20:01:04.456664] [1/1] running size is 33554432 [D 20:01:04.456688] [2/1] running size is 33554432 [D 20:01:04.456715] no context errors found; using: Success [D 20:01:04.456740] total bytes transferred is 33554432 [D 20:01:04.456765] acache: invalidate_size(): H=146 [D 20:01:04.456790] [SM Exiting]: (0x15e8f790) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.456816] [SM Entering]: (0x15e8f790) client_io_sm:io_cleanup (status: 3) [D 20:01:04.456841] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.456867] (0x15e8f828) io state: io_cleanup [D 20:01:04.456892] [SM Exiting]: (0x15e8f790) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.456918] client_state_machine_terminate smcb 0x15e8f790 [D 20:01:04.456942] add smcb 0x15e8f790 to completion list [D 20:01:04.456967] [SM frame get]: (0x15e75f20) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32578 curbuf 17 buffer_ahead_offset 17 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 3 aio_req 0x15e75570 b->offset 67108864 b->size 33554432 [D 20:01:04.457004] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.457030] PINT_client_state_machine_test id 28 [D 20:01:04.457055] [SM frame get]: (0x15e85bb0) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e325a0 curbuf 17 buffer_ahead_offset 18 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 4 aio_req 0x15e74440 b->offset 100663296 b->size 33554432 [D 20:01:04.457091] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.457117] PINT_client_state_machine_test id 31 [D 20:01:04.457142] [SM frame get]: (0x15e88fa0) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e325c8 curbuf 17 buffer_ahead_offset 19 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 5 aio_req 0x15e741f8 b->offset 134217728 b->size 33554432 [D 20:01:04.457180] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.457205] PINT_client_state_machine_test id 34 [D 20:01:04.457230] [SM frame get]: (0x15e8c340) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e325f0 curbuf 17 buffer_ahead_offset 20 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 6 aio_req 0x15e74218 b->offset 167772160 b->size 33554432 [D 20:01:04.457266] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.457292] PINT_client_state_machine_test id 37 [D 20:01:04.457317] [SM frame get]: (0x15e8f790) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32618 curbuf 17 buffer_ahead_offset 21 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 7 aio_req 0x15e74238 b->offset 201326592 b->size 33554432 [D 20:01:04.457353] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.457379] PINT_client_state_machine_test id 40 [D 20:01:04.457407] openib_check_cq: completed recv wr_id 15e73b60. [D 20:01:04.457435] openib_post_rr: da13:3346 bh 69 wr_id 15e73b60. [D 20:01:04.457462] openib_check_cq: completed recv wr_id 15e73b78. [D 20:01:04.457488] openib_post_rr: da13:3346 bh 70 wr_id 15e73b78. [D 20:01:04.457532] sm control op 6 [D 20:01:04.457557] [SM Entering]: (0x15e92b88) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.457583] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.457609] (0x15e92c20) io_datafile_complete_operations (tag 3) [D 20:01:04.457633] matched completed ack for context 0x15e933d0 [D 20:01:04.457660] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.457686] [SM Exiting]: (0x15e92b88) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.457715] [SM Entering]: (0x15e92b88) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.457740] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.457765] (0x15e92c20) io state: io_analyze_results [D 20:01:04.457790] - io_check_context_status called [D 20:01:04.457814] PINT_decode [D 20:01:04.457839] lebf_decode_resp [D 20:01:04.457862] 33554432 bytes written to context 0x15e933d0 [D 20:01:04.457887] PINT_decode_release [D 20:01:04.457911] lebf_decode_rel [D 20:01:04.457937] [1/1] running size is 33554432 [D 20:01:04.457962] [2/1] running size is 33554432 [D 20:01:04.457988] no context errors found; using: Success [D 20:01:04.458012] total bytes transferred is 33554432 [D 20:01:04.458037] acache: invalidate_size(): H=146 [D 20:01:04.458063] [SM Exiting]: (0x15e92b88) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.458090] [SM Entering]: (0x15e92b88) client_io_sm:io_cleanup (status: 3) [D 20:01:04.458116] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.458141] (0x15e92c20) io state: io_cleanup [D 20:01:04.458168] [SM Exiting]: (0x15e92b88) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.458193] client_state_machine_terminate smcb 0x15e92b88 [D 20:01:04.458217] add smcb 0x15e92b88 to completion list [D 20:01:04.458243] sm control op 6 [D 20:01:04.458267] [SM Entering]: (0x15e95f68) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.458292] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.458323] (0x15e96000) io_datafile_complete_operations (tag 3) [D 20:01:04.458348] matched completed ack for context 0x15e967b0 [D 20:01:04.458373] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.458396] [SM Exiting]: (0x15e95f68) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.458423] [SM Entering]: (0x15e95f68) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.458448] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.458474] (0x15e96000) io state: io_analyze_results [D 20:01:04.458498] - io_check_context_status called [D 20:01:04.458522] PINT_decode [D 20:01:04.458547] lebf_decode_resp [D 20:01:04.458571] 33554432 bytes written to context 0x15e967b0 [D 20:01:04.458596] PINT_decode_release [D 20:01:04.458620] lebf_decode_rel [D 20:01:04.458645] [1/1] running size is 33554432 [D 20:01:04.458671] [2/1] running size is 33554432 [D 20:01:04.458697] no context errors found; using: Success [D 20:01:04.458722] total bytes transferred is 33554432 [D 20:01:04.458746] acache: invalidate_size(): H=146 [D 20:01:04.458772] [SM Exiting]: (0x15e95f68) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.458798] [SM Entering]: (0x15e95f68) client_io_sm:io_cleanup (status: 3) [D 20:01:04.458823] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.458848] (0x15e96000) io state: io_cleanup [D 20:01:04.458875] [SM Exiting]: (0x15e95f68) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.458901] client_state_machine_terminate smcb 0x15e95f68 [D 20:01:04.458926] add smcb 0x15e95f68 to completion list [D 20:01:04.458950] sm control op 6 [D 20:01:04.458974] [SM Entering]: (0x15e99308) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.459000] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.459025] (0x15e993a0) io_datafile_complete_operations (tag 3) [D 20:01:04.459050] matched completed ack for context 0x15e99ba8 [D 20:01:04.459074] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.459099] [SM Exiting]: (0x15e99308) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.459125] [SM Entering]: (0x15e99308) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.459153] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.459178] (0x15e993a0) io state: io_analyze_results [D 20:01:04.459203] - io_check_context_status called [D 20:01:04.459227] PINT_decode [D 20:01:04.459251] lebf_decode_resp [D 20:01:04.459276] 33554432 bytes written to context 0x15e99ba8 [D 20:01:04.459300] PINT_decode_release [D 20:01:04.459323] lebf_decode_rel [D 20:01:04.459350] [1/1] running size is 33554432 [D 20:01:04.459375] [2/1] running size is 33554432 [D 20:01:04.459401] no context errors found; using: Success [D 20:01:04.459425] total bytes transferred is 33554432 [D 20:01:04.459450] acache: invalidate_size(): H=146 [D 20:01:04.459476] [SM Exiting]: (0x15e99308) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.459502] [SM Entering]: (0x15e99308) client_io_sm:io_cleanup (status: 3) [D 20:01:04.459527] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.459552] (0x15e993a0) io state: io_cleanup [D 20:01:04.459579] [SM Exiting]: (0x15e99308) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.459605] client_state_machine_terminate smcb 0x15e99308 [D 20:01:04.459629] add smcb 0x15e99308 to completion list [D 20:01:04.459653] sm control op 6 [D 20:01:04.459678] [SM Entering]: (0x15e9c758) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.459706] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.459735] (0x15e9c7f0) io_datafile_complete_operations (tag 3) [D 20:01:04.459764] matched completed ack for context 0x15e9cfa0 [D 20:01:04.459792] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.459819] [SM Exiting]: (0x15e9c758) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.459845] [SM Entering]: (0x15e9c758) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.459871] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.459897] (0x15e9c7f0) io state: io_analyze_results [D 20:01:04.459921] - io_check_context_status called [D 20:01:04.459945] PINT_decode [D 20:01:04.459970] lebf_decode_resp [D 20:01:04.459994] 33554432 bytes written to context 0x15e9cfa0 [D 20:01:04.460019] PINT_decode_release [D 20:01:04.460042] lebf_decode_rel [D 20:01:04.460068] [1/1] running size is 33554432 [D 20:01:04.460094] [2/1] running size is 33554432 [D 20:01:04.460120] no context errors found; using: Success [D 20:01:04.460144] total bytes transferred is 33554432 [D 20:01:04.460169] acache: invalidate_size(): H=146 [D 20:01:04.460195] [SM Exiting]: (0x15e9c758) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.460221] [SM Entering]: (0x15e9c758) client_io_sm:io_cleanup (status: 3) [D 20:01:04.460247] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.460271] (0x15e9c7f0) io state: io_cleanup [D 20:01:04.460297] [SM Exiting]: (0x15e9c758) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.460323] client_state_machine_terminate smcb 0x15e9c758 [D 20:01:04.460347] add smcb 0x15e9c758 to completion list [D 20:01:04.460371] sm control op 6 [D 20:01:04.460396] [SM Entering]: (0x15e9fb50) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.460421] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.460447] (0x15e9fbe8) io_datafile_complete_operations (tag 3) [D 20:01:04.460472] matched completed ack for context 0x15ea0398 [D 20:01:04.460496] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.460521] [SM Exiting]: (0x15e9fb50) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.460547] [SM Entering]: (0x15e9fb50) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.460573] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.460597] (0x15e9fbe8) io state: io_analyze_results [D 20:01:04.460622] - io_check_context_status called [D 20:01:04.460646] PINT_decode [D 20:01:04.460670] lebf_decode_resp [D 20:01:04.460698] 33554432 bytes written to context 0x15ea0398 [D 20:01:04.460722] PINT_decode_release [D 20:01:04.460746] lebf_decode_rel [D 20:01:04.460772] [1/1] running size is 33554432 [D 20:01:04.460797] [2/1] running size is 33554432 [D 20:01:04.460824] no context errors found; using: Success [D 20:01:04.460848] total bytes transferred is 33554432 [D 20:01:04.460873] acache: invalidate_size(): H=146 [D 20:01:04.460899] [SM Exiting]: (0x15e9fb50) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.460925] [SM Entering]: (0x15e9fb50) client_io_sm:io_cleanup (status: 3) [D 20:01:04.460949] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.460975] (0x15e9fbe8) io state: io_cleanup [D 20:01:04.461001] [SM Exiting]: (0x15e9fb50) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.461027] client_state_machine_terminate smcb 0x15e9fb50 [D 20:01:04.461051] add smcb 0x15e9fb50 to completion list [D 20:01:04.461076] [SM frame get]: (0x15e92b88) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32640 curbuf 17 buffer_ahead_offset 22 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 8 aio_req 0x15e744e0 b->offset 234881024 b->size 33554432 [D 20:01:04.461113] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.461139] PINT_client_state_machine_test id 43 [D 20:01:04.461164] [SM frame get]: (0x15e95f68) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32668 curbuf 17 buffer_ahead_offset 23 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 9 aio_req 0x15e74500 b->offset 268435456 b->size 33554432 [D 20:01:04.461200] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.461226] PINT_client_state_machine_test id 46 [D 20:01:04.461252] [SM frame get]: (0x15e99308) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32690 curbuf 17 buffer_ahead_offset 24 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 10 aio_req 0x15e74520 b->offset 301989888 b->size 33554432 [D 20:01:04.461288] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.461314] PINT_client_state_machine_test id 49 [D 20:01:04.461338] [SM frame get]: (0x15e9c758) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e326b8 curbuf 17 buffer_ahead_offset 25 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 11 aio_req 0x15e74540 b->offset 335544320 b->size 33554432 [D 20:01:04.461376] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.461401] PINT_client_state_machine_test id 52 [D 20:01:04.461426] [SM frame get]: (0x15e9fb50) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e326e0 curbuf 17 buffer_ahead_offset 26 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 12 aio_req 0x15e74560 b->offset 369098752 b->size 33554432 [D 20:01:04.461462] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.461488] PINT_client_state_machine_test id 55 [D 20:01:04.461517] openib_check_cq: completed recv wr_id 15e73b90. [D 20:01:04.461544] openib_post_rr: da13:3346 bh 71 wr_id 15e73b90. [D 20:01:04.461570] openib_check_cq: completed recv wr_id 15e73ba8. [D 20:01:04.461596] openib_post_rr: da13:3346 bh 72 wr_id 15e73ba8. [D 20:01:04.461632] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:04.461659] ========================================================= [D 20:01:04.461684] PINT_process_request [D 20:01:04.461710] tiling 8 copies [D 20:01:04.461735] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.461761] lvl 0 el 0 blk 0 by 203423744 [D 20:01:04.461787] to 203423744 ta 201326592 fi 234881024 [D 20:01:04.461812] mto 2097152 mta 0 mfi 2097152 [D 20:01:04.461836] basic type or contiguous data [D 20:01:04.461860] Request disp [D 20:01:04.461887] contig_offset = 203423744 contig_size = 65011712 lvl_flag = 1 [D 20:01:04.461913] calling distribute [D 20:01:04.461936] PINT_distribute [D 20:01:04.461960] of 203423744 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:04.461987] begin iteration loff: 203423744 [D 20:01:04.462012] segment exceeds byte limit [D 20:01:04.462037] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:04.462061] **********CALL***PROCESS********* [D 20:01:04.462085] segment of 2097152 sz 2097152 [D 20:01:04.462109] PINT_process_request [D 20:01:04.462133] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.462160] lvl 0 el 0 blk 0 by 2097152 [D 20:01:04.462184] to 2097152 ta 2097152 fi 4194304 [D 20:01:04.462209] basic type or contiguous data [D 20:01:04.462233] Request disp [D 20:01:04.462257] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:04.462282] process a segment [D 20:01:04.462306] of 2097152 sz 2097152 [D 20:01:04.462329] add a segment 0 [D 20:01:04.462353] retval = 2097152 [D 20:01:04.462377] exiting distribute returned less than expected [D 20:01:04.462402] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:04.462428] *****RETURN***FROM***PROCESS***** [D 20:01:04.462451] end iteration [D 20:01:04.462475] done with segments or bytes [D 20:01:04.462499] of 205520896 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.462525] next loff: 205520896 next poff: 205520896 [D 20:01:04.462550] return value 2097152 [D 20:01:04.462574] finished [D 20:01:04.462597] retval = 2097152 [D 20:01:04.462622] exiting distribute returned less than expected [D 20:01:04.462646] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 205520896 fo 234881024 eof 0 [D 20:01:04.462672] ========================================================= [D 20:01:04.462697] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:04.462722] element 0: offset: 0x96645000, size: 2097152 [D 20:01:04.462752] post_sr: da13:3346 bh 93 len 48 wr_id 15e73438. [D 20:01:04.462785] sm control op 6 [D 20:01:04.462810] [SM Entering]: (0x15ea2f48) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.462836] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.462862] (0x15ea2fe0) io_datafile_complete_operations (tag 3) [D 20:01:04.462886] matched completed ack for context 0x15ea3790 [D 20:01:04.462911] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.462936] [SM Exiting]: (0x15ea2f48) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.462962] [SM Entering]: (0x15ea2f48) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.462987] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.463013] (0x15ea2fe0) io state: io_analyze_results [D 20:01:04.463037] - io_check_context_status called [D 20:01:04.463062] PINT_decode [D 20:01:04.463086] lebf_decode_resp [D 20:01:04.463109] 33554432 bytes written to context 0x15ea3790 [D 20:01:04.463134] PINT_decode_release [D 20:01:04.463158] lebf_decode_rel [D 20:01:04.463185] [1/1] running size is 33554432 [D 20:01:04.463210] [2/1] running size is 33554432 [D 20:01:04.463236] no context errors found; using: Success [D 20:01:04.463261] total bytes transferred is 33554432 [D 20:01:04.463286] acache: invalidate_size(): H=146 [D 20:01:04.463311] [SM Exiting]: (0x15ea2f48) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.463337] [SM Entering]: (0x15ea2f48) client_io_sm:io_cleanup (status: 3) [D 20:01:04.463362] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.463388] (0x15ea2fe0) io state: io_cleanup [D 20:01:04.463414] [SM Exiting]: (0x15ea2f48) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.463439] client_state_machine_terminate smcb 0x15ea2f48 [D 20:01:04.463464] add smcb 0x15ea2f48 to completion list [D 20:01:04.463489] sm control op 6 [D 20:01:04.463514] [SM Entering]: (0x15ea6340) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.463542] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.463567] (0x15ea63d8) io_datafile_complete_operations (tag 3) [D 20:01:04.463593] matched completed ack for context 0x15ea6b88 [D 20:01:04.463618] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.463642] [SM Exiting]: (0x15ea6340) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.463667] [SM Entering]: (0x15ea6340) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.463693] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.463722] (0x15ea63d8) io state: io_analyze_results [D 20:01:04.463750] - io_check_context_status called [D 20:01:04.463778] PINT_decode [D 20:01:04.463806] lebf_decode_resp [D 20:01:04.463830] 33554432 bytes written to context 0x15ea6b88 [D 20:01:04.463855] PINT_decode_release [D 20:01:04.463879] lebf_decode_rel [D 20:01:04.463905] [1/1] running size is 33554432 [D 20:01:04.463931] [2/1] running size is 33554432 [D 20:01:04.463956] no context errors found; using: Success [D 20:01:04.463981] total bytes transferred is 33554432 [D 20:01:04.464006] acache: invalidate_size(): H=146 [D 20:01:04.464032] [SM Exiting]: (0x15ea6340) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.464057] [SM Entering]: (0x15ea6340) client_io_sm:io_cleanup (status: 3) [D 20:01:04.464083] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.464108] (0x15ea63d8) io state: io_cleanup [D 20:01:04.464134] [SM Exiting]: (0x15ea6340) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.464159] client_state_machine_terminate smcb 0x15ea6340 [D 20:01:04.464184] add smcb 0x15ea6340 to completion list [D 20:01:04.464209] sm control op 6 [D 20:01:04.464233] [SM Entering]: (0x15ea9738) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.464259] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.464283] (0x15ea97d0) io_datafile_complete_operations (tag 3) [D 20:01:04.464309] matched completed ack for context 0x15ea9f80 [D 20:01:04.464333] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.464358] [SM Exiting]: (0x15ea9738) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.464384] [SM Entering]: (0x15ea9738) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.464409] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.464434] (0x15ea97d0) io state: io_analyze_results [D 20:01:04.464459] - io_check_context_status called [D 20:01:04.464483] PINT_decode [D 20:01:04.464508] lebf_decode_resp [D 20:01:04.464531] 33554432 bytes written to context 0x15ea9f80 [D 20:01:04.464556] PINT_decode_release [D 20:01:04.464580] lebf_decode_rel [D 20:01:04.464606] [1/1] running size is 33554432 [D 20:01:04.464631] [2/1] running size is 33554432 [D 20:01:04.464657] no context errors found; using: Success [D 20:01:04.464682] total bytes transferred is 33554432 [D 20:01:04.464706] acache: invalidate_size(): H=146 [D 20:01:04.464732] [SM Exiting]: (0x15ea9738) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.464758] [SM Entering]: (0x15ea9738) client_io_sm:io_cleanup (status: 3) [D 20:01:04.464783] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.464809] (0x15ea97d0) io state: io_cleanup [D 20:01:04.464834] [SM Exiting]: (0x15ea9738) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.464859] client_state_machine_terminate smcb 0x15ea9738 [D 20:01:04.464884] add smcb 0x15ea9738 to completion list [D 20:01:04.464909] sm control op 6 [D 20:01:04.464934] [SM Entering]: (0x15eadd18) client_io_sm:io_datafile_complete_operations (status: 3) [D 20:01:04.464959] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.464983] (0x15eaddb0) io_datafile_complete_operations (tag 3) [D 20:01:04.465009] matched completed ack for context 0x15eae590 [D 20:01:04.465036] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.465061] [SM Exiting]: (0x15eadd18) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.465087] [SM Entering]: (0x15eadd18) client_io_sm:io_analyze_results (status: 3) [D 20:01:04.465112] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.465138] (0x15eaddb0) io state: io_analyze_results [D 20:01:04.465162] - io_check_context_status called [D 20:01:04.465187] PINT_decode [D 20:01:04.465210] lebf_decode_resp [D 20:01:04.465235] 33554432 bytes written to context 0x15eae590 [D 20:01:04.465259] PINT_decode_release [D 20:01:04.465284] lebf_decode_rel [D 20:01:04.465310] [1/1] running size is 33554432 [D 20:01:04.465334] [2/1] running size is 33554432 [D 20:01:04.465360] no context errors found; using: Success [D 20:01:04.465385] total bytes transferred is 33554432 [D 20:01:04.465410] acache: invalidate_size(): H=146 [D 20:01:04.465435] [SM Exiting]: (0x15eadd18) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.465462] [SM Entering]: (0x15eadd18) client_io_sm:io_cleanup (status: 3) [D 20:01:04.465488] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.465513] (0x15eaddb0) io state: io_cleanup [D 20:01:04.465538] [SM Exiting]: (0x15eadd18) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.465564] client_state_machine_terminate smcb 0x15eadd18 [D 20:01:04.465589] add smcb 0x15eadd18 to completion list [D 20:01:04.465614] [SM frame get]: (0x15ea2f48) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32708 curbuf 17 buffer_ahead_offset 27 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 13 aio_req 0x15e74580 b->offset 402653184 b->size 33554432 [D 20:01:04.465651] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.465676] PINT_client_state_machine_test id 58 [D 20:01:04.465703] [SM frame get]: (0x15ea6340) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32730 curbuf 17 buffer_ahead_offset 28 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 14 aio_req 0x15e745a0 b->offset 436207616 b->size 33554432 [D 20:01:04.465741] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.465766] PINT_client_state_machine_test id 61 [D 20:01:04.465792] [SM frame get]: (0x15ea9738) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32758 curbuf 17 buffer_ahead_offset 29 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 15 aio_req 0x15e745c0 b->offset 469762048 b->size 33554432 [D 20:01:04.465828] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.465855] PINT_client_state_machine_test id 64 [D 20:01:04.465880] [SM frame get]: (0x15eadd18) op-id: 6 index: 0 base-frm: 0 -- buffer 0x15e32780 curbuf 17 buffer_ahead_offset 30 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill -- buffer 0x15e327a8 curbuf 17 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 17 aio_req 0x15e74600 b->offset 0 b->size 33554432 [D 20:01:04.465922] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.465948] PINT_client_state_machine_test id 67 [D 20:01:04.465973] [SM frame get]: (0x15eb1140) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 17 get_next_full_readbuf: enter userFilePos 33554432, read_ahead_offset 503316480 -- buffer 0x15e327d0 curbuf 18 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 18 aio_req 0x15e74620 b->offset 33554432 b->size 33554432 [D 20:01:04.825459] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.825507] PINT_client_state_machine_test id 70 [D 20:01:04.825541] openib_check_cq: completed send wr_id 15e73438. [D 20:01:04.825575] openib_check_cq: completed recv wr_id 15e73bc0. [D 20:01:04.825625] openib_post_rr: da13:3346 bh 73 wr_id 15e73bc0. [D 20:01:04.825654] openib_check_cq: completed recv wr_id 15e73bd8. [D 20:01:04.825681] openib_post_rr: da13:3346 bh 74 wr_id 15e73bd8. [D 20:01:04.825711] openib_check_cq: completed recv wr_id 15e73bf0. [D 20:01:04.825742] openib_post_rr: da13:3346 bh 75 wr_id 15e73bf0. [D 20:01:04.825772] openib_check_cq: completed recv wr_id 15e73c08. [D 20:01:04.825803] openib_post_rr: da13:3346 bh 76 wr_id 15e73c08. [D 20:01:04.825832] openib_check_cq: completed recv wr_id 15e73c20. [D 20:01:04.825860] openib_post_rr: da13:3346 bh 77 wr_id 15e73c20. [D 20:01:04.825886] openib_check_cq: completed recv wr_id 15e73c38. [D 20:01:04.825914] openib_post_rr: da13:3346 bh 78 wr_id 15e73c38. [D 20:01:04.825940] openib_check_cq: completed recv wr_id 15e73c50. [D 20:01:04.825968] openib_post_rr: da13:3346 bh 79 wr_id 15e73c50. [D 20:01:04.825994] openib_check_cq: completed recv wr_id 15e73c68. [D 20:01:04.826022] openib_post_rr: da13:3346 bh 80 wr_id 15e73c68. [D 20:01:04.826048] openib_check_cq: completed recv wr_id 15e73c80. [D 20:01:04.826076] openib_post_rr: da13:3346 bh 81 wr_id 15e73c80. [D 20:01:04.826103] openib_check_cq: completed recv wr_id 15e73c98. [D 20:01:04.826130] openib_post_rr: da13:3346 bh 82 wr_id 15e73c98. [D 20:01:04.826156] openib_check_cq: completed recv wr_id 15e73cb0. [D 20:01:04.826182] openib_post_rr: da13:3346 bh 83 wr_id 15e73cb0. [D 20:01:04.826210] post_sr: da13:3346 bh 94 len 48 wr_id 15e73450. [D 20:01:04.826237] openib_check_cq: completed recv wr_id 15e73cc8. [D 20:01:04.826264] openib_post_rr: da13:3346 bh 84 wr_id 15e73cc8. [D 20:01:04.826290] openib_check_cq: completed recv wr_id 15e73ce0. [D 20:01:04.826316] openib_post_rr: da13:3346 bh 85 wr_id 15e73ce0. [D 20:01:04.826342] openib_check_cq: completed recv wr_id 15e73cf8. [D 20:01:04.826369] openib_post_rr: da13:3346 bh 86 wr_id 15e73cf8. [D 20:01:04.826395] openib_check_cq: completed recv wr_id 15e73d10. [D 20:01:04.826423] openib_post_rr: da13:3346 bh 87 wr_id 15e73d10. [D 20:01:04.826450] openib_check_cq: completed recv wr_id 15e73d28. [D 20:01:04.826478] openib_post_rr: da13:3346 bh 88 wr_id 15e73d28. [D 20:01:04.826504] openib_check_cq: completed recv wr_id 15e73d40. [D 20:01:04.826530] openib_post_rr: da13:3346 bh 89 wr_id 15e73d40. [D 20:01:04.826557] post_sr: da13:3346 bh 95 len 48 wr_id 15e73468. [D 20:01:04.826583] openib_check_cq: completed recv wr_id 15e73d58. [D 20:01:04.826611] openib_post_rr: da13:3346 bh 90 wr_id 15e73d58. [D 20:01:04.826637] openib_check_cq: completed recv wr_id 15e73d70. [D 20:01:04.826665] openib_post_rr: da13:3346 bh 91 wr_id 15e73d70. [D 20:01:04.826692] openib_check_cq: completed recv wr_id 15e73d88. [D 20:01:04.826720] openib_post_rr: da13:3346 bh 92 wr_id 15e73d88. [D 20:01:04.826746] openib_check_cq: completed recv wr_id 15e73da0. [D 20:01:04.826775] openib_post_rr: da13:3346 bh 93 wr_id 15e73da0. [D 20:01:04.826802] openib_check_cq: completed recv wr_id 15e73db8. [D 20:01:04.826829] openib_post_rr: da13:3346 bh 94 wr_id 15e73db8. [D 20:01:04.826855] openib_check_cq: completed recv wr_id 15e73dd0. [D 20:01:04.826884] openib_post_rr: da13:3346 bh 95 wr_id 15e73dd0. [D 20:01:04.826910] openib_check_cq: completed recv wr_id 15e73de8. [D 20:01:04.826936] openib_post_rr: da13:3346 bh 96 wr_id 15e73de8. [D 20:01:04.826963] post_sr: da13:3346 bh 96 len 48 wr_id 15e73480. [D 20:01:04.826989] openib_check_cq: completed recv wr_id 15e73e00. [D 20:01:04.827017] openib_post_rr: da13:3346 bh 97 wr_id 15e73e00. [D 20:01:04.827043] openib_check_cq: completed recv wr_id 15e73e18. [D 20:01:04.827072] openib_post_rr: da13:3346 bh 98 wr_id 15e73e18. [D 20:01:04.827097] openib_check_cq: completed recv wr_id 15e73e30. [D 20:01:04.827125] openib_post_rr: da13:3346 bh 99 wr_id 15e73e30. [D 20:01:04.827152] openib_check_cq: completed recv wr_id 15e734e8. [D 20:01:04.827179] openib_post_rr: da13:3346 bh 0 wr_id 15e734e8. [D 20:01:04.827206] openib_check_cq: completed recv wr_id 15e73500. [D 20:01:04.827237] openib_post_rr: da13:3346 bh 1 wr_id 15e73500. [D 20:01:04.827263] openib_check_cq: completed recv wr_id 15e73518. [D 20:01:04.827292] openib_post_rr: da13:3346 bh 2 wr_id 15e73518. [D 20:01:04.827318] openib_check_cq: completed recv wr_id 15e73530. [D 20:01:04.827346] openib_post_rr: da13:3346 bh 3 wr_id 15e73530. [D 20:01:04.827372] openib_check_cq: completed recv wr_id 15e73548. [D 20:01:04.827401] openib_post_rr: da13:3346 bh 4 wr_id 15e73548. [D 20:01:04.827426] openib_check_cq: completed recv wr_id 15e73560. [D 20:01:04.827455] openib_post_rr: da13:3346 bh 5 wr_id 15e73560. [D 20:01:04.827481] openib_check_cq: completed recv wr_id 15e73578. [D 20:01:04.827509] openib_post_rr: da13:3346 bh 6 wr_id 15e73578. [D 20:01:04.827535] openib_check_cq: completed recv wr_id 15e73590. [D 20:01:04.827564] openib_post_rr: da13:3346 bh 7 wr_id 15e73590. [D 20:01:04.827590] openib_check_cq: completed recv wr_id 15e735a8. [D 20:01:04.827618] openib_post_rr: da13:3346 bh 8 wr_id 15e735a8. [D 20:01:04.827644] openib_check_cq: completed recv wr_id 15e735c0. [D 20:01:04.827672] openib_post_rr: da13:3346 bh 9 wr_id 15e735c0. [D 20:01:04.827699] openib_check_cq: completed recv wr_id 15e735d8. [D 20:01:04.827729] openib_post_rr: da13:3346 bh 10 wr_id 15e735d8. [D 20:01:04.827756] post_sr: da13:3346 bh 97 len 48 wr_id 15e73498. [D 20:01:04.827782] openib_check_cq: completed recv wr_id 15e735f0. [D 20:01:04.827811] openib_post_rr: da13:3346 bh 11 wr_id 15e735f0. [D 20:01:04.827837] openib_check_cq: completed recv wr_id 15e73608. [D 20:01:04.827865] openib_post_rr: da13:3346 bh 12 wr_id 15e73608. [D 20:01:04.827892] openib_check_cq: completed recv wr_id 15e73620. [D 20:01:04.827920] openib_post_rr: da13:3346 bh 13 wr_id 15e73620. [D 20:01:04.827947] openib_check_cq: completed recv wr_id 15e73638. [D 20:01:04.827975] openib_post_rr: da13:3346 bh 14 wr_id 15e73638. [D 20:01:04.828001] openib_check_cq: completed recv wr_id 15e73650. [D 20:01:04.828029] openib_post_rr: da13:3346 bh 15 wr_id 15e73650. [D 20:01:04.828058] openib_check_cq: completed recv wr_id 15e73668. [D 20:01:04.828086] openib_post_rr: da13:3346 bh 16 wr_id 15e73668. [D 20:01:04.828113] openib_check_cq: completed recv wr_id 15e73680. [D 20:01:04.828142] openib_post_rr: da13:3346 bh 17 wr_id 15e73680. [D 20:01:04.828167] openib_check_cq: completed recv wr_id 15e73698. [D 20:01:04.828194] openib_post_rr: da13:3346 bh 18 wr_id 15e73698. [D 20:01:04.828221] post_sr: da13:3346 bh 98 len 48 wr_id 15e734b0. [D 20:01:04.828248] openib_check_cq: completed recv wr_id 15e736b0. [D 20:01:04.828275] openib_post_rr: da13:3346 bh 19 wr_id 15e736b0. [D 20:01:04.828302] openib_check_cq: completed recv wr_id 15e736c8. [D 20:01:04.828330] openib_post_rr: da13:3346 bh 20 wr_id 15e736c8. [D 20:01:04.828357] openib_check_cq: completed recv wr_id 15e736e0. [D 20:01:04.828386] openib_post_rr: da13:3346 bh 21 wr_id 15e736e0. [D 20:01:04.828412] openib_check_cq: completed recv wr_id 15e736f8. [D 20:01:04.828440] openib_post_rr: da13:3346 bh 22 wr_id 15e736f8. [D 20:01:04.828466] openib_check_cq: completed recv wr_id 15e73710. [D 20:01:04.828495] openib_post_rr: da13:3346 bh 23 wr_id 15e73710. [D 20:01:04.828521] openib_check_cq: completed recv wr_id 15e73728. [D 20:01:04.828550] openib_post_rr: da13:3346 bh 24 wr_id 15e73728. [D 20:01:04.828576] openib_check_cq: completed recv wr_id 15e73740. [D 20:01:04.828604] openib_post_rr: da13:3346 bh 25 wr_id 15e73740. [D 20:01:04.828631] openib_check_cq: completed recv wr_id 15e73758. [D 20:01:04.828660] openib_post_rr: da13:3346 bh 26 wr_id 15e73758. [D 20:01:04.828686] openib_check_cq: completed recv wr_id 15e73770. [D 20:01:04.828715] openib_post_rr: da13:3346 bh 27 wr_id 15e73770. [D 20:01:04.828742] openib_check_cq: completed recv wr_id 15e73788. [D 20:01:04.828770] openib_post_rr: da13:3346 bh 28 wr_id 15e73788. [D 20:01:04.828797] openib_check_cq: completed recv wr_id 15e737a0. [D 20:01:04.828825] openib_post_rr: da13:3346 bh 29 wr_id 15e737a0. [D 20:01:04.828851] openib_check_cq: completed recv wr_id 15e737b8. [D 20:01:04.828883] openib_post_rr: da13:3346 bh 30 wr_id 15e737b8. [D 20:01:04.828908] openib_check_cq: completed recv wr_id 15e737d0. [D 20:01:04.828938] openib_post_rr: da13:3346 bh 31 wr_id 15e737d0. [D 20:01:04.828964] openib_check_cq: completed recv wr_id 15e737e8. [D 20:01:04.828993] openib_post_rr: da13:3346 bh 32 wr_id 15e737e8. [D 20:01:04.829019] openib_check_cq: completed recv wr_id 15e73800. [D 20:01:04.829048] openib_post_rr: da13:3346 bh 33 wr_id 15e73800. [D 20:01:04.829074] openib_check_cq: completed send wr_id 15e73450. [D 20:01:04.829099] openib_check_cq: completed send wr_id 15e73468. [D 20:01:04.829125] openib_check_cq: completed send wr_id 15e73480. [D 20:01:04.829150] openib_check_cq: completed send wr_id 15e73498. [D 20:01:04.829175] openib_check_cq: completed send wr_id 15e734b0. [D 20:01:04.829213] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:04.829239] ========================================================= [D 20:01:04.829264] PINT_process_request [D 20:01:04.829289] tiling 9 copies [D 20:01:04.829314] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.829339] lvl 0 el 0 blk 0 by 236978176 [D 20:01:04.829365] to 236978176 ta 234881024 fi 268435456 [D 20:01:04.829390] mto 2097152 mta 0 mfi 2097152 [D 20:01:04.829415] basic type or contiguous data [D 20:01:04.829439] Request disp [D 20:01:04.829463] contig_offset = 236978176 contig_size = 65011712 lvl_flag = 1 [D 20:01:04.829488] calling distribute [D 20:01:04.829512] PINT_distribute [D 20:01:04.829536] of 236978176 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:04.829564] begin iteration loff: 236978176 [D 20:01:04.829589] segment exceeds byte limit [D 20:01:04.829613] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:04.829638] **********CALL***PROCESS********* [D 20:01:04.829662] segment of 2097152 sz 2097152 [D 20:01:04.829686] PINT_process_request [D 20:01:04.829713] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.829744] lvl 0 el 0 blk 0 by 2097152 [D 20:01:04.829771] to 2097152 ta 2097152 fi 4194304 [D 20:01:04.829800] basic type or contiguous data [D 20:01:04.829827] Request disp [D 20:01:04.829851] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:04.829875] process a segment [D 20:01:04.829899] of 2097152 sz 2097152 [D 20:01:04.829924] add a segment 0 [D 20:01:04.829948] retval = 2097152 [D 20:01:04.829972] exiting distribute returned less than expected [D 20:01:04.829996] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:04.830022] *****RETURN***FROM***PROCESS***** [D 20:01:04.830047] end iteration [D 20:01:04.830071] done with segments or bytes [D 20:01:04.830095] of 239075328 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.830121] next loff: 239075328 next poff: 239075328 [D 20:01:04.830145] return value 2097152 [D 20:01:04.830169] finished [D 20:01:04.830193] retval = 2097152 [D 20:01:04.830218] exiting distribute returned less than expected [D 20:01:04.830242] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 239075328 fo 268435456 eof 0 [D 20:01:04.830267] ========================================================= [D 20:01:04.830293] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:04.830319] element 0: offset: 0x94643000, size: 2097152 [D 20:01:04.830349] post_sr: da13:3346 bh 99 len 48 wr_id 15e734c8. [D 20:01:04.830378] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:04.830404] ========================================================= [D 20:01:04.830428] PINT_process_request [D 20:01:04.830452] tiling 10 copies [D 20:01:04.830476] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.830502] lvl 0 el 0 blk 0 by 270532608 [D 20:01:04.830527] to 270532608 ta 268435456 fi 301989888 [D 20:01:04.830552] mto 2097152 mta 0 mfi 2097152 [D 20:01:04.830576] basic type or contiguous data [D 20:01:04.830603] Request disp [D 20:01:04.830627] contig_offset = 270532608 contig_size = 65011712 lvl_flag = 1 [D 20:01:04.830653] calling distribute [D 20:01:04.830677] PINT_distribute [D 20:01:04.830700] of 270532608 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:04.830727] begin iteration loff: 270532608 [D 20:01:04.830752] segment exceeds byte limit [D 20:01:04.830776] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:04.830801] **********CALL***PROCESS********* [D 20:01:04.830824] segment of 2097152 sz 2097152 [D 20:01:04.830849] PINT_process_request [D 20:01:04.830873] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.830899] lvl 0 el 0 blk 0 by 2097152 [D 20:01:04.830924] to 2097152 ta 2097152 fi 4194304 [D 20:01:04.830948] basic type or contiguous data [D 20:01:04.830972] Request disp [D 20:01:04.830995] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:04.831020] process a segment [D 20:01:04.831044] of 2097152 sz 2097152 [D 20:01:04.831068] add a segment 0 [D 20:01:04.831092] retval = 2097152 [D 20:01:04.831116] exiting distribute returned less than expected [D 20:01:04.831140] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:04.831166] *****RETURN***FROM***PROCESS***** [D 20:01:04.831190] end iteration [D 20:01:04.831213] done with segments or bytes [D 20:01:04.831237] of 272629760 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.831263] next loff: 272629760 next poff: 272629760 [D 20:01:04.831288] return value 2097152 [D 20:01:04.831312] finished [D 20:01:04.831336] retval = 2097152 [D 20:01:04.831359] exiting distribute returned less than expected [D 20:01:04.831384] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 272629760 fo 301989888 eof 0 [D 20:01:04.831410] ========================================================= [D 20:01:04.831434] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:04.831459] element 0: offset: 0x92641000, size: 2097152 [D 20:01:04.831485] openib_check_cq: completed send wr_id 15e734c8. [D 20:01:04.831516] post_sr: da13:3346 bh 0 len 48 wr_id 15e72b80. [D 20:01:04.831543] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:04.831569] ========================================================= [D 20:01:04.831593] PINT_process_request [D 20:01:04.831617] tiling 5 copies [D 20:01:04.831642] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.831667] lvl 0 el 0 blk 0 by 115343360 [D 20:01:04.831692] to 115343360 ta 100663296 fi 134217728 [D 20:01:04.831719] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:04.831745] basic type or contiguous data [D 20:01:04.831768] Request disp [D 20:01:04.831792] contig_offset = 115343360 contig_size = 52428800 lvl_flag = 1 [D 20:01:04.831817] calling distribute [D 20:01:04.831841] PINT_distribute [D 20:01:04.831865] of 115343360 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:04.831891] begin iteration loff: 115343360 [D 20:01:04.831916] segment exceeds byte limit [D 20:01:04.831940] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:04.831965] **********CALL***PROCESS********* [D 20:01:04.831989] segment of 14680064 sz 2097152 [D 20:01:04.832013] PINT_process_request [D 20:01:04.832036] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.832063] lvl 0 el 0 blk 0 by 14680064 [D 20:01:04.832088] to 14680064 ta 14680064 fi 16777216 [D 20:01:04.832113] basic type or contiguous data [D 20:01:04.832137] Request disp [D 20:01:04.832160] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:04.832185] process a segment [D 20:01:04.832208] of 14680064 sz 2097152 [D 20:01:04.832233] add a segment 0 [D 20:01:04.832257] retval = 2097152 [D 20:01:04.832281] exiting distribute returned less than expected [D 20:01:04.832304] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:04.832334] *****RETURN***FROM***PROCESS***** [D 20:01:04.832358] end iteration [D 20:01:04.832382] done with segments or bytes [D 20:01:04.832406] of 117440512 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.832431] next loff: 117440512 next poff: 117440512 [D 20:01:04.832456] return value 2097152 [D 20:01:04.832481] finished [D 20:01:04.832505] retval = 2097152 [D 20:01:04.832529] exiting distribute returned less than expected [D 20:01:04.832552] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 117440512 fo 134217728 eof 0 [D 20:01:04.832579] ========================================================= [D 20:01:04.832603] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:04.832629] element 0: offset: 0x9d24b000, size: 2097152 [D 20:01:04.832654] openib_check_cq: completed send wr_id 15e72b80. [D 20:01:04.832683] post_sr: da13:3346 bh 1 len 48 wr_id 15e72b98. [D 20:01:04.832711] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:04.832736] ========================================================= [D 20:01:04.832761] PINT_process_request [D 20:01:04.832785] tiling 3 copies [D 20:01:04.832809] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.832835] lvl 0 el 0 blk 0 by 65011712 [D 20:01:04.832860] to 65011712 ta 33554432 fi 67108864 [D 20:01:04.832885] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:04.832910] basic type or contiguous data [D 20:01:04.832934] Request disp [D 20:01:04.832958] contig_offset = 65011712 contig_size = 35651584 lvl_flag = 1 [D 20:01:04.832983] calling distribute [D 20:01:04.833007] PINT_distribute [D 20:01:04.833030] of 65011712 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:04.833057] begin iteration loff: 65011712 [D 20:01:04.833082] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:04.833106] **********CALL***PROCESS********* [D 20:01:04.833130] segment of 31457280 sz 2097152 [D 20:01:04.833154] PINT_process_request [D 20:01:04.833178] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.833204] lvl 0 el 0 blk 0 by 31457280 [D 20:01:04.833229] to 31457280 ta 31457280 fi 33554432 [D 20:01:04.833253] basic type or contiguous data [D 20:01:04.833277] Request disp [D 20:01:04.833301] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:04.833326] process a segment [D 20:01:04.833350] of 31457280 sz 2097152 [D 20:01:04.833374] add a segment 0 [D 20:01:04.833398] retval = 2097152 [D 20:01:04.833422] return from level -1 [D 20:01:04.833446] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:04.833472] *****RETURN***FROM***PROCESS***** [D 20:01:04.833497] end iteration [D 20:01:04.833520] done with segments or bytes [D 20:01:04.833544] of 67108864 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.833570] next loff: 67108864 next poff: 67108864 [D 20:01:04.833595] (return value) 2097152 [D 20:01:04.833620] finished [D 20:01:04.833642] retval = 2097152 [D 20:01:04.833666] exiting distribute returned less than expected [D 20:01:04.833691] done sg 1 sm 64 by 2097152 bm 2097152 ta 33554432 to 67108864 fo 67108864 eof 0 [D 20:01:04.833721] ========================================================= [D 20:01:04.833749] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 24 [D 20:01:04.833778] element 0: offset: 0xa224f000, size: 2097152 [D 20:01:04.833808] openib_check_cq: completed send wr_id 15e72b98. [D 20:01:04.833838] post_sr: da13:3346 bh 2 len 48 wr_id 15e72bb0. [D 20:01:04.833866] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:04.833891] ========================================================= [D 20:01:04.833915] PINT_process_request [D 20:01:04.833939] tiling 4 copies [D 20:01:04.833963] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.833990] lvl 0 el 0 blk 0 by 96468992 [D 20:01:04.834014] to 96468992 ta 67108864 fi 100663296 [D 20:01:04.834041] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:04.834067] basic type or contiguous data [D 20:01:04.834091] Request disp [D 20:01:04.834114] contig_offset = 96468992 contig_size = 37748736 lvl_flag = 1 [D 20:01:04.834139] calling distribute [D 20:01:04.834163] PINT_distribute [D 20:01:04.834186] of 96468992 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:04.834214] begin iteration loff: 96468992 [D 20:01:04.834239] segment exceeds byte limit [D 20:01:04.834263] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:04.834288] **********CALL***PROCESS********* [D 20:01:04.834311] segment of 29360128 sz 2097152 [D 20:01:04.834336] PINT_process_request [D 20:01:04.834360] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.834385] lvl 0 el 0 blk 0 by 29360128 [D 20:01:04.834411] to 29360128 ta 29360128 fi 31457280 [D 20:01:04.834436] basic type or contiguous data [D 20:01:04.834459] Request disp [D 20:01:04.834483] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:04.834508] process a segment [D 20:01:04.834531] of 29360128 sz 2097152 [D 20:01:04.834556] add a segment 0 [D 20:01:04.834580] retval = 2097152 [D 20:01:04.834604] exiting distribute returned less than expected [D 20:01:04.834629] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:04.834654] *****RETURN***FROM***PROCESS***** [D 20:01:04.834678] end iteration [D 20:01:04.834702] done with segments or bytes [D 20:01:04.834726] of 98566144 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.834752] next loff: 98566144 next poff: 98566144 [D 20:01:04.834776] return value 2097152 [D 20:01:04.834800] finished [D 20:01:04.834824] retval = 2097152 [D 20:01:04.834848] exiting distribute returned less than expected [D 20:01:04.834872] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 98566144 fo 100663296 eof 0 [D 20:01:04.834897] ========================================================= [D 20:01:04.834921] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:04.834947] element 0: offset: 0xa004d000, size: 2097152 [D 20:01:04.834973] openib_check_cq: completed send wr_id 15e72bb0. [D 20:01:04.835001] post_sr: da13:3346 bh 3 len 48 wr_id 15e72bc8. [D 20:01:04.835040] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:04.835066] ========================================================= [D 20:01:04.835091] PINT_process_request [D 20:01:04.835115] tiling 6 copies [D 20:01:04.835140] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.835165] lvl 0 el 0 blk 0 by 140509184 [D 20:01:04.835190] to 140509184 ta 134217728 fi 167772160 [D 20:01:04.835215] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:04.835240] basic type or contiguous data [D 20:01:04.835265] Request disp [D 20:01:04.835289] contig_offset = 140509184 contig_size = 60817408 lvl_flag = 1 [D 20:01:04.835313] calling distribute [D 20:01:04.835338] PINT_distribute [D 20:01:04.835362] of 140509184 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:04.835389] begin iteration loff: 140509184 [D 20:01:04.835413] segment exceeds byte limit [D 20:01:04.835437] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:04.835462] **********CALL***PROCESS********* [D 20:01:04.835486] segment of 6291456 sz 2097152 [D 20:01:04.835511] PINT_process_request [D 20:01:04.835535] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.835561] lvl 0 el 0 blk 0 by 6291456 [D 20:01:04.835586] to 6291456 ta 6291456 fi 8388608 [D 20:01:04.835611] basic type or contiguous data [D 20:01:04.835635] Request disp [D 20:01:04.835659] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:04.835683] process a segment [D 20:01:04.835709] of 6291456 sz 2097152 [D 20:01:04.835734] add a segment 0 [D 20:01:04.835758] retval = 2097152 [D 20:01:04.835782] exiting distribute returned less than expected [D 20:01:04.835812] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:04.835839] *****RETURN***FROM***PROCESS***** [D 20:01:04.835864] end iteration [D 20:01:04.835887] done with segments or bytes [D 20:01:04.835911] of 142606336 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.835936] next loff: 142606336 next poff: 142606336 [D 20:01:04.835961] return value 2097152 [D 20:01:04.835986] finished [D 20:01:04.836009] retval = 2097152 [D 20:01:04.836033] exiting distribute returned less than expected [D 20:01:04.836057] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 142606336 fo 167772160 eof 0 [D 20:01:04.836083] ========================================================= [D 20:01:04.836108] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:04.836133] element 0: offset: 0x9aa49000, size: 2097152 [D 20:01:04.836158] openib_check_cq: completed send wr_id 15e72bc8. [D 20:01:04.836188] post_sr: da13:3346 bh 4 len 48 wr_id 15e72be0. [D 20:01:04.836215] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:04.836241] ========================================================= [D 20:01:04.836266] PINT_process_request [D 20:01:04.836290] tiling 7 copies [D 20:01:04.836314] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.836340] lvl 0 el 0 blk 0 by 171966464 [D 20:01:04.836365] to 171966464 ta 167772160 fi 201326592 [D 20:01:04.836390] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:04.836415] basic type or contiguous data [D 20:01:04.836439] Request disp [D 20:01:04.836462] contig_offset = 171966464 contig_size = 62914560 lvl_flag = 1 [D 20:01:04.836487] calling distribute [D 20:01:04.836511] PINT_distribute [D 20:01:04.836535] of 171966464 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:04.836562] begin iteration loff: 171966464 [D 20:01:04.836586] segment exceeds byte limit [D 20:01:04.836610] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:04.836635] **********CALL***PROCESS********* [D 20:01:04.836659] segment of 4194304 sz 2097152 [D 20:01:04.836684] PINT_process_request [D 20:01:04.836708] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.836733] lvl 0 el 0 blk 0 by 4194304 [D 20:01:04.836758] to 4194304 ta 4194304 fi 6291456 [D 20:01:04.836783] basic type or contiguous data [D 20:01:04.836807] Request disp [D 20:01:04.836831] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:04.836855] process a segment [D 20:01:04.836879] of 4194304 sz 2097152 [D 20:01:04.836904] add a segment 0 [D 20:01:04.836928] retval = 2097152 [D 20:01:04.836952] exiting distribute returned less than expected [D 20:01:04.836976] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:04.837001] *****RETURN***FROM***PROCESS***** [D 20:01:04.837026] end iteration [D 20:01:04.837050] done with segments or bytes [D 20:01:04.837074] of 174063616 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.837100] next loff: 174063616 next poff: 174063616 [D 20:01:04.837123] return value 2097152 [D 20:01:04.837148] finished [D 20:01:04.837172] retval = 2097152 [D 20:01:04.837196] exiting distribute returned less than expected [D 20:01:04.837220] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 174063616 fo 201326592 eof 0 [D 20:01:04.837245] ========================================================= [D 20:01:04.837270] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:04.837295] element 0: offset: 0x98847000, size: 2097152 [D 20:01:04.837322] openib_check_cq: completed send wr_id 15e72be0. [D 20:01:04.837350] post_sr: da13:3346 bh 5 len 48 wr_id 15e72bf8. [D 20:01:04.837378] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:04.837404] ========================================================= [D 20:01:04.837429] PINT_process_request [D 20:01:04.837455] tiling 8 copies [D 20:01:04.837480] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.837506] lvl 0 el 0 blk 0 by 205520896 [D 20:01:04.837531] to 205520896 ta 201326592 fi 234881024 [D 20:01:04.837555] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:04.837580] basic type or contiguous data [D 20:01:04.837604] Request disp [D 20:01:04.837628] contig_offset = 205520896 contig_size = 62914560 lvl_flag = 1 [D 20:01:04.837654] calling distribute [D 20:01:04.837677] PINT_distribute [D 20:01:04.837702] of 205520896 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:04.837735] begin iteration loff: 205520896 [D 20:01:04.837762] segment exceeds byte limit [D 20:01:04.837790] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:04.837819] **********CALL***PROCESS********* [D 20:01:04.837842] segment of 4194304 sz 2097152 [D 20:01:04.837867] PINT_process_request [D 20:01:04.837891] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.837917] lvl 0 el 0 blk 0 by 4194304 [D 20:01:04.837942] to 4194304 ta 4194304 fi 6291456 [D 20:01:04.837966] basic type or contiguous data [D 20:01:04.837990] Request disp [D 20:01:04.838014] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:04.838039] process a segment [D 20:01:04.838065] of 4194304 sz 2097152 [D 20:01:04.838089] add a segment 0 [D 20:01:04.838113] retval = 2097152 [D 20:01:04.838137] exiting distribute returned less than expected [D 20:01:04.838161] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:04.838187] *****RETURN***FROM***PROCESS***** [D 20:01:04.838212] end iteration [D 20:01:04.838235] done with segments or bytes [D 20:01:04.838259] of 207618048 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.838285] next loff: 207618048 next poff: 207618048 [D 20:01:04.838310] return value 2097152 [D 20:01:04.838334] finished [D 20:01:04.838357] retval = 2097152 [D 20:01:04.838381] exiting distribute returned less than expected [D 20:01:04.838405] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 207618048 fo 234881024 eof 0 [D 20:01:04.838431] ========================================================= [D 20:01:04.838456] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:04.838480] element 0: offset: 0x96845000, size: 2097152 [D 20:01:04.838506] openib_check_cq: completed send wr_id 15e72bf8. [D 20:01:04.838532] openib_check_cq: completed recv wr_id 15e73818. [D 20:01:04.838558] openib_post_rr: da13:3346 bh 34 wr_id 15e73818. [D 20:01:04.838588] post_sr: da13:3346 bh 6 len 48 wr_id 15e72c10. [D 20:01:04.838617] PINT_client_state_machine_test id 70 [D 20:01:04.838654] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:04.838680] ========================================================= [D 20:01:04.838705] PINT_process_request [D 20:01:04.838728] tiling 12 copies [D 20:01:04.838752] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.838779] lvl 0 el 0 blk 0 by 337641472 [D 20:01:04.838804] to 337641472 ta 335544320 fi 369098752 [D 20:01:04.838829] mto 2097152 mta 0 mfi 2097152 [D 20:01:04.838853] basic type or contiguous data [D 20:01:04.838877] Request disp [D 20:01:04.838901] contig_offset = 337641472 contig_size = 65011712 lvl_flag = 1 [D 20:01:04.838926] calling distribute [D 20:01:04.838950] PINT_distribute [D 20:01:04.838973] of 337641472 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:04.839000] begin iteration loff: 337641472 [D 20:01:04.839025] segment exceeds byte limit [D 20:01:04.839049] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:04.839075] **********CALL***PROCESS********* [D 20:01:04.839097] segment of 2097152 sz 2097152 [D 20:01:04.839122] PINT_process_request [D 20:01:04.839146] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.839173] lvl 0 el 0 blk 0 by 2097152 [D 20:01:04.839198] to 2097152 ta 2097152 fi 4194304 [D 20:01:04.839225] basic type or contiguous data [D 20:01:04.839249] Request disp [D 20:01:04.839273] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:04.839298] process a segment [D 20:01:04.839322] of 2097152 sz 2097152 [D 20:01:04.839346] add a segment 0 [D 20:01:04.839370] retval = 2097152 [D 20:01:04.839394] exiting distribute returned less than expected [D 20:01:04.839418] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:04.839445] *****RETURN***FROM***PROCESS***** [D 20:01:04.839468] end iteration [D 20:01:04.839491] done with segments or bytes [D 20:01:04.839515] of 339738624 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.839541] next loff: 339738624 next poff: 339738624 [D 20:01:04.839566] return value 2097152 [D 20:01:04.839591] finished [D 20:01:04.839614] retval = 2097152 [D 20:01:04.839638] exiting distribute returned less than expected [D 20:01:04.839662] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 339738624 fo 369098752 eof 0 [D 20:01:04.839688] ========================================================= [D 20:01:04.839715] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:04.839740] element 0: offset: 0x8e63d000, size: 2097152 [D 20:01:04.839766] openib_check_cq: completed send wr_id 15e72c10. [D 20:01:04.839792] openib_check_cq: completed recv wr_id 15e73830. [D 20:01:04.839818] openib_post_rr: da13:3346 bh 35 wr_id 15e73830. [D 20:01:04.839848] post_sr: da13:3346 bh 7 len 48 wr_id 15e72c28. [D 20:01:04.839885] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:04.839911] ========================================================= [D 20:01:04.839936] PINT_process_request [D 20:01:04.839960] tiling 13 copies [D 20:01:04.839983] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.840010] lvl 0 el 0 blk 0 by 371195904 [D 20:01:04.840035] to 371195904 ta 369098752 fi 402653184 [D 20:01:04.840060] mto 2097152 mta 0 mfi 2097152 [D 20:01:04.840085] basic type or contiguous data [D 20:01:04.840108] Request disp [D 20:01:04.840131] contig_offset = 371195904 contig_size = 65011712 lvl_flag = 1 [D 20:01:04.840157] calling distribute [D 20:01:04.840181] PINT_distribute [D 20:01:04.840205] of 371195904 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:04.840231] begin iteration loff: 371195904 [D 20:01:04.840256] segment exceeds byte limit [D 20:01:04.840280] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:04.840304] **********CALL***PROCESS********* [D 20:01:04.840328] segment of 2097152 sz 2097152 [D 20:01:04.840352] PINT_process_request [D 20:01:04.840376] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.840402] lvl 0 el 0 blk 0 by 2097152 [D 20:01:04.840427] to 2097152 ta 2097152 fi 4194304 [D 20:01:04.840452] basic type or contiguous data [D 20:01:04.840476] Request disp [D 20:01:04.840498] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:04.840524] process a segment [D 20:01:04.840547] of 2097152 sz 2097152 [D 20:01:04.840572] add a segment 0 [D 20:01:04.840596] retval = 2097152 [D 20:01:04.840620] exiting distribute returned less than expected [D 20:01:04.840643] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:04.840669] *****RETURN***FROM***PROCESS***** [D 20:01:04.840694] end iteration [D 20:01:04.840718] done with segments or bytes [D 20:01:04.840741] of 373293056 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.840766] next loff: 373293056 next poff: 373293056 [D 20:01:04.840791] return value 2097152 [D 20:01:04.840815] finished [D 20:01:04.840839] retval = 2097152 [D 20:01:04.840863] exiting distribute returned less than expected [D 20:01:04.840887] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 373293056 fo 402653184 eof 0 [D 20:01:04.840913] ========================================================= [D 20:01:04.840940] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:04.840966] element 0: offset: 0x8c63b000, size: 2097152 [D 20:01:04.840991] openib_check_cq: completed send wr_id 15e72c28. [D 20:01:04.841022] post_sr: da13:3346 bh 8 len 48 wr_id 15e72c40. [D 20:01:04.841093] openib_check_cq: completed send wr_id 15e72c40. [D 20:01:04.844128] openib_check_cq: completed recv wr_id 15e73848. [D 20:01:04.844156] openib_post_rr: da13:3346 bh 36 wr_id 15e73848. [D 20:01:04.844191] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:04.844217] ========================================================= [D 20:01:04.844242] PINT_process_request [D 20:01:04.844265] tiling 11 copies [D 20:01:04.844289] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.844316] lvl 0 el 0 blk 0 by 304087040 [D 20:01:04.844340] to 304087040 ta 301989888 fi 335544320 [D 20:01:04.844365] mto 2097152 mta 0 mfi 2097152 [D 20:01:04.844389] basic type or contiguous data [D 20:01:04.844413] Request disp [D 20:01:04.844437] contig_offset = 304087040 contig_size = 65011712 lvl_flag = 1 [D 20:01:04.844463] calling distribute [D 20:01:04.844487] PINT_distribute [D 20:01:04.844510] of 304087040 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:04.844536] begin iteration loff: 304087040 [D 20:01:04.844561] segment exceeds byte limit [D 20:01:04.844585] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:04.844610] **********CALL***PROCESS********* [D 20:01:04.844634] segment of 2097152 sz 2097152 [D 20:01:04.844658] PINT_process_request [D 20:01:04.844682] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.844713] lvl 0 el 0 blk 0 by 2097152 [D 20:01:04.844738] to 2097152 ta 2097152 fi 4194304 [D 20:01:04.844762] basic type or contiguous data [D 20:01:04.844786] Request disp [D 20:01:04.844810] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:04.844835] process a segment [D 20:01:04.844859] of 2097152 sz 2097152 [D 20:01:04.844882] add a segment 0 [D 20:01:04.844906] retval = 2097152 [D 20:01:04.844930] exiting distribute returned less than expected [D 20:01:04.844955] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:04.844981] *****RETURN***FROM***PROCESS***** [D 20:01:04.845005] end iteration [D 20:01:04.845028] done with segments or bytes [D 20:01:04.845052] of 306184192 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.845078] next loff: 306184192 next poff: 306184192 [D 20:01:04.845102] return value 2097152 [D 20:01:04.845127] finished [D 20:01:04.845150] retval = 2097152 [D 20:01:04.845174] exiting distribute returned less than expected [D 20:01:04.845198] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 306184192 fo 335544320 eof 0 [D 20:01:04.845224] ========================================================= [D 20:01:04.845249] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:04.845273] element 0: offset: 0x9063f000, size: 2097152 [D 20:01:04.845304] post_sr: da13:3346 bh 9 len 48 wr_id 15e72c58. [D 20:01:04.845375] openib_check_cq: completed send wr_id 15e72c58. [D 20:01:04.848575] openib_check_cq: completed recv wr_id 15e73860. [D 20:01:04.848602] openib_post_rr: da13:3346 bh 37 wr_id 15e73860. [D 20:01:04.848637] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:04.848664] ========================================================= [D 20:01:04.848688] PINT_process_request [D 20:01:04.848712] tiling 14 copies [D 20:01:04.848736] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.848763] lvl 0 el 0 blk 0 by 404750336 [D 20:01:04.848788] to 404750336 ta 402653184 fi 436207616 [D 20:01:04.848813] mto 2097152 mta 0 mfi 2097152 [D 20:01:04.848837] basic type or contiguous data [D 20:01:04.848861] Request disp [D 20:01:04.848885] contig_offset = 404750336 contig_size = 65011712 lvl_flag = 1 [D 20:01:04.848913] calling distribute [D 20:01:04.848937] PINT_distribute [D 20:01:04.848960] of 404750336 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:04.848987] begin iteration loff: 404750336 [D 20:01:04.849012] segment exceeds byte limit [D 20:01:04.849036] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:04.849061] **********CALL***PROCESS********* [D 20:01:04.849084] segment of 2097152 sz 2097152 [D 20:01:04.849109] PINT_process_request [D 20:01:04.849133] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.849159] lvl 0 el 0 blk 0 by 2097152 [D 20:01:04.849184] to 2097152 ta 2097152 fi 4194304 [D 20:01:04.849208] basic type or contiguous data [D 20:01:04.849232] Request disp [D 20:01:04.849256] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:04.849281] process a segment [D 20:01:04.849304] of 2097152 sz 2097152 [D 20:01:04.849328] add a segment 0 [D 20:01:04.849352] retval = 2097152 [D 20:01:04.849376] exiting distribute returned less than expected [D 20:01:04.849400] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:04.849427] *****RETURN***FROM***PROCESS***** [D 20:01:04.849451] end iteration [D 20:01:04.849474] done with segments or bytes [D 20:01:04.849498] of 406847488 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.849524] next loff: 406847488 next poff: 406847488 [D 20:01:04.849549] return value 2097152 [D 20:01:04.849573] finished [D 20:01:04.849596] retval = 2097152 [D 20:01:04.849620] exiting distribute returned less than expected [D 20:01:04.849645] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 406847488 fo 436207616 eof 0 [D 20:01:04.849671] ========================================================= [D 20:01:04.849696] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:04.849724] element 0: offset: 0x8a639000, size: 2097152 [D 20:01:04.849761] post_sr: da13:3346 bh 10 len 48 wr_id 15e72c70. [D 20:01:04.849792] PINT_client_state_machine_test id 70 [D 20:01:04.849832] openib_check_cq: completed send wr_id 15e72c70. [D 20:01:04.853026] openib_check_cq: completed recv wr_id 15e73878. [D 20:01:04.853053] openib_post_rr: da13:3346 bh 38 wr_id 15e73878. [D 20:01:04.853087] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ee0210. [D 20:01:04.853114] ========================================================= [D 20:01:04.853137] PINT_process_request [D 20:01:04.853161] tiling 16 copies [D 20:01:04.853186] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.853212] lvl 0 el 0 blk 0 by 471859200 [D 20:01:04.853237] to 471859200 ta 469762048 fi 503316480 [D 20:01:04.853261] mto 2097152 mta 0 mfi 2097152 [D 20:01:04.853286] basic type or contiguous data [D 20:01:04.853310] Request disp [D 20:01:04.853334] contig_offset = 471859200 contig_size = 65011712 lvl_flag = 1 [D 20:01:04.853359] calling distribute [D 20:01:04.853384] PINT_distribute [D 20:01:04.853407] of 471859200 sz 31457280 ix 0 sm 64 by 0 bm 2097152 fsz 480247808 exfl 0 [D 20:01:04.853433] begin iteration loff: 471859200 [D 20:01:04.853459] segment exceeds byte limit [D 20:01:04.853483] client lstof 2097152 diff 0 sgof 2097152 [D 20:01:04.853507] **********CALL***PROCESS********* [D 20:01:04.853531] segment of 2097152 sz 2097152 [D 20:01:04.853556] PINT_process_request [D 20:01:04.853580] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.853606] lvl 0 el 0 blk 0 by 2097152 [D 20:01:04.853631] to 2097152 ta 2097152 fi 4194304 [D 20:01:04.853655] basic type or contiguous data [D 20:01:04.853679] Request disp [D 20:01:04.853705] contig_offset = 2097152 contig_size = 31457280 lvl_flag = 1 [D 20:01:04.853734] process a segment [D 20:01:04.853762] of 2097152 sz 2097152 [D 20:01:04.853790] add a segment 0 [D 20:01:04.853817] retval = 2097152 [D 20:01:04.853841] exiting distribute returned less than expected [D 20:01:04.853869] done sg 1 sm 64 by 2097152 bm 2097152 ta 2097152 to 4194304 fo 4194304 eof 0 [D 20:01:04.853895] *****RETURN***FROM***PROCESS***** [D 20:01:04.853919] end iteration [D 20:01:04.853943] done with segments or bytes [D 20:01:04.853967] of 473956352 sz 29360128 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.853993] next loff: 473956352 next poff: 473956352 [D 20:01:04.854018] return value 2097152 [D 20:01:04.854042] finished [D 20:01:04.854065] retval = 2097152 [D 20:01:04.854089] exiting distribute returned less than expected [D 20:01:04.854113] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 473956352 fo 503316480 eof 0 [D 20:01:04.854139] ========================================================= [D 20:01:04.854164] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 37 [D 20:01:04.854188] element 0: offset: 0x86635000, size: 2097152 [D 20:01:04.854220] post_sr: da13:3346 bh 11 len 48 wr_id 15e72c88. [D 20:01:04.854290] openib_check_cq: completed send wr_id 15e72c88. [D 20:01:04.855248] openib_check_cq: completed recv wr_id 15e73890. [D 20:01:04.855276] openib_post_rr: da13:3346 bh 39 wr_id 15e73890. [D 20:01:04.855309] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:04.855335] ========================================================= [D 20:01:04.855360] PINT_process_request [D 20:01:04.855383] tiling 9 copies [D 20:01:04.855407] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.855434] lvl 0 el 0 blk 0 by 239075328 [D 20:01:04.855459] to 239075328 ta 234881024 fi 268435456 [D 20:01:04.855484] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:04.855508] basic type or contiguous data [D 20:01:04.855532] Request disp [D 20:01:04.855555] contig_offset = 239075328 contig_size = 62914560 lvl_flag = 1 [D 20:01:04.855581] calling distribute [D 20:01:04.855605] PINT_distribute [D 20:01:04.855628] of 239075328 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:04.855654] begin iteration loff: 239075328 [D 20:01:04.855679] segment exceeds byte limit [D 20:01:04.855705] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:04.855731] **********CALL***PROCESS********* [D 20:01:04.855754] segment of 4194304 sz 2097152 [D 20:01:04.855778] PINT_process_request [D 20:01:04.855803] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.855829] lvl 0 el 0 blk 0 by 4194304 [D 20:01:04.855854] to 4194304 ta 4194304 fi 6291456 [D 20:01:04.855878] basic type or contiguous data [D 20:01:04.855902] Request disp [D 20:01:04.855925] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:04.855951] process a segment [D 20:01:04.855974] of 4194304 sz 2097152 [D 20:01:04.855998] add a segment 0 [D 20:01:04.856022] retval = 2097152 [D 20:01:04.856046] exiting distribute returned less than expected [D 20:01:04.856070] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:04.856096] *****RETURN***FROM***PROCESS***** [D 20:01:04.856121] end iteration [D 20:01:04.856144] done with segments or bytes [D 20:01:04.856168] of 241172480 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.856194] next loff: 241172480 next poff: 241172480 [D 20:01:04.856219] return value 2097152 [D 20:01:04.856243] finished [D 20:01:04.856266] retval = 2097152 [D 20:01:04.856290] exiting distribute returned less than expected [D 20:01:04.856314] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 241172480 fo 268435456 eof 0 [D 20:01:04.856341] ========================================================= [D 20:01:04.856365] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:04.856390] element 0: offset: 0x94843000, size: 2097152 [D 20:01:04.856420] post_sr: da13:3346 bh 12 len 48 wr_id 15e72ca0. [D 20:01:04.856490] openib_check_cq: completed send wr_id 15e72ca0. [D 20:01:04.859699] openib_check_cq: completed recv wr_id 15e738a8. [D 20:01:04.859729] openib_post_rr: da13:3346 bh 40 wr_id 15e738a8. [D 20:01:04.859763] openib_check_cq: completed recv wr_id 15e738c0. [D 20:01:04.859789] openib_post_rr: da13:3346 bh 41 wr_id 15e738c0. [D 20:01:04.859824] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:04.859849] ========================================================= [D 20:01:04.859873] PINT_process_request [D 20:01:04.859898] tiling 5 copies [D 20:01:04.859922] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.859948] lvl 0 el 0 blk 0 by 117440512 [D 20:01:04.859972] to 117440512 ta 100663296 fi 134217728 [D 20:01:04.859997] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:04.860022] basic type or contiguous data [D 20:01:04.860046] Request disp [D 20:01:04.860070] contig_offset = 117440512 contig_size = 50331648 lvl_flag = 1 [D 20:01:04.860095] calling distribute [D 20:01:04.860119] PINT_distribute [D 20:01:04.860143] of 117440512 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:04.860170] begin iteration loff: 117440512 [D 20:01:04.860195] segment exceeds byte limit [D 20:01:04.860219] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:04.860243] **********CALL***PROCESS********* [D 20:01:04.860267] segment of 16777216 sz 2097152 [D 20:01:04.860292] PINT_process_request [D 20:01:04.860316] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.860342] lvl 0 el 0 blk 0 by 16777216 [D 20:01:04.860366] to 16777216 ta 16777216 fi 18874368 [D 20:01:04.860391] basic type or contiguous data [D 20:01:04.860415] Request disp [D 20:01:04.860439] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:04.860464] process a segment [D 20:01:04.860487] of 16777216 sz 2097152 [D 20:01:04.860511] add a segment 0 [D 20:01:04.860535] retval = 2097152 [D 20:01:04.860559] exiting distribute returned less than expected [D 20:01:04.860583] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:04.860610] *****RETURN***FROM***PROCESS***** [D 20:01:04.860633] end iteration [D 20:01:04.860657] done with segments or bytes [D 20:01:04.860681] of 119537664 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.860707] next loff: 119537664 next poff: 119537664 [D 20:01:04.860732] return value 2097152 [D 20:01:04.860755] finished [D 20:01:04.860779] retval = 2097152 [D 20:01:04.860803] exiting distribute returned less than expected [D 20:01:04.860827] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 119537664 fo 134217728 eof 0 [D 20:01:04.860854] ========================================================= [D 20:01:04.860877] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:04.860903] element 0: offset: 0x9d44b000, size: 2097152 [D 20:01:04.860932] post_sr: da13:3346 bh 13 len 48 wr_id 15e72cb8. [D 20:01:04.860959] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:04.860983] ========================================================= [D 20:01:04.861008] PINT_process_request [D 20:01:04.861032] tiling 10 copies [D 20:01:04.861056] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.861083] lvl 0 el 0 blk 0 by 272629760 [D 20:01:04.861107] to 272629760 ta 268435456 fi 301989888 [D 20:01:04.861131] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:04.861156] basic type or contiguous data [D 20:01:04.861180] Request disp [D 20:01:04.861205] contig_offset = 272629760 contig_size = 62914560 lvl_flag = 1 [D 20:01:04.861229] calling distribute [D 20:01:04.861253] PINT_distribute [D 20:01:04.861277] of 272629760 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:04.861304] begin iteration loff: 272629760 [D 20:01:04.861329] segment exceeds byte limit [D 20:01:04.861352] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:04.861377] **********CALL***PROCESS********* [D 20:01:04.861401] segment of 4194304 sz 2097152 [D 20:01:04.861425] PINT_process_request [D 20:01:04.861449] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.861478] lvl 0 el 0 blk 0 by 4194304 [D 20:01:04.861503] to 4194304 ta 4194304 fi 6291456 [D 20:01:04.861528] basic type or contiguous data [D 20:01:04.861552] Request disp [D 20:01:04.861576] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:04.861600] process a segment [D 20:01:04.861624] of 4194304 sz 2097152 [D 20:01:04.861648] add a segment 0 [D 20:01:04.861672] retval = 2097152 [D 20:01:04.861696] exiting distribute returned less than expected [D 20:01:04.861724] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:04.861755] *****RETURN***FROM***PROCESS***** [D 20:01:04.861783] end iteration [D 20:01:04.861810] done with segments or bytes [D 20:01:04.861835] of 274726912 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.861860] next loff: 274726912 next poff: 274726912 [D 20:01:04.861885] return value 2097152 [D 20:01:04.861909] finished [D 20:01:04.861933] retval = 2097152 [D 20:01:04.861957] exiting distribute returned less than expected [D 20:01:04.861981] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 274726912 fo 301989888 eof 0 [D 20:01:04.862008] ========================================================= [D 20:01:04.862031] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:04.862057] element 0: offset: 0x92841000, size: 2097152 [D 20:01:04.862083] openib_check_cq: completed send wr_id 15e72cb8. [D 20:01:04.862109] openib_check_cq: completed recv wr_id 15e738d8. [D 20:01:04.862135] openib_post_rr: da13:3346 bh 42 wr_id 15e738d8. [D 20:01:04.862161] openib_check_cq: completed recv wr_id 15e738f0. [D 20:01:04.862188] openib_post_rr: da13:3346 bh 43 wr_id 15e738f0. [D 20:01:04.862214] openib_check_cq: completed recv wr_id 15e73908. [D 20:01:04.862242] openib_post_rr: da13:3346 bh 44 wr_id 15e73908. [D 20:01:04.862269] openib_check_cq: completed recv wr_id 15e73920. [D 20:01:04.862297] openib_post_rr: da13:3346 bh 45 wr_id 15e73920. [D 20:01:04.862323] openib_check_cq: completed recv wr_id 15e73938. [D 20:01:04.862351] openib_post_rr: da13:3346 bh 46 wr_id 15e73938. [D 20:01:04.862381] post_sr: da13:3346 bh 14 len 48 wr_id 15e72cd0. [D 20:01:04.862410] PINT_client_state_machine_test id 70 [D 20:01:04.862446] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb5410. [D 20:01:04.862473] flowproto completing 0x15eb5410 [D 20:01:04.862501] Job flows in progress (callback time): 12 [D 20:01:04.862526] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:04.862551] ========================================================= [D 20:01:04.862576] PINT_process_request [D 20:01:04.862600] tiling 4 copies [D 20:01:04.862624] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.862649] lvl 0 el 0 blk 0 by 98566144 [D 20:01:04.862674] to 98566144 ta 67108864 fi 100663296 [D 20:01:04.862699] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:04.862724] basic type or contiguous data [D 20:01:04.862748] Request disp [D 20:01:04.862771] contig_offset = 98566144 contig_size = 35651584 lvl_flag = 1 [D 20:01:04.862796] calling distribute [D 20:01:04.862820] PINT_distribute [D 20:01:04.862844] of 98566144 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 440401920 exfl 0 [D 20:01:04.862871] begin iteration loff: 98566144 [D 20:01:04.862895] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:04.862921] **********CALL***PROCESS********* [D 20:01:04.862944] segment of 31457280 sz 2097152 [D 20:01:04.862969] PINT_process_request [D 20:01:04.862993] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:04.863019] lvl 0 el 0 blk 0 by 31457280 [D 20:01:04.863044] to 31457280 ta 31457280 fi 33554432 [D 20:01:04.863069] basic type or contiguous data [D 20:01:04.863093] Request disp [D 20:01:04.863117] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:04.863142] process a segment [D 20:01:04.863165] of 31457280 sz 2097152 [D 20:01:04.863191] add a segment 0 [D 20:01:04.863216] retval = 2097152 [D 20:01:04.863240] return from level -1 [D 20:01:04.863264] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:04.863290] *****RETURN***FROM***PROCESS***** [D 20:01:04.863314] end iteration [D 20:01:04.863338] done with segments or bytes [D 20:01:04.863362] of 100663296 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:04.863388] next loff: 100663296 next poff: 100663296 [D 20:01:04.863412] (return value) 2097152 [D 20:01:04.863436] finished [D 20:01:04.863460] retval = 2097152 [D 20:01:04.863484] exiting distribute returned less than expected [D 20:01:04.863509] done sg 1 sm 64 by 2097152 bm 2097152 ta 67108864 to 100663296 fo 100663296 eof 0 [D 20:01:04.863534] ========================================================= [D 20:01:04.863559] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 25 [D 20:01:04.863584] element 0: offset: 0xa024d000, size: 2097152 [D 20:01:04.863611] openib_check_cq: completed send wr_id 15e72cd0. [D 20:01:04.863639] post_sr: da13:3346 bh 15 len 48 wr_id 15e72ce8. [D 20:01:04.863670] sm control op 6 [D 20:01:04.863696] [SM Entering]: (0x15eb45f8) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:04.863724] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.863749] (0x15eb4690) io_datafile_complete_operations (tag 2) [D 20:01:04.863775] matched completed flow for context 0x15eb4e60 [D 20:01:04.863801] *** all operations completed (msgpairs, flows, write acks) [D 20:01:04.863825] [SM Exiting]: (0x15eb45f8) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:04.863851] [SM Entering]: (0x15eb45f8) client_io_sm:io_analyze_results (status: 2) [D 20:01:04.863877] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.863902] (0x15eb4690) io state: io_analyze_results [D 20:01:04.863926] - io_check_context_status called [D 20:01:04.863951] 33554432 bytes read from context 0x15eb4e60 [D 20:01:04.863976] [1/1] running size is 33554432 [D 20:01:04.864001] [2/1] running size is 33554432 [D 20:01:04.864029] no context errors found; using: Success [D 20:01:04.864055] total bytes transferred is 33554432 [D 20:01:04.864080] [SM Exiting]: (0x15eb45f8) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:04.864106] [SM Entering]: (0x15eb45f8) client_io_sm:io_cleanup (status: 2) [D 20:01:04.864131] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:04.864156] (0x15eb4690) io state: io_cleanup [D 20:01:04.864184] [SM Exiting]: (0x15eb45f8) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:04.864210] client_state_machine_terminate smcb 0x15eb45f8 [D 20:01:04.864234] add smcb 0x15eb45f8 to completion list [D 20:01:04.864259] [SM frame get]: (0x15eb45f8) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 18 get_next_full_readbuf: enter userFilePos 67108864, read_ahead_offset 503316480 -- buffer 0x15e327f8 curbuf 19 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 19 aio_req 0x15e74640 b->offset 67108864 b->size 33554432 [D 20:01:05.221025] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:01:05.221071] PINT_client_state_machine_test id 73 [D 20:01:05.221106] openib_check_cq: completed send wr_id 15e72ce8. [D 20:01:05.221134] openib_check_cq: completed recv wr_id 15e73950. [D 20:01:05.221162] openib_post_rr: da13:3346 bh 47 wr_id 15e73950. [D 20:01:05.221190] openib_check_cq: completed recv wr_id 15e73968. [D 20:01:05.221229] openib_post_rr: da13:3346 bh 48 wr_id 15e73968. [D 20:01:05.221256] openib_check_cq: completed recv wr_id 15e73980. [D 20:01:05.221283] openib_post_rr: da13:3346 bh 49 wr_id 15e73980. [D 20:01:05.221309] openib_check_cq: completed recv wr_id 15e73998. [D 20:01:05.221336] openib_post_rr: da13:3346 bh 50 wr_id 15e73998. [D 20:01:05.221363] openib_check_cq: completed recv wr_id 15e739b0. [D 20:01:05.221396] openib_post_rr: da13:3346 bh 51 wr_id 15e739b0. [D 20:01:05.221423] openib_check_cq: completed recv wr_id 15e739c8. [D 20:01:05.221450] openib_post_rr: da13:3346 bh 52 wr_id 15e739c8. [D 20:01:05.221476] openib_check_cq: completed recv wr_id 15e739e0. [D 20:01:05.221506] openib_post_rr: da13:3346 bh 53 wr_id 15e739e0. [D 20:01:05.221532] openib_check_cq: completed recv wr_id 15e739f8. [D 20:01:05.221561] openib_post_rr: da13:3346 bh 54 wr_id 15e739f8. [D 20:01:05.221587] openib_check_cq: completed recv wr_id 15e73a10. [D 20:01:05.221614] openib_post_rr: da13:3346 bh 55 wr_id 15e73a10. [D 20:01:05.221641] openib_check_cq: completed recv wr_id 15e73a28. [D 20:01:05.221670] openib_post_rr: da13:3346 bh 56 wr_id 15e73a28. [D 20:01:05.221696] openib_check_cq: completed recv wr_id 15e73a40. [D 20:01:05.221729] openib_post_rr: da13:3346 bh 57 wr_id 15e73a40. [D 20:01:05.221759] openib_check_cq: completed recv wr_id 15e73a58. [D 20:01:05.221791] openib_post_rr: da13:3346 bh 58 wr_id 15e73a58. [D 20:01:05.221820] openib_check_cq: completed recv wr_id 15e73a70. [D 20:01:05.221848] openib_post_rr: da13:3346 bh 59 wr_id 15e73a70. [D 20:01:05.221875] openib_check_cq: completed recv wr_id 15e73a88. [D 20:01:05.221900] openib_post_rr: da13:3346 bh 60 wr_id 15e73a88. [D 20:01:05.221927] openib_check_cq: completed recv wr_id 15e73aa0. [D 20:01:05.221954] openib_post_rr: da13:3346 bh 61 wr_id 15e73aa0. [D 20:01:05.221981] openib_check_cq: completed recv wr_id 15e73ab8. [D 20:01:05.222006] openib_post_rr: da13:3346 bh 62 wr_id 15e73ab8. [D 20:01:05.222033] openib_check_cq: completed recv wr_id 15e73ad0. [D 20:01:05.222061] openib_post_rr: da13:3346 bh 63 wr_id 15e73ad0. [D 20:01:05.222087] openib_check_cq: completed recv wr_id 15e73ae8. [D 20:01:05.222116] openib_post_rr: da13:3346 bh 64 wr_id 15e73ae8. [D 20:01:05.222143] openib_check_cq: completed recv wr_id 15e73b00. [D 20:01:05.222171] openib_post_rr: da13:3346 bh 65 wr_id 15e73b00. [D 20:01:05.222198] openib_check_cq: completed recv wr_id 15e73b18. [D 20:01:05.222227] openib_post_rr: da13:3346 bh 66 wr_id 15e73b18. [D 20:01:05.222253] openib_check_cq: completed recv wr_id 15e73b30. [D 20:01:05.222282] openib_post_rr: da13:3346 bh 67 wr_id 15e73b30. [D 20:01:05.222308] openib_check_cq: completed recv wr_id 15e73b48. [D 20:01:05.222336] openib_post_rr: da13:3346 bh 68 wr_id 15e73b48. [D 20:01:05.222363] openib_check_cq: completed recv wr_id 15e73b60. [D 20:01:05.222392] openib_post_rr: da13:3346 bh 69 wr_id 15e73b60. [D 20:01:05.222417] openib_check_cq: completed recv wr_id 15e73b78. [D 20:01:05.222447] openib_post_rr: da13:3346 bh 70 wr_id 15e73b78. [D 20:01:05.222473] openib_check_cq: completed recv wr_id 15e73b90. [D 20:01:05.222501] openib_post_rr: da13:3346 bh 71 wr_id 15e73b90. [D 20:01:05.222541] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:05.222568] ========================================================= [D 20:01:05.222593] PINT_process_request [D 20:01:05.222618] tiling 6 copies [D 20:01:05.222642] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.222669] lvl 0 el 0 blk 0 by 142606336 [D 20:01:05.222694] to 142606336 ta 134217728 fi 167772160 [D 20:01:05.222719] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.222743] basic type or contiguous data [D 20:01:05.222767] Request disp [D 20:01:05.222791] contig_offset = 142606336 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.222817] calling distribute [D 20:01:05.222842] PINT_distribute [D 20:01:05.222866] of 142606336 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.222893] begin iteration loff: 142606336 [D 20:01:05.222919] segment exceeds byte limit [D 20:01:05.222943] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.222968] **********CALL***PROCESS********* [D 20:01:05.222991] segment of 8388608 sz 2097152 [D 20:01:05.223016] PINT_process_request [D 20:01:05.223041] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.223070] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.223094] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.223119] basic type or contiguous data [D 20:01:05.223143] Request disp [D 20:01:05.223167] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.223193] process a segment [D 20:01:05.223217] of 8388608 sz 2097152 [D 20:01:05.223240] add a segment 0 [D 20:01:05.223265] retval = 2097152 [D 20:01:05.223289] exiting distribute returned less than expected [D 20:01:05.223313] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.223339] *****RETURN***FROM***PROCESS***** [D 20:01:05.223363] end iteration [D 20:01:05.223387] done with segments or bytes [D 20:01:05.223411] of 144703488 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.223436] next loff: 144703488 next poff: 144703488 [D 20:01:05.223461] return value 2097152 [D 20:01:05.223485] finished [D 20:01:05.223509] retval = 2097152 [D 20:01:05.223533] exiting distribute returned less than expected [D 20:01:05.223557] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 144703488 fo 167772160 eof 0 [D 20:01:05.223584] ========================================================= [D 20:01:05.223608] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:05.223634] element 0: offset: 0x9ac49000, size: 2097152 [D 20:01:05.223666] post_sr: da13:3346 bh 16 len 48 wr_id 15e72d00. [D 20:01:05.223694] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:05.223723] ========================================================= [D 20:01:05.223747] PINT_process_request [D 20:01:05.223771] tiling 5 copies [D 20:01:05.223794] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.223821] lvl 0 el 0 blk 0 by 119537664 [D 20:01:05.223846] to 119537664 ta 100663296 fi 134217728 [D 20:01:05.223871] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:05.223896] basic type or contiguous data [D 20:01:05.223919] Request disp [D 20:01:05.223943] contig_offset = 119537664 contig_size = 48234496 lvl_flag = 1 [D 20:01:05.223968] calling distribute [D 20:01:05.223992] PINT_distribute [D 20:01:05.224016] of 119537664 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:05.224042] begin iteration loff: 119537664 [D 20:01:05.224067] segment exceeds byte limit [D 20:01:05.224091] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:05.224116] **********CALL***PROCESS********* [D 20:01:05.224139] segment of 18874368 sz 2097152 [D 20:01:05.224164] PINT_process_request [D 20:01:05.224187] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.224213] lvl 0 el 0 blk 0 by 18874368 [D 20:01:05.224238] to 18874368 ta 18874368 fi 20971520 [D 20:01:05.224263] basic type or contiguous data [D 20:01:05.224287] Request disp [D 20:01:05.224310] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:05.224335] process a segment [D 20:01:05.224359] of 18874368 sz 2097152 [D 20:01:05.224383] add a segment 0 [D 20:01:05.224407] retval = 2097152 [D 20:01:05.224431] exiting distribute returned less than expected [D 20:01:05.224454] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:05.224481] *****RETURN***FROM***PROCESS***** [D 20:01:05.224505] end iteration [D 20:01:05.224529] done with segments or bytes [D 20:01:05.224553] of 121634816 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.224578] next loff: 121634816 next poff: 121634816 [D 20:01:05.224603] return value 2097152 [D 20:01:05.224627] finished [D 20:01:05.224651] retval = 2097152 [D 20:01:05.224675] exiting distribute returned less than expected [D 20:01:05.224699] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 121634816 fo 134217728 eof 0 [D 20:01:05.224725] ========================================================= [D 20:01:05.224749] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:05.224777] element 0: offset: 0x9d64b000, size: 2097152 [D 20:01:05.224803] openib_check_cq: completed send wr_id 15e72d00. [D 20:01:05.224833] post_sr: da13:3346 bh 17 len 48 wr_id 15e72d18. [D 20:01:05.224860] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15eb88e8. [D 20:01:05.224886] flowproto completing 0x15eb88e8 [D 20:01:05.224914] Job flows in progress (callback time): 11 [D 20:01:05.224940] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:05.224966] ========================================================= [D 20:01:05.224990] PINT_process_request [D 20:01:05.225014] tiling 7 copies [D 20:01:05.225038] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.225065] lvl 0 el 0 blk 0 by 174063616 [D 20:01:05.225090] to 174063616 ta 167772160 fi 201326592 [D 20:01:05.225114] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:05.225139] basic type or contiguous data [D 20:01:05.225163] Request disp [D 20:01:05.225187] contig_offset = 174063616 contig_size = 60817408 lvl_flag = 1 [D 20:01:05.225212] calling distribute [D 20:01:05.225235] PINT_distribute [D 20:01:05.225259] of 174063616 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.225286] begin iteration loff: 174063616 [D 20:01:05.225311] segment exceeds byte limit [D 20:01:05.225335] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:05.225359] **********CALL***PROCESS********* [D 20:01:05.225383] segment of 6291456 sz 2097152 [D 20:01:05.225408] PINT_process_request [D 20:01:05.225432] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.225458] lvl 0 el 0 blk 0 by 6291456 [D 20:01:05.225482] to 6291456 ta 6291456 fi 8388608 [D 20:01:05.225507] basic type or contiguous data [D 20:01:05.225531] Request disp [D 20:01:05.225555] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:05.225580] process a segment [D 20:01:05.225604] of 6291456 sz 2097152 [D 20:01:05.225627] add a segment 0 [D 20:01:05.225651] retval = 2097152 [D 20:01:05.225675] exiting distribute returned less than expected [D 20:01:05.225699] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:05.225730] *****RETURN***FROM***PROCESS***** [D 20:01:05.225758] end iteration [D 20:01:05.225785] done with segments or bytes [D 20:01:05.225812] of 176160768 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.225839] next loff: 176160768 next poff: 176160768 [D 20:01:05.225864] return value 2097152 [D 20:01:05.225888] finished [D 20:01:05.225912] retval = 2097152 [D 20:01:05.225935] exiting distribute returned less than expected [D 20:01:05.225959] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 176160768 fo 201326592 eof 0 [D 20:01:05.225986] ========================================================= [D 20:01:05.226010] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:05.226036] element 0: offset: 0x98a47000, size: 2097152 [D 20:01:05.226061] openib_check_cq: completed send wr_id 15e72d18. [D 20:01:05.226090] post_sr: da13:3346 bh 18 len 48 wr_id 15e72d30. [D 20:01:05.226119] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:05.226144] ========================================================= [D 20:01:05.226169] PINT_process_request [D 20:01:05.226193] tiling 8 copies [D 20:01:05.226218] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.226244] lvl 0 el 0 blk 0 by 207618048 [D 20:01:05.226269] to 207618048 ta 201326592 fi 234881024 [D 20:01:05.226294] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:05.226319] basic type or contiguous data [D 20:01:05.226343] Request disp [D 20:01:05.226366] contig_offset = 207618048 contig_size = 60817408 lvl_flag = 1 [D 20:01:05.226392] calling distribute [D 20:01:05.226416] PINT_distribute [D 20:01:05.226439] of 207618048 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.226467] begin iteration loff: 207618048 [D 20:01:05.226494] segment exceeds byte limit [D 20:01:05.226518] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:05.226543] **********CALL***PROCESS********* [D 20:01:05.226567] segment of 6291456 sz 2097152 [D 20:01:05.226592] PINT_process_request [D 20:01:05.226615] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.226641] lvl 0 el 0 blk 0 by 6291456 [D 20:01:05.226666] to 6291456 ta 6291456 fi 8388608 [D 20:01:05.226691] basic type or contiguous data [D 20:01:05.226715] Request disp [D 20:01:05.226738] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:05.226763] process a segment [D 20:01:05.226787] of 6291456 sz 2097152 [D 20:01:05.226811] add a segment 0 [D 20:01:05.226835] retval = 2097152 [D 20:01:05.226860] exiting distribute returned less than expected [D 20:01:05.226883] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:05.226909] *****RETURN***FROM***PROCESS***** [D 20:01:05.226933] end iteration [D 20:01:05.226958] done with segments or bytes [D 20:01:05.226981] of 209715200 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.227006] next loff: 209715200 next poff: 209715200 [D 20:01:05.227031] return value 2097152 [D 20:01:05.227056] finished [D 20:01:05.227079] retval = 2097152 [D 20:01:05.227103] exiting distribute returned less than expected [D 20:01:05.227128] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 209715200 fo 234881024 eof 0 [D 20:01:05.227153] ========================================================= [D 20:01:05.227178] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:05.227203] element 0: offset: 0x96a45000, size: 2097152 [D 20:01:05.227229] openib_check_cq: completed send wr_id 15e72d30. [D 20:01:05.227258] post_sr: da13:3346 bh 19 len 48 wr_id 15e72d48. [D 20:01:05.227289] sm control op 6 [D 20:01:05.227314] [SM Entering]: (0x15eb7ad0) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:05.227340] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:01:05.227366] (0x15eb7b68) io_datafile_complete_operations (tag 2) [D 20:01:05.227392] matched completed flow for context 0x15eb8338 [D 20:01:05.227417] *** all operations completed (msgpairs, flows, write acks) [D 20:01:05.227442] [SM Exiting]: (0x15eb7ad0) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:05.227468] [SM Entering]: (0x15eb7ad0) client_io_sm:io_analyze_results (status: 2) [D 20:01:05.227494] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:01:05.227519] (0x15eb7b68) io state: io_analyze_results [D 20:01:05.227544] - io_check_context_status called [D 20:01:05.227569] 33554432 bytes read from context 0x15eb8338 [D 20:01:05.227594] [1/1] running size is 33554432 [D 20:01:05.227619] [2/1] running size is 33554432 [D 20:01:05.227649] no context errors found; using: Success [D 20:01:05.227674] total bytes transferred is 33554432 [D 20:01:05.227699] [SM Exiting]: (0x15eb7ad0) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:05.227729] [SM Entering]: (0x15eb7ad0) client_io_sm:io_cleanup (status: 2) [D 20:01:05.227754] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 [D 20:01:05.227780] (0x15eb7b68) io state: io_cleanup [D 20:01:05.227807] [SM Exiting]: (0x15eb7ad0) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:05.227833] client_state_machine_terminate smcb 0x15eb7ad0 [D 20:01:05.227857] add smcb 0x15eb7ad0 to completion list [D 20:01:05.227883] [SM frame get]: (0x15eb7ad0) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 19 get_next_full_readbuf: enter userFilePos 100663296, read_ahead_offset 503316480 -- buffer 0x15e32820 curbuf 20 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 20 aio_req 0x15e74660 b->offset 100663296 b->size 33554432 [D 20:01:05.581972] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:01:05.582025] PINT_client_state_machine_test id 76 [D 20:01:05.582074] sm control op 300 [D 20:01:05.582099] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:01:05.582126] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:01:05.582159] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:01:05.582186] PINT_client_state_machine_test id 76 [D 20:01:05.582219] openib_check_cq: completed send wr_id 15e72d48. [D 20:01:05.582263] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:05.582291] ========================================================= [D 20:01:05.582317] PINT_process_request [D 20:01:05.582341] tiling 9 copies [D 20:01:05.582365] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.582391] lvl 0 el 0 blk 0 by 241172480 [D 20:01:05.582417] to 241172480 ta 234881024 fi 268435456 [D 20:01:05.582442] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:05.582467] basic type or contiguous data [D 20:01:05.582490] Request disp [D 20:01:05.582514] contig_offset = 241172480 contig_size = 60817408 lvl_flag = 1 [D 20:01:05.582540] calling distribute [D 20:01:05.582565] PINT_distribute [D 20:01:05.582589] of 241172480 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.582616] begin iteration loff: 241172480 [D 20:01:05.582641] segment exceeds byte limit [D 20:01:05.582666] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:05.582691] **********CALL***PROCESS********* [D 20:01:05.582715] segment of 6291456 sz 2097152 [D 20:01:05.582739] PINT_process_request [D 20:01:05.582763] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.582790] lvl 0 el 0 blk 0 by 6291456 [D 20:01:05.582815] to 6291456 ta 6291456 fi 8388608 [D 20:01:05.582839] basic type or contiguous data [D 20:01:05.582863] Request disp [D 20:01:05.582887] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:05.582912] process a segment [D 20:01:05.582937] of 6291456 sz 2097152 [D 20:01:05.582961] add a segment 0 [D 20:01:05.582984] retval = 2097152 [D 20:01:05.583008] exiting distribute returned less than expected [D 20:01:05.583032] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:05.583059] *****RETURN***FROM***PROCESS***** [D 20:01:05.583083] end iteration [D 20:01:05.583106] done with segments or bytes [D 20:01:05.583130] of 243269632 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.583156] next loff: 243269632 next poff: 243269632 [D 20:01:05.583181] return value 2097152 [D 20:01:05.583205] finished [D 20:01:05.583229] retval = 2097152 [D 20:01:05.583253] exiting distribute returned less than expected [D 20:01:05.583277] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 243269632 fo 268435456 eof 0 [D 20:01:05.583303] ========================================================= [D 20:01:05.583328] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:05.583353] element 0: offset: 0x94a43000, size: 2097152 [D 20:01:05.583385] post_sr: da13:3346 bh 20 len 48 wr_id 15e72d60. [D 20:01:05.583415] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:05.583440] ========================================================= [D 20:01:05.583464] PINT_process_request [D 20:01:05.583488] tiling 10 copies [D 20:01:05.583513] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.583539] lvl 0 el 0 blk 0 by 274726912 [D 20:01:05.583563] to 274726912 ta 268435456 fi 301989888 [D 20:01:05.583588] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:05.583613] basic type or contiguous data [D 20:01:05.583637] Request disp [D 20:01:05.583661] contig_offset = 274726912 contig_size = 60817408 lvl_flag = 1 [D 20:01:05.583686] calling distribute [D 20:01:05.583712] PINT_distribute [D 20:01:05.583736] of 274726912 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.583768] begin iteration loff: 274726912 [D 20:01:05.583794] segment exceeds byte limit [D 20:01:05.583819] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:05.583843] **********CALL***PROCESS********* [D 20:01:05.583867] segment of 6291456 sz 2097152 [D 20:01:05.583891] PINT_process_request [D 20:01:05.583915] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.583942] lvl 0 el 0 blk 0 by 6291456 [D 20:01:05.583967] to 6291456 ta 6291456 fi 8388608 [D 20:01:05.583992] basic type or contiguous data [D 20:01:05.584015] Request disp [D 20:01:05.584039] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:05.584064] process a segment [D 20:01:05.584088] of 6291456 sz 2097152 [D 20:01:05.584112] add a segment 0 [D 20:01:05.584136] retval = 2097152 [D 20:01:05.584159] exiting distribute returned less than expected [D 20:01:05.584184] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:05.584210] *****RETURN***FROM***PROCESS***** [D 20:01:05.584234] end iteration [D 20:01:05.584258] done with segments or bytes [D 20:01:05.584281] of 276824064 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.584307] next loff: 276824064 next poff: 276824064 [D 20:01:05.584332] return value 2097152 [D 20:01:05.584356] finished [D 20:01:05.584380] retval = 2097152 [D 20:01:05.584403] exiting distribute returned less than expected [D 20:01:05.584427] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 276824064 fo 301989888 eof 0 [D 20:01:05.584454] ========================================================= [D 20:01:05.584478] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:05.584504] element 0: offset: 0x92a41000, size: 2097152 [D 20:01:05.584529] openib_check_cq: completed send wr_id 15e72d60. [D 20:01:05.584559] post_sr: da13:3346 bh 21 len 48 wr_id 15e72d78. [D 20:01:05.584588] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:05.584613] ========================================================= [D 20:01:05.584637] PINT_process_request [D 20:01:05.584661] tiling 12 copies [D 20:01:05.584686] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.584712] lvl 0 el 0 blk 0 by 339738624 [D 20:01:05.584736] to 339738624 ta 335544320 fi 369098752 [D 20:01:05.584761] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:05.584786] basic type or contiguous data [D 20:01:05.584810] Request disp [D 20:01:05.584834] contig_offset = 339738624 contig_size = 62914560 lvl_flag = 1 [D 20:01:05.584858] calling distribute [D 20:01:05.584883] PINT_distribute [D 20:01:05.584906] of 339738624 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.584933] begin iteration loff: 339738624 [D 20:01:05.584958] segment exceeds byte limit [D 20:01:05.584982] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:05.585007] **********CALL***PROCESS********* [D 20:01:05.585031] segment of 4194304 sz 2097152 [D 20:01:05.585056] PINT_process_request [D 20:01:05.585080] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.585105] lvl 0 el 0 blk 0 by 4194304 [D 20:01:05.585130] to 4194304 ta 4194304 fi 6291456 [D 20:01:05.585155] basic type or contiguous data [D 20:01:05.585179] Request disp [D 20:01:05.585203] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:05.585227] process a segment [D 20:01:05.585251] of 4194304 sz 2097152 [D 20:01:05.585276] add a segment 0 [D 20:01:05.585300] retval = 2097152 [D 20:01:05.585324] exiting distribute returned less than expected [D 20:01:05.585349] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:05.585374] *****RETURN***FROM***PROCESS***** [D 20:01:05.585399] end iteration [D 20:01:05.585423] done with segments or bytes [D 20:01:05.585447] of 341835776 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.585473] next loff: 341835776 next poff: 341835776 [D 20:01:05.585500] return value 2097152 [D 20:01:05.585525] finished [D 20:01:05.585549] retval = 2097152 [D 20:01:05.585573] exiting distribute returned less than expected [D 20:01:05.585597] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 341835776 fo 369098752 eof 0 [D 20:01:05.585623] ========================================================= [D 20:01:05.585647] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:05.585673] element 0: offset: 0x8e83d000, size: 2097152 [D 20:01:05.585699] openib_check_cq: completed send wr_id 15e72d78. [D 20:01:05.585734] post_sr: da13:3346 bh 22 len 48 wr_id 15e72d90. [D 20:01:05.585765] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:05.585795] ========================================================= [D 20:01:05.585823] PINT_process_request [D 20:01:05.585846] tiling 13 copies [D 20:01:05.585871] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.585897] lvl 0 el 0 blk 0 by 373293056 [D 20:01:05.585922] to 373293056 ta 369098752 fi 402653184 [D 20:01:05.585947] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:05.585971] basic type or contiguous data [D 20:01:05.585995] Request disp [D 20:01:05.586019] contig_offset = 373293056 contig_size = 62914560 lvl_flag = 1 [D 20:01:05.586045] calling distribute [D 20:01:05.586069] PINT_distribute [D 20:01:05.586092] of 373293056 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.586119] begin iteration loff: 373293056 [D 20:01:05.586144] segment exceeds byte limit [D 20:01:05.586168] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:05.586193] **********CALL***PROCESS********* [D 20:01:05.586216] segment of 4194304 sz 2097152 [D 20:01:05.586241] PINT_process_request [D 20:01:05.586265] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.586291] lvl 0 el 0 blk 0 by 4194304 [D 20:01:05.586316] to 4194304 ta 4194304 fi 6291456 [D 20:01:05.586340] basic type or contiguous data [D 20:01:05.586364] Request disp [D 20:01:05.586388] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:05.586413] process a segment [D 20:01:05.586437] of 4194304 sz 2097152 [D 20:01:05.586461] add a segment 0 [D 20:01:05.586484] retval = 2097152 [D 20:01:05.586508] exiting distribute returned less than expected [D 20:01:05.586532] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:05.586559] *****RETURN***FROM***PROCESS***** [D 20:01:05.586583] end iteration [D 20:01:05.586606] done with segments or bytes [D 20:01:05.586630] of 375390208 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.586656] next loff: 375390208 next poff: 375390208 [D 20:01:05.586681] return value 2097152 [D 20:01:05.586705] finished [D 20:01:05.586729] retval = 2097152 [D 20:01:05.586752] exiting distribute returned less than expected [D 20:01:05.586776] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 375390208 fo 402653184 eof 0 [D 20:01:05.586802] ========================================================= [D 20:01:05.586827] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:05.586853] element 0: offset: 0x8c83b000, size: 2097152 [D 20:01:05.586878] openib_check_cq: completed send wr_id 15e72d90. [D 20:01:05.586909] post_sr: da13:3346 bh 23 len 48 wr_id 15e72da8. [D 20:01:05.586936] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:05.586962] ========================================================= [D 20:01:05.586987] PINT_process_request [D 20:01:05.587011] tiling 11 copies [D 20:01:05.587035] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.587061] lvl 0 el 0 blk 0 by 306184192 [D 20:01:05.587086] to 306184192 ta 301989888 fi 335544320 [D 20:01:05.587111] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:05.587136] basic type or contiguous data [D 20:01:05.587160] Request disp [D 20:01:05.587183] contig_offset = 306184192 contig_size = 62914560 lvl_flag = 1 [D 20:01:05.587211] calling distribute [D 20:01:05.587235] PINT_distribute [D 20:01:05.587259] of 306184192 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.587285] begin iteration loff: 306184192 [D 20:01:05.587310] segment exceeds byte limit [D 20:01:05.587334] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:05.587360] **********CALL***PROCESS********* [D 20:01:05.587383] segment of 4194304 sz 2097152 [D 20:01:05.587408] PINT_process_request [D 20:01:05.587431] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.587457] lvl 0 el 0 blk 0 by 4194304 [D 20:01:05.587482] to 4194304 ta 4194304 fi 6291456 [D 20:01:05.587507] basic type or contiguous data [D 20:01:05.587531] Request disp [D 20:01:05.587554] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:05.587579] process a segment [D 20:01:05.587603] of 4194304 sz 2097152 [D 20:01:05.587627] add a segment 0 [D 20:01:05.587651] retval = 2097152 [D 20:01:05.587675] exiting distribute returned less than expected [D 20:01:05.587698] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:05.587727] *****RETURN***FROM***PROCESS***** [D 20:01:05.587752] end iteration [D 20:01:05.587776] done with segments or bytes [D 20:01:05.587800] of 308281344 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.587825] next loff: 308281344 next poff: 308281344 [D 20:01:05.587849] return value 2097152 [D 20:01:05.587874] finished [D 20:01:05.587898] retval = 2097152 [D 20:01:05.587922] exiting distribute returned less than expected [D 20:01:05.587945] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 308281344 fo 335544320 eof 0 [D 20:01:05.587971] ========================================================= [D 20:01:05.587998] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:05.588024] element 0: offset: 0x9083f000, size: 2097152 [D 20:01:05.588050] openib_check_cq: completed send wr_id 15e72da8. [D 20:01:05.588080] post_sr: da13:3346 bh 24 len 48 wr_id 15e72dc0. [D 20:01:05.588118] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:05.588145] ========================================================= [D 20:01:05.588169] PINT_process_request [D 20:01:05.588193] tiling 14 copies [D 20:01:05.588217] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.588243] lvl 0 el 0 blk 0 by 406847488 [D 20:01:05.588268] to 406847488 ta 402653184 fi 436207616 [D 20:01:05.588293] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:05.588317] basic type or contiguous data [D 20:01:05.588342] Request disp [D 20:01:05.588366] contig_offset = 406847488 contig_size = 62914560 lvl_flag = 1 [D 20:01:05.588391] calling distribute [D 20:01:05.588415] PINT_distribute [D 20:01:05.588438] of 406847488 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.588465] begin iteration loff: 406847488 [D 20:01:05.588490] segment exceeds byte limit [D 20:01:05.588514] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:05.588539] **********CALL***PROCESS********* [D 20:01:05.588562] segment of 4194304 sz 2097152 [D 20:01:05.588587] PINT_process_request [D 20:01:05.588611] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.588638] lvl 0 el 0 blk 0 by 4194304 [D 20:01:05.588663] to 4194304 ta 4194304 fi 6291456 [D 20:01:05.588687] basic type or contiguous data [D 20:01:05.588711] Request disp [D 20:01:05.588734] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:05.588760] process a segment [D 20:01:05.588783] of 4194304 sz 2097152 [D 20:01:05.588808] add a segment 0 [D 20:01:05.588831] retval = 2097152 [D 20:01:05.588855] exiting distribute returned less than expected [D 20:01:05.588879] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:05.588905] *****RETURN***FROM***PROCESS***** [D 20:01:05.588932] end iteration [D 20:01:05.588956] done with segments or bytes [D 20:01:05.588980] of 408944640 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.589006] next loff: 408944640 next poff: 408944640 [D 20:01:05.589031] return value 2097152 [D 20:01:05.589055] finished [D 20:01:05.589078] retval = 2097152 [D 20:01:05.589103] exiting distribute returned less than expected [D 20:01:05.589127] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 408944640 fo 436207616 eof 0 [D 20:01:05.589154] ========================================================= [D 20:01:05.589179] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:05.589203] element 0: offset: 0x8a839000, size: 2097152 [D 20:01:05.589230] openib_check_cq: completed send wr_id 15e72dc0. [D 20:01:05.589261] post_sr: da13:3346 bh 25 len 48 wr_id 15e72dd8. [D 20:01:05.589288] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ee0210. [D 20:01:05.589314] ========================================================= [D 20:01:05.589338] PINT_process_request [D 20:01:05.589362] tiling 16 copies [D 20:01:05.589385] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.589412] lvl 0 el 0 blk 0 by 473956352 [D 20:01:05.589437] to 473956352 ta 469762048 fi 503316480 [D 20:01:05.589462] mto 4194304 mta 2097152 mfi 4194304 [D 20:01:05.589487] basic type or contiguous data [D 20:01:05.589510] Request disp [D 20:01:05.589534] contig_offset = 473956352 contig_size = 62914560 lvl_flag = 1 [D 20:01:05.589559] calling distribute [D 20:01:05.589583] PINT_distribute [D 20:01:05.589607] of 473956352 sz 29360128 ix 0 sm 64 by 0 bm 2097152 fsz 480247808 exfl 0 [D 20:01:05.589633] begin iteration loff: 473956352 [D 20:01:05.589658] segment exceeds byte limit [D 20:01:05.589682] client lstof 4194304 diff 0 sgof 4194304 [D 20:01:05.589710] **********CALL***PROCESS********* [D 20:01:05.589737] segment of 4194304 sz 2097152 [D 20:01:05.589766] PINT_process_request [D 20:01:05.589793] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.589822] lvl 0 el 0 blk 0 by 4194304 [D 20:01:05.589847] to 4194304 ta 4194304 fi 6291456 [D 20:01:05.589872] basic type or contiguous data [D 20:01:05.589896] Request disp [D 20:01:05.589920] contig_offset = 4194304 contig_size = 29360128 lvl_flag = 1 [D 20:01:05.589945] process a segment [D 20:01:05.589968] of 4194304 sz 2097152 [D 20:01:05.589993] add a segment 0 [D 20:01:05.590017] retval = 2097152 [D 20:01:05.590041] exiting distribute returned less than expected [D 20:01:05.590065] done sg 1 sm 64 by 2097152 bm 2097152 ta 4194304 to 6291456 fo 6291456 eof 0 [D 20:01:05.590090] *****RETURN***FROM***PROCESS***** [D 20:01:05.590115] end iteration [D 20:01:05.590139] done with segments or bytes [D 20:01:05.590163] of 476053504 sz 27262976 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.590188] next loff: 476053504 next poff: 476053504 [D 20:01:05.590213] return value 2097152 [D 20:01:05.590237] finished [D 20:01:05.590261] retval = 2097152 [D 20:01:05.590285] exiting distribute returned less than expected [D 20:01:05.590309] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 476053504 fo 503316480 eof 0 [D 20:01:05.590335] ========================================================= [D 20:01:05.590359] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 37 [D 20:01:05.590385] element 0: offset: 0x86835000, size: 2097152 [D 20:01:05.590411] openib_check_cq: completed send wr_id 15e72dd8. [D 20:01:05.590441] post_sr: da13:3346 bh 26 len 48 wr_id 15e72df0. [D 20:01:05.590478] openib_check_cq: completed send wr_id 15e72df0. [D 20:01:05.592214] PINT_client_state_machine_test id 76 [D 20:01:05.602249] PINT_client_state_machine_test id 76 [D 20:01:05.612282] PINT_client_state_machine_test id 76 [D 20:01:05.622311] PINT_client_state_machine_test id 76 [D 20:01:05.632346] PINT_client_state_machine_test id 76 [D 20:01:05.642379] PINT_client_state_machine_test id 76 [D 20:01:05.652411] PINT_client_state_machine_test id 76 [D 20:01:05.662441] PINT_client_state_machine_test id 76 [D 20:01:05.672471] PINT_client_state_machine_test id 76 [D 20:01:05.683714] PINT_client_state_machine_test id 76 [D 20:01:05.695713] PINT_client_state_machine_test id 76 [D 20:01:05.707713] PINT_client_state_machine_test id 76 [D 20:01:05.719713] PINT_client_state_machine_test id 76 [D 20:01:05.731713] PINT_client_state_machine_test id 76 [D 20:01:05.743713] PINT_client_state_machine_test id 76 [D 20:01:05.755713] PINT_client_state_machine_test id 76 [D 20:01:05.767713] PINT_client_state_machine_test id 76 [D 20:01:05.779716] PINT_client_state_machine_test id 76 -[D 20:01:05.788191] openib_check_cq: completed recv wr_id 15e73ba8. [D 20:01:05.788242] openib_post_rr: da13:3346 bh 72 wr_id 15e73ba8. [D 20:01:05.788290] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:05.788316] ========================================================= [D 20:01:05.788341] PINT_process_request [D 20:01:05.788366] tiling 6 copies [D 20:01:05.788391] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.788417] lvl 0 el 0 blk 0 by 144703488 [D 20:01:05.788442] to 144703488 ta 134217728 fi 167772160 [D 20:01:05.788468] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:05.788493] basic type or contiguous data [D 20:01:05.788516] Request disp [D 20:01:05.788541] contig_offset = 144703488 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.788567] calling distribute [D 20:01:05.788591] PINT_distribute [D 20:01:05.788616] of 144703488 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.788642] begin iteration loff: 144703488 [D 20:01:05.788668] segment exceeds byte limit [D 20:01:05.788692] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.788718] **********CALL***PROCESS********* [D 20:01:05.788742] segment of 10485760 sz 2097152 [D 20:01:05.788766] PINT_process_request [D 20:01:05.788790] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.788817] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.788842] to 10485760 ta 10485760 fi 12582912 [D 20:01:05.788866] basic type or contiguous data [D 20:01:05.788891] Request disp [D 20:01:05.788915] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.788940] process a segment [D 20:01:05.788964] of 10485760 sz 2097152 [D 20:01:05.788989] add a segment 0 [D 20:01:05.789012] retval = 2097152 [D 20:01:05.789037] exiting distribute returned less than expected [D 20:01:05.789061] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:05.789088] *****RETURN***FROM***PROCESS***** [D 20:01:05.789112] end iteration [D 20:01:05.789136] done with segments or bytes [D 20:01:05.789160] of 146800640 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.789186] next loff: 146800640 next poff: 146800640 [D 20:01:05.789211] return value 2097152 [D 20:01:05.789236] finished [D 20:01:05.789259] retval = 2097152 [D 20:01:05.789283] exiting distribute returned less than expected [D 20:01:05.789307] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 146800640 fo 167772160 eof 0 [D 20:01:05.789334] ========================================================= [D 20:01:05.789359] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:05.789384] element 0: offset: 0x9ae49000, size: 2097152 [D 20:01:05.789416] post_sr: da13:3346 bh 27 len 48 wr_id 15e72e08. [D 20:01:05.789486] openib_check_cq: completed send wr_id 15e72e08. [D 20:01:05.789752] PINT_client_state_machine_test id 76 [D 20:01:05.792634] openib_check_cq: completed recv wr_id 15e73bc0. [D 20:01:05.792661] openib_post_rr: da13:3346 bh 73 wr_id 15e73bc0. [D 20:01:05.792696] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:05.792722] ========================================================= [D 20:01:05.792747] PINT_process_request [D 20:01:05.792771] tiling 5 copies [D 20:01:05.792799] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.792826] lvl 0 el 0 blk 0 by 121634816 [D 20:01:05.792851] to 121634816 ta 100663296 fi 134217728 [D 20:01:05.792876] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:05.792901] basic type or contiguous data [D 20:01:05.792925] Request disp [D 20:01:05.792949] contig_offset = 121634816 contig_size = 46137344 lvl_flag = 1 [D 20:01:05.792975] calling distribute [D 20:01:05.792999] PINT_distribute [D 20:01:05.793023] of 121634816 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:05.793049] begin iteration loff: 121634816 [D 20:01:05.793074] segment exceeds byte limit [D 20:01:05.793098] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:05.793123] **********CALL***PROCESS********* [D 20:01:05.793147] segment of 20971520 sz 2097152 [D 20:01:05.793171] PINT_process_request [D 20:01:05.793195] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.793222] lvl 0 el 0 blk 0 by 20971520 [D 20:01:05.793247] to 20971520 ta 20971520 fi 23068672 [D 20:01:05.793272] basic type or contiguous data [D 20:01:05.793295] Request disp [D 20:01:05.793319] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:05.793344] process a segment [D 20:01:05.793368] of 20971520 sz 2097152 [D 20:01:05.793392] add a segment 0 [D 20:01:05.793416] retval = 2097152 [D 20:01:05.793440] exiting distribute returned less than expected [D 20:01:05.793465] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:05.793492] *****RETURN***FROM***PROCESS***** [D 20:01:05.793517] end iteration [D 20:01:05.793540] done with segments or bytes [D 20:01:05.793564] of 123731968 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.793591] next loff: 123731968 next poff: 123731968 [D 20:01:05.793616] return value 2097152 [D 20:01:05.793640] finished [D 20:01:05.793663] retval = 2097152 [D 20:01:05.793688] exiting distribute returned less than expected [D 20:01:05.793712] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 123731968 fo 134217728 eof 0 [D 20:01:05.793739] ========================================================= [D 20:01:05.793763] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:05.793788] element 0: offset: 0x9d84b000, size: 2097152 [D 20:01:05.793818] post_sr: da13:3346 bh 28 len 48 wr_id 15e72e20. [D 20:01:05.793887] openib_check_cq: completed send wr_id 15e72e20. [D 20:01:05.799785] PINT_client_state_machine_test id 76 [D 20:01:05.809819] PINT_client_state_machine_test id 76 [D 20:01:05.810405] openib_check_cq: completed recv wr_id 15e73bd8. [D 20:01:05.810433] openib_post_rr: da13:3346 bh 74 wr_id 15e73bd8. [D 20:01:05.810460] openib_check_cq: completed recv wr_id 15e73bf0. [D 20:01:05.810487] openib_post_rr: da13:3346 bh 75 wr_id 15e73bf0. [D 20:01:05.810514] openib_check_cq: completed recv wr_id 15e73c08. [D 20:01:05.810541] openib_post_rr: da13:3346 bh 76 wr_id 15e73c08. [D 20:01:05.810575] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:05.810602] ========================================================= [D 20:01:05.810626] PINT_process_request [D 20:01:05.810655] tiling 7 copies [D 20:01:05.810679] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.810705] lvl 0 el 0 blk 0 by 176160768 [D 20:01:05.810730] to 176160768 ta 167772160 fi 201326592 [D 20:01:05.810755] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.810780] basic type or contiguous data [D 20:01:05.810804] Request disp [D 20:01:05.810829] contig_offset = 176160768 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.810853] calling distribute [D 20:01:05.810877] PINT_distribute [D 20:01:05.810901] of 176160768 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.810928] begin iteration loff: 176160768 [D 20:01:05.810953] segment exceeds byte limit [D 20:01:05.810976] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.811004] **********CALL***PROCESS********* [D 20:01:05.811028] segment of 8388608 sz 2097152 [D 20:01:05.811053] PINT_process_request [D 20:01:05.811077] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.811103] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.811128] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.811152] basic type or contiguous data [D 20:01:05.811177] Request disp [D 20:01:05.811201] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.811225] process a segment [D 20:01:05.811249] of 8388608 sz 2097152 [D 20:01:05.811274] add a segment 0 [D 20:01:05.811298] retval = 2097152 [D 20:01:05.811322] exiting distribute returned less than expected [D 20:01:05.811346] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.811372] *****RETURN***FROM***PROCESS***** [D 20:01:05.811396] end iteration [D 20:01:05.811420] done with segments or bytes [D 20:01:05.811444] of 178257920 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.811470] next loff: 178257920 next poff: 178257920 [D 20:01:05.811494] return value 2097152 [D 20:01:05.811518] finished [D 20:01:05.811542] retval = 2097152 [D 20:01:05.811567] exiting distribute returned less than expected [D 20:01:05.811591] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 178257920 fo 201326592 eof 0 [D 20:01:05.811616] ========================================================= [D 20:01:05.811641] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:05.811666] element 0: offset: 0x98c47000, size: 2097152 [D 20:01:05.811696] post_sr: da13:3346 bh 29 len 48 wr_id 15e72e38. [D 20:01:05.811723] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:05.811750] ========================================================= [D 20:01:05.811774] PINT_process_request [D 20:01:05.811798] tiling 8 copies [D 20:01:05.811821] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.811848] lvl 0 el 0 blk 0 by 209715200 [D 20:01:05.811873] to 209715200 ta 201326592 fi 234881024 [D 20:01:05.811898] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.811923] basic type or contiguous data [D 20:01:05.811946] Request disp [D 20:01:05.811970] contig_offset = 209715200 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.811995] calling distribute [D 20:01:05.812019] PINT_distribute [D 20:01:05.812043] of 209715200 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.812069] begin iteration loff: 209715200 [D 20:01:05.812094] segment exceeds byte limit [D 20:01:05.812119] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.812143] **********CALL***PROCESS********* [D 20:01:05.812168] segment of 8388608 sz 2097152 [D 20:01:05.812191] PINT_process_request [D 20:01:05.812218] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.812244] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.812269] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.812294] basic type or contiguous data [D 20:01:05.812317] Request disp [D 20:01:05.812341] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.812366] process a segment [D 20:01:05.812390] of 8388608 sz 2097152 [D 20:01:05.812414] add a segment 0 [D 20:01:05.812438] retval = 2097152 [D 20:01:05.812462] exiting distribute returned less than expected [D 20:01:05.812486] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.812512] *****RETURN***FROM***PROCESS***** [D 20:01:05.812537] end iteration [D 20:01:05.812561] done with segments or bytes [D 20:01:05.812584] of 211812352 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.812610] next loff: 211812352 next poff: 211812352 [D 20:01:05.812635] return value 2097152 [D 20:01:05.812659] finished [D 20:01:05.812683] retval = 2097152 [D 20:01:05.812706] exiting distribute returned less than expected [D 20:01:05.812731] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 211812352 fo 234881024 eof 0 [D 20:01:05.812760] ========================================================= [D 20:01:05.812786] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:05.812811] element 0: offset: 0x96c45000, size: 2097152 [D 20:01:05.812837] openib_check_cq: completed send wr_id 15e72e38. [D 20:01:05.812863] openib_check_cq: completed recv wr_id 15e73c20. [D 20:01:05.812890] openib_post_rr: da13:3346 bh 77 wr_id 15e73c20. [D 20:01:05.812915] openib_check_cq: completed recv wr_id 15e73c38. [D 20:01:05.812942] openib_post_rr: da13:3346 bh 78 wr_id 15e73c38. [D 20:01:05.812969] openib_check_cq: completed recv wr_id 15e73c50. [D 20:01:05.812995] openib_post_rr: da13:3346 bh 79 wr_id 15e73c50. [D 20:01:05.813022] openib_check_cq: completed recv wr_id 15e73c68. [D 20:01:05.813050] openib_post_rr: da13:3346 bh 80 wr_id 15e73c68. [D 20:01:05.813076] openib_check_cq: completed recv wr_id 15e73c80. [D 20:01:05.813103] openib_post_rr: da13:3346 bh 81 wr_id 15e73c80. [D 20:01:05.813129] openib_check_cq: completed recv wr_id 15e73c98. [D 20:01:05.813157] openib_post_rr: da13:3346 bh 82 wr_id 15e73c98. [D 20:01:05.813183] openib_check_cq: completed recv wr_id 15e73cb0. [D 20:01:05.813211] openib_post_rr: da13:3346 bh 83 wr_id 15e73cb0. [D 20:01:05.813237] openib_check_cq: completed recv wr_id 15e73cc8. [D 20:01:05.813262] openib_post_rr: da13:3346 bh 84 wr_id 15e73cc8. [D 20:01:05.813293] post_sr: da13:3346 bh 30 len 48 wr_id 15e72e50. [D 20:01:05.813331] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:05.813357] ========================================================= [D 20:01:05.813382] PINT_process_request [D 20:01:05.813405] tiling 6 copies [D 20:01:05.813430] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.813458] lvl 0 el 0 blk 0 by 146800640 [D 20:01:05.813483] to 146800640 ta 134217728 fi 167772160 [D 20:01:05.813508] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:05.813533] basic type or contiguous data [D 20:01:05.813557] Request disp [D 20:01:05.813581] contig_offset = 146800640 contig_size = 54525952 lvl_flag = 1 [D 20:01:05.813607] calling distribute [D 20:01:05.813631] PINT_distribute [D 20:01:05.813654] of 146800640 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.813681] begin iteration loff: 146800640 [D 20:01:05.813707] segment exceeds byte limit [D 20:01:05.813731] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:05.813755] **********CALL***PROCESS********* [D 20:01:05.813779] segment of 12582912 sz 2097152 [D 20:01:05.813803] PINT_process_request [D 20:01:05.813828] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.813854] lvl 0 el 0 blk 0 by 12582912 [D 20:01:05.813879] to 12582912 ta 12582912 fi 14680064 [D 20:01:05.813903] basic type or contiguous data [D 20:01:05.813927] Request disp [D 20:01:05.813951] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:05.813976] process a segment [D 20:01:05.814000] of 12582912 sz 2097152 [D 20:01:05.814024] add a segment 0 [D 20:01:05.814048] retval = 2097152 [D 20:01:05.814072] exiting distribute returned less than expected [D 20:01:05.814096] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:05.814123] *****RETURN***FROM***PROCESS***** [D 20:01:05.814146] end iteration [D 20:01:05.814170] done with segments or bytes [D 20:01:05.814194] of 148897792 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.814224] next loff: 148897792 next poff: 148897792 [D 20:01:05.814252] return value 2097152 [D 20:01:05.814280] finished [D 20:01:05.814308] retval = 2097152 [D 20:01:05.814334] exiting distribute returned less than expected [D 20:01:05.814358] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 148897792 fo 167772160 eof 0 [D 20:01:05.814385] ========================================================= [D 20:01:05.814410] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:05.814437] element 0: offset: 0x9b049000, size: 2097152 [D 20:01:05.814464] openib_check_cq: completed send wr_id 15e72e50. [D 20:01:05.814494] post_sr: da13:3346 bh 31 len 48 wr_id 15e72e68. [D 20:01:05.814520] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:05.814546] ========================================================= [D 20:01:05.814571] PINT_process_request [D 20:01:05.814595] tiling 9 copies [D 20:01:05.814619] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.814645] lvl 0 el 0 blk 0 by 243269632 [D 20:01:05.814670] to 243269632 ta 234881024 fi 268435456 [D 20:01:05.814696] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.814721] basic type or contiguous data [D 20:01:05.814744] Request disp [D 20:01:05.814768] contig_offset = 243269632 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.814793] calling distribute [D 20:01:05.814818] PINT_distribute [D 20:01:05.814842] of 243269632 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.814867] begin iteration loff: 243269632 [D 20:01:05.814892] segment exceeds byte limit [D 20:01:05.814916] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.814942] **********CALL***PROCESS********* [D 20:01:05.814966] segment of 8388608 sz 2097152 [D 20:01:05.814990] PINT_process_request [D 20:01:05.815013] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.815040] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.815065] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.815089] basic type or contiguous data [D 20:01:05.815113] Request disp [D 20:01:05.815136] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.815162] process a segment [D 20:01:05.815186] of 8388608 sz 2097152 [D 20:01:05.815211] add a segment 0 [D 20:01:05.815235] retval = 2097152 [D 20:01:05.815258] exiting distribute returned less than expected [D 20:01:05.815282] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.815309] *****RETURN***FROM***PROCESS***** [D 20:01:05.815333] end iteration [D 20:01:05.815357] done with segments or bytes [D 20:01:05.815381] of 245366784 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.815407] next loff: 245366784 next poff: 245366784 [D 20:01:05.815431] return value 2097152 [D 20:01:05.815456] finished [D 20:01:05.815480] retval = 2097152 [D 20:01:05.815504] exiting distribute returned less than expected [D 20:01:05.815527] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 245366784 fo 268435456 eof 0 [D 20:01:05.815553] ========================================================= [D 20:01:05.815578] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:05.815603] element 0: offset: 0x94c43000, size: 2097152 [D 20:01:05.815629] openib_check_cq: completed send wr_id 15e72e68. [D 20:01:05.815655] openib_check_cq: completed recv wr_id 15e73ce0. [D 20:01:05.815681] openib_post_rr: da13:3346 bh 85 wr_id 15e73ce0. [D 20:01:05.815711] post_sr: da13:3346 bh 32 len 48 wr_id 15e72e80. [D 20:01:05.815739] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:05.815765] ========================================================= [D 20:01:05.815789] PINT_process_request [D 20:01:05.815814] tiling 10 copies [D 20:01:05.815837] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.815864] lvl 0 el 0 blk 0 by 276824064 [D 20:01:05.815889] to 276824064 ta 268435456 fi 301989888 [D 20:01:05.815914] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.815938] basic type or contiguous data [D 20:01:05.815962] Request disp [D 20:01:05.815986] contig_offset = 276824064 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.816011] calling distribute [D 20:01:05.816036] PINT_distribute [D 20:01:05.816059] of 276824064 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.816085] begin iteration loff: 276824064 [D 20:01:05.816111] segment exceeds byte limit [D 20:01:05.816137] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.816163] **********CALL***PROCESS********* [D 20:01:05.816186] segment of 8388608 sz 2097152 [D 20:01:05.816214] PINT_process_request [D 20:01:05.816238] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.816265] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.816289] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.816314] basic type or contiguous data [D 20:01:05.816337] Request disp [D 20:01:05.816362] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.816387] process a segment [D 20:01:05.816411] of 8388608 sz 2097152 [D 20:01:05.816434] add a segment 0 [D 20:01:05.816458] retval = 2097152 [D 20:01:05.816483] exiting distribute returned less than expected [D 20:01:05.816507] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.816533] *****RETURN***FROM***PROCESS***** [D 20:01:05.816557] end iteration [D 20:01:05.816581] done with segments or bytes [D 20:01:05.816605] of 278921216 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.816631] next loff: 278921216 next poff: 278921216 [D 20:01:05.816656] return value 2097152 [D 20:01:05.816680] finished [D 20:01:05.816703] retval = 2097152 [D 20:01:05.816728] exiting distribute returned less than expected [D 20:01:05.816752] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 278921216 fo 301989888 eof 0 [D 20:01:05.816778] ========================================================= [D 20:01:05.816802] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:05.816827] element 0: offset: 0x92c41000, size: 2097152 [D 20:01:05.816854] openib_check_cq: completed send wr_id 15e72e80. [D 20:01:05.816884] post_sr: da13:3346 bh 33 len 48 wr_id 15e72e98. [D 20:01:05.816911] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:05.816937] ========================================================= [D 20:01:05.816962] PINT_process_request [D 20:01:05.816986] tiling 12 copies [D 20:01:05.817009] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.817035] lvl 0 el 0 blk 0 by 341835776 [D 20:01:05.817061] to 341835776 ta 335544320 fi 369098752 [D 20:01:05.817086] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:05.817111] basic type or contiguous data [D 20:01:05.817134] Request disp [D 20:01:05.817158] contig_offset = 341835776 contig_size = 60817408 lvl_flag = 1 [D 20:01:05.817183] calling distribute [D 20:01:05.817208] PINT_distribute [D 20:01:05.817232] of 341835776 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.817258] begin iteration loff: 341835776 [D 20:01:05.817283] segment exceeds byte limit [D 20:01:05.817307] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:05.817332] **********CALL***PROCESS********* [D 20:01:05.817356] segment of 6291456 sz 2097152 [D 20:01:05.817380] PINT_process_request [D 20:01:05.817404] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.817430] lvl 0 el 0 blk 0 by 6291456 [D 20:01:05.817455] to 6291456 ta 6291456 fi 8388608 [D 20:01:05.817480] basic type or contiguous data [D 20:01:05.817503] Request disp [D 20:01:05.817527] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:05.817553] process a segment [D 20:01:05.817577] of 6291456 sz 2097152 [D 20:01:05.817601] add a segment 0 [D 20:01:05.817625] retval = 2097152 [D 20:01:05.817649] exiting distribute returned less than expected [D 20:01:05.817673] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:05.817700] *****RETURN***FROM***PROCESS***** [D 20:01:05.817724] end iteration [D 20:01:05.817748] done with segments or bytes [D 20:01:05.817771] of 343932928 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.817797] next loff: 343932928 next poff: 343932928 [D 20:01:05.817823] return value 2097152 [D 20:01:05.817847] finished [D 20:01:05.817880] retval = 2097152 [D 20:01:05.817905] exiting distribute returned less than expected [D 20:01:05.817929] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 343932928 fo 369098752 eof 0 [D 20:01:05.817955] ========================================================= [D 20:01:05.817980] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:05.818005] element 0: offset: 0x8ea3d000, size: 2097152 [D 20:01:05.818032] openib_check_cq: completed send wr_id 15e72e98. [D 20:01:05.818061] post_sr: da13:3346 bh 34 len 48 wr_id 15e72eb0. [D 20:01:05.818089] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:05.818115] ========================================================= [D 20:01:05.818139] PINT_process_request [D 20:01:05.818163] tiling 11 copies [D 20:01:05.818187] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.818217] lvl 0 el 0 blk 0 by 308281344 [D 20:01:05.818245] to 308281344 ta 301989888 fi 335544320 [D 20:01:05.818274] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:05.818303] basic type or contiguous data [D 20:01:05.818330] Request disp [D 20:01:05.818354] contig_offset = 308281344 contig_size = 60817408 lvl_flag = 1 [D 20:01:05.818380] calling distribute [D 20:01:05.818404] PINT_distribute [D 20:01:05.818428] of 308281344 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.818454] begin iteration loff: 308281344 [D 20:01:05.818479] segment exceeds byte limit [D 20:01:05.818504] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:05.818529] **********CALL***PROCESS********* [D 20:01:05.818552] segment of 6291456 sz 2097152 [D 20:01:05.818577] PINT_process_request [D 20:01:05.818601] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.818627] lvl 0 el 0 blk 0 by 6291456 [D 20:01:05.818652] to 6291456 ta 6291456 fi 8388608 [D 20:01:05.818676] basic type or contiguous data [D 20:01:05.818700] Request disp [D 20:01:05.818725] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:05.818750] process a segment [D 20:01:05.818774] of 6291456 sz 2097152 [D 20:01:05.818799] add a segment 0 [D 20:01:05.818822] retval = 2097152 [D 20:01:05.818846] exiting distribute returned less than expected [D 20:01:05.818871] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:05.818897] *****RETURN***FROM***PROCESS***** [D 20:01:05.818922] end iteration [D 20:01:05.818945] done with segments or bytes [D 20:01:05.818968] of 310378496 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.818995] next loff: 310378496 next poff: 310378496 [D 20:01:05.819020] return value 2097152 [D 20:01:05.819044] finished [D 20:01:05.819067] retval = 2097152 [D 20:01:05.819091] exiting distribute returned less than expected [D 20:01:05.819116] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 310378496 fo 335544320 eof 0 [D 20:01:05.819142] ========================================================= [D 20:01:05.819167] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:05.819192] element 0: offset: 0x90a3f000, size: 2097152 [D 20:01:05.819218] openib_check_cq: completed send wr_id 15e72eb0. [D 20:01:05.819248] post_sr: da13:3346 bh 35 len 48 wr_id 15e72ec8. [D 20:01:05.819285] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:05.819311] ========================================================= [D 20:01:05.819335] PINT_process_request [D 20:01:05.819360] tiling 5 copies [D 20:01:05.819384] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.819410] lvl 0 el 0 blk 0 by 123731968 [D 20:01:05.819436] to 123731968 ta 100663296 fi 134217728 [D 20:01:05.819460] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:05.819485] basic type or contiguous data [D 20:01:05.819509] Request disp [D 20:01:05.819533] contig_offset = 123731968 contig_size = 44040192 lvl_flag = 1 [D 20:01:05.819558] calling distribute [D 20:01:05.819584] PINT_distribute [D 20:01:05.819608] of 123731968 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:05.819635] begin iteration loff: 123731968 [D 20:01:05.819660] segment exceeds byte limit [D 20:01:05.819684] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:05.819708] **********CALL***PROCESS********* [D 20:01:05.819732] segment of 23068672 sz 2097152 [D 20:01:05.819757] PINT_process_request [D 20:01:05.819781] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.819808] lvl 0 el 0 blk 0 by 23068672 [D 20:01:05.819832] to 23068672 ta 23068672 fi 25165824 [D 20:01:05.819857] basic type or contiguous data [D 20:01:05.819881] Request disp [D 20:01:05.819905] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:05.819930] process a segment [D 20:01:05.819953] of 23068672 sz 2097152 [D 20:01:05.819978] add a segment 0 [D 20:01:05.820002] retval = 2097152 [D 20:01:05.820026] exiting distribute returned less than expected [D 20:01:05.820050] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:05.820076] *****RETURN***FROM***PROCESS***** [D 20:01:05.820100] end iteration [D 20:01:05.820124] done with segments or bytes [D 20:01:05.820148] of 125829120 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.820174] next loff: 125829120 next poff: 125829120 [D 20:01:05.820199] return value 2097152 [D 20:01:05.820225] finished [D 20:01:05.820249] retval = 2097152 [D 20:01:05.820274] exiting distribute returned less than expected [D 20:01:05.820298] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 125829120 fo 134217728 eof 0 [D 20:01:05.820323] ========================================================= [D 20:01:05.820348] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:05.820374] element 0: offset: 0x9da4b000, size: 2097152 [D 20:01:05.820400] openib_check_cq: completed send wr_id 15e72ec8. [D 20:01:05.820425] openib_check_cq: completed recv wr_id 15e73cf8. [D 20:01:05.820451] openib_post_rr: da13:3346 bh 86 wr_id 15e73cf8. [D 20:01:05.820482] post_sr: da13:3346 bh 36 len 48 wr_id 15e72ee0. [D 20:01:05.820508] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:05.820534] ========================================================= [D 20:01:05.820559] PINT_process_request [D 20:01:05.820583] tiling 13 copies [D 20:01:05.820607] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.820633] lvl 0 el 0 blk 0 by 375390208 [D 20:01:05.820658] to 375390208 ta 369098752 fi 402653184 [D 20:01:05.820684] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:05.820709] basic type or contiguous data [D 20:01:05.820733] Request disp [D 20:01:05.820756] contig_offset = 375390208 contig_size = 60817408 lvl_flag = 1 [D 20:01:05.820781] calling distribute [D 20:01:05.820805] PINT_distribute [D 20:01:05.820829] of 375390208 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.820856] begin iteration loff: 375390208 [D 20:01:05.820880] segment exceeds byte limit [D 20:01:05.820904] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:05.820929] **********CALL***PROCESS********* [D 20:01:05.820953] segment of 6291456 sz 2097152 [D 20:01:05.820978] PINT_process_request [D 20:01:05.821002] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.821027] lvl 0 el 0 blk 0 by 6291456 [D 20:01:05.821052] to 6291456 ta 6291456 fi 8388608 [D 20:01:05.821077] basic type or contiguous data [D 20:01:05.821101] Request disp [D 20:01:05.821125] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:05.821149] process a segment [D 20:01:05.821173] of 6291456 sz 2097152 [D 20:01:05.821198] add a segment 0 [D 20:01:05.821222] retval = 2097152 [D 20:01:05.821246] exiting distribute returned less than expected [D 20:01:05.821269] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:05.821299] *****RETURN***FROM***PROCESS***** [D 20:01:05.821324] end iteration [D 20:01:05.821348] done with segments or bytes [D 20:01:05.821372] of 377487360 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.821397] next loff: 377487360 next poff: 377487360 [D 20:01:05.821422] return value 2097152 [D 20:01:05.821447] finished [D 20:01:05.821471] retval = 2097152 [D 20:01:05.821495] exiting distribute returned less than expected [D 20:01:05.821518] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 377487360 fo 402653184 eof 0 [D 20:01:05.821545] ========================================================= [D 20:01:05.821569] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:05.821595] element 0: offset: 0x8ca3b000, size: 2097152 [D 20:01:05.821620] openib_check_cq: completed send wr_id 15e72ee0. [D 20:01:05.821650] post_sr: da13:3346 bh 37 len 48 wr_id 15e72ef8. [D 20:01:05.821678] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:05.821704] ========================================================= [D 20:01:05.821728] PINT_process_request [D 20:01:05.821752] tiling 14 copies [D 20:01:05.821777] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.821804] lvl 0 el 0 blk 0 by 408944640 [D 20:01:05.821828] to 408944640 ta 402653184 fi 436207616 [D 20:01:05.821853] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:05.821878] basic type or contiguous data [D 20:01:05.821902] Request disp [D 20:01:05.821926] contig_offset = 408944640 contig_size = 60817408 lvl_flag = 1 [D 20:01:05.821951] calling distribute [D 20:01:05.821975] PINT_distribute [D 20:01:05.821999] of 408944640 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.822026] begin iteration loff: 408944640 [D 20:01:05.822051] segment exceeds byte limit [D 20:01:05.822074] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:05.822099] **********CALL***PROCESS********* [D 20:01:05.822123] segment of 6291456 sz 2097152 [D 20:01:05.822148] PINT_process_request [D 20:01:05.822172] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.822197] lvl 0 el 0 blk 0 by 6291456 [D 20:01:05.822227] to 6291456 ta 6291456 fi 8388608 [D 20:01:05.822256] basic type or contiguous data [D 20:01:05.822283] Request disp [D 20:01:05.822310] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:05.822337] process a segment [D 20:01:05.822360] of 6291456 sz 2097152 [D 20:01:05.822385] add a segment 0 [D 20:01:05.822410] retval = 2097152 [D 20:01:05.822434] exiting distribute returned less than expected [D 20:01:05.822458] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:05.822484] *****RETURN***FROM***PROCESS***** [D 20:01:05.822509] end iteration [D 20:01:05.822533] done with segments or bytes [D 20:01:05.822557] of 411041792 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.822583] next loff: 411041792 next poff: 411041792 [D 20:01:05.822607] return value 2097152 [D 20:01:05.822631] finished [D 20:01:05.822655] retval = 2097152 [D 20:01:05.822680] exiting distribute returned less than expected [D 20:01:05.822704] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 411041792 fo 436207616 eof 0 [D 20:01:05.822731] ========================================================= [D 20:01:05.822754] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:05.822780] element 0: offset: 0x8aa39000, size: 2097152 [D 20:01:05.822807] openib_check_cq: completed send wr_id 15e72ef8. [D 20:01:05.822836] post_sr: da13:3346 bh 38 len 48 wr_id 15e72f10. [D 20:01:05.822864] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ee0210. [D 20:01:05.822890] ========================================================= [D 20:01:05.822915] PINT_process_request [D 20:01:05.822938] tiling 16 copies [D 20:01:05.822962] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.822991] lvl 0 el 0 blk 0 by 476053504 [D 20:01:05.823017] to 476053504 ta 469762048 fi 503316480 [D 20:01:05.823041] mto 6291456 mta 4194304 mfi 6291456 [D 20:01:05.823066] basic type or contiguous data [D 20:01:05.823091] Request disp [D 20:01:05.823115] contig_offset = 476053504 contig_size = 60817408 lvl_flag = 1 [D 20:01:05.823140] calling distribute [D 20:01:05.823163] PINT_distribute [D 20:01:05.823187] of 476053504 sz 27262976 ix 0 sm 64 by 0 bm 2097152 fsz 480247808 exfl 0 [D 20:01:05.823214] begin iteration loff: 476053504 [D 20:01:05.823239] segment exceeds byte limit [D 20:01:05.823263] client lstof 6291456 diff 0 sgof 6291456 [D 20:01:05.823288] **********CALL***PROCESS********* [D 20:01:05.823312] segment of 6291456 sz 2097152 [D 20:01:05.823337] PINT_process_request [D 20:01:05.823361] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.823387] lvl 0 el 0 blk 0 by 6291456 [D 20:01:05.823411] to 6291456 ta 6291456 fi 8388608 [D 20:01:05.823436] basic type or contiguous data [D 20:01:05.823463] Request disp [D 20:01:05.823487] contig_offset = 6291456 contig_size = 27262976 lvl_flag = 1 [D 20:01:05.823512] process a segment [D 20:01:05.823535] of 6291456 sz 2097152 [D 20:01:05.823560] add a segment 0 [D 20:01:05.823584] retval = 2097152 [D 20:01:05.823608] exiting distribute returned less than expected [D 20:01:05.823633] done sg 1 sm 64 by 2097152 bm 2097152 ta 6291456 to 8388608 fo 8388608 eof 0 [D 20:01:05.823658] *****RETURN***FROM***PROCESS***** [D 20:01:05.823683] end iteration [D 20:01:05.823707] done with segments or bytes [D 20:01:05.823731] of 478150656 sz 25165824 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.823757] next loff: 478150656 next poff: 478150656 [D 20:01:05.823781] return value 2097152 [D 20:01:05.823806] finished [D 20:01:05.823830] retval = 2097152 [D 20:01:05.823854] exiting distribute returned less than expected [D 20:01:05.823878] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 478150656 fo 503316480 eof 0 [D 20:01:05.823904] ========================================================= [D 20:01:05.823928] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 37 [D 20:01:05.823954] element 0: offset: 0x86a35000, size: 2097152 [D 20:01:05.823981] openib_check_cq: completed send wr_id 15e72f10. [D 20:01:05.824006] openib_check_cq: completed recv wr_id 15e73d10. [D 20:01:05.824032] openib_post_rr: da13:3346 bh 87 wr_id 15e73d10. [D 20:01:05.824059] openib_check_cq: completed recv wr_id 15e73d28. [D 20:01:05.824087] openib_post_rr: da13:3346 bh 88 wr_id 15e73d28. [D 20:01:05.824117] post_sr: da13:3346 bh 39 len 48 wr_id 15e72f28. [D 20:01:05.824146] PINT_client_state_machine_test id 76 [D 20:01:05.824181] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:05.824210] ========================================================= [D 20:01:05.824234] PINT_process_request [D 20:01:05.824258] tiling 7 copies [D 20:01:05.824283] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.824309] lvl 0 el 0 blk 0 by 178257920 [D 20:01:05.824335] to 178257920 ta 167772160 fi 201326592 [D 20:01:05.824359] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:05.824384] basic type or contiguous data [D 20:01:05.824409] Request disp [D 20:01:05.824433] contig_offset = 178257920 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.824458] calling distribute [D 20:01:05.824481] PINT_distribute [D 20:01:05.824505] of 178257920 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.824532] begin iteration loff: 178257920 [D 20:01:05.824558] segment exceeds byte limit [D 20:01:05.824582] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.824606] **********CALL***PROCESS********* [D 20:01:05.824630] segment of 10485760 sz 2097152 [D 20:01:05.824655] PINT_process_request [D 20:01:05.824679] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.824706] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.824735] to 10485760 ta 10485760 fi 12582912 [D 20:01:05.824761] basic type or contiguous data [D 20:01:05.824786] Request disp [D 20:01:05.824810] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.824834] process a segment [D 20:01:05.824858] of 10485760 sz 2097152 [D 20:01:05.824883] add a segment 0 [D 20:01:05.824907] retval = 2097152 [D 20:01:05.824931] exiting distribute returned less than expected [D 20:01:05.824955] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:05.824981] *****RETURN***FROM***PROCESS***** [D 20:01:05.825006] end iteration [D 20:01:05.825030] done with segments or bytes [D 20:01:05.825054] of 180355072 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.825079] next loff: 180355072 next poff: 180355072 [D 20:01:05.825104] return value 2097152 [D 20:01:05.825128] finished [D 20:01:05.825152] retval = 2097152 [D 20:01:05.825177] exiting distribute returned less than expected [D 20:01:05.825201] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 180355072 fo 201326592 eof 0 [D 20:01:05.825226] ========================================================= [D 20:01:05.825251] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:05.825276] element 0: offset: 0x98e47000, size: 2097152 [D 20:01:05.825303] openib_check_cq: completed send wr_id 15e72f28. [D 20:01:05.825331] post_sr: da13:3346 bh 40 len 48 wr_id 15e72f40. [D 20:01:05.825358] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:05.825384] ========================================================= [D 20:01:05.825408] PINT_process_request [D 20:01:05.825432] tiling 8 copies [D 20:01:05.825456] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.825483] lvl 0 el 0 blk 0 by 211812352 [D 20:01:05.825508] to 211812352 ta 201326592 fi 234881024 [D 20:01:05.825532] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:05.825557] basic type or contiguous data [D 20:01:05.825581] Request disp [D 20:01:05.825605] contig_offset = 211812352 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.825630] calling distribute [D 20:01:05.825654] PINT_distribute [D 20:01:05.825678] of 211812352 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.825704] begin iteration loff: 211812352 [D 20:01:05.825730] segment exceeds byte limit [D 20:01:05.825754] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.825778] **********CALL***PROCESS********* [D 20:01:05.825802] segment of 10485760 sz 2097152 [D 20:01:05.825827] PINT_process_request [D 20:01:05.825851] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.825877] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.825901] to 10485760 ta 10485760 fi 12582912 [D 20:01:05.825926] basic type or contiguous data [D 20:01:05.825951] Request disp [D 20:01:05.825975] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.826000] process a segment [D 20:01:05.826024] of 10485760 sz 2097152 [D 20:01:05.826047] add a segment 0 [D 20:01:05.826072] retval = 2097152 [D 20:01:05.826096] exiting distribute returned less than expected [D 20:01:05.826120] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:05.826146] *****RETURN***FROM***PROCESS***** [D 20:01:05.826170] end iteration [D 20:01:05.826194] done with segments or bytes [D 20:01:05.826222] of 213909504 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.826251] next loff: 213909504 next poff: 213909504 [D 20:01:05.826280] return value 2097152 [D 20:01:05.826308] finished [D 20:01:05.826333] retval = 2097152 [D 20:01:05.826358] exiting distribute returned less than expected [D 20:01:05.826382] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 213909504 fo 234881024 eof 0 [D 20:01:05.826409] ========================================================= [D 20:01:05.826433] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:05.826461] element 0: offset: 0x96e45000, size: 2097152 [D 20:01:05.826488] openib_check_cq: completed send wr_id 15e72f40. [D 20:01:05.826517] post_sr: da13:3346 bh 41 len 48 wr_id 15e72f58. [D 20:01:05.826589] openib_check_cq: completed send wr_id 15e72f58. [D 20:01:05.828234] openib_check_cq: completed recv wr_id 15e73d40. [D 20:01:05.828261] openib_post_rr: da13:3346 bh 89 wr_id 15e73d40. [D 20:01:05.828294] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:05.828320] ========================================================= [D 20:01:05.828345] PINT_process_request [D 20:01:05.828369] tiling 6 copies [D 20:01:05.828392] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.828418] lvl 0 el 0 blk 0 by 148897792 [D 20:01:05.828443] to 148897792 ta 134217728 fi 167772160 [D 20:01:05.828469] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:05.828494] basic type or contiguous data [D 20:01:05.828517] Request disp [D 20:01:05.828541] contig_offset = 148897792 contig_size = 52428800 lvl_flag = 1 [D 20:01:05.828566] calling distribute [D 20:01:05.828591] PINT_distribute [D 20:01:05.828615] of 148897792 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.828641] begin iteration loff: 148897792 [D 20:01:05.828666] segment exceeds byte limit [D 20:01:05.828690] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:05.828715] **********CALL***PROCESS********* [D 20:01:05.828739] segment of 14680064 sz 2097152 [D 20:01:05.828763] PINT_process_request [D 20:01:05.828787] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.828813] lvl 0 el 0 blk 0 by 14680064 [D 20:01:05.828838] to 14680064 ta 14680064 fi 16777216 [D 20:01:05.828863] basic type or contiguous data [D 20:01:05.828886] Request disp [D 20:01:05.828910] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:05.828935] process a segment [D 20:01:05.828959] of 14680064 sz 2097152 [D 20:01:05.828984] add a segment 0 [D 20:01:05.829008] retval = 2097152 [D 20:01:05.829031] exiting distribute returned less than expected [D 20:01:05.829055] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:05.829081] *****RETURN***FROM***PROCESS***** [D 20:01:05.829106] end iteration [D 20:01:05.829130] done with segments or bytes [D 20:01:05.829154] of 150994944 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.829179] next loff: 150994944 next poff: 150994944 [D 20:01:05.829204] return value 2097152 [D 20:01:05.829228] finished [D 20:01:05.829252] retval = 2097152 [D 20:01:05.829276] exiting distribute returned less than expected [D 20:01:05.829300] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 150994944 fo 167772160 eof 0 [D 20:01:05.829326] ========================================================= [D 20:01:05.829351] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:05.829376] element 0: offset: 0x9b249000, size: 2097152 [D 20:01:05.829405] post_sr: da13:3346 bh 42 len 48 wr_id 15e72f70. [D 20:01:05.829477] openib_check_cq: completed send wr_id 15e72f70. [D 20:01:05.830459] openib_check_cq: completed recv wr_id 15e73d58. [D 20:01:05.830487] openib_post_rr: da13:3346 bh 90 wr_id 15e73d58. [D 20:01:05.830520] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:05.830546] ========================================================= [D 20:01:05.830571] PINT_process_request [D 20:01:05.830595] tiling 9 copies [D 20:01:05.830619] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.830646] lvl 0 el 0 blk 0 by 245366784 [D 20:01:05.830671] to 245366784 ta 234881024 fi 268435456 [D 20:01:05.830696] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:05.830720] basic type or contiguous data [D 20:01:05.830744] Request disp [D 20:01:05.830768] contig_offset = 245366784 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.830794] calling distribute [D 20:01:05.830821] PINT_distribute [D 20:01:05.830844] of 245366784 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.830871] begin iteration loff: 245366784 [D 20:01:05.830896] segment exceeds byte limit [D 20:01:05.830921] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.830945] **********CALL***PROCESS********* [D 20:01:05.830969] segment of 10485760 sz 2097152 [D 20:01:05.830994] PINT_process_request [D 20:01:05.831018] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.831044] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.831068] to 10485760 ta 10485760 fi 12582912 [D 20:01:05.831093] basic type or contiguous data [D 20:01:05.831117] Request disp [D 20:01:05.831141] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.831166] process a segment [D 20:01:05.831190] of 10485760 sz 2097152 [D 20:01:05.831214] add a segment 0 [D 20:01:05.831238] retval = 2097152 [D 20:01:05.831262] exiting distribute returned less than expected [D 20:01:05.831286] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:05.831313] *****RETURN***FROM***PROCESS***** [D 20:01:05.831336] end iteration [D 20:01:05.831360] done with segments or bytes [D 20:01:05.831384] of 247463936 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.831411] next loff: 247463936 next poff: 247463936 [D 20:01:05.831436] return value 2097152 [D 20:01:05.831459] finished [D 20:01:05.831483] retval = 2097152 [D 20:01:05.831508] exiting distribute returned less than expected [D 20:01:05.831532] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 247463936 fo 268435456 eof 0 [D 20:01:05.831558] ========================================================= [D 20:01:05.831583] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:05.831607] element 0: offset: 0x94e43000, size: 2097152 [D 20:01:05.831637] post_sr: da13:3346 bh 43 len 48 wr_id 15e72f88. [D 20:01:05.831707] openib_check_cq: completed send wr_id 15e72f88. [D 20:01:05.834173] PINT_client_state_machine_test id 76 [D 20:01:05.834907] openib_check_cq: completed recv wr_id 15e73d70. [D 20:01:05.834935] openib_post_rr: da13:3346 bh 91 wr_id 15e73d70. [D 20:01:05.834968] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:05.834994] ========================================================= [D 20:01:05.835019] PINT_process_request [D 20:01:05.835043] tiling 10 copies [D 20:01:05.835067] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.835094] lvl 0 el 0 blk 0 by 278921216 [D 20:01:05.835119] to 278921216 ta 268435456 fi 301989888 [D 20:01:05.835144] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:05.835168] basic type or contiguous data [D 20:01:05.835192] Request disp [D 20:01:05.835217] contig_offset = 278921216 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.835242] calling distribute [D 20:01:05.835266] PINT_distribute [D 20:01:05.835289] of 278921216 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.835316] begin iteration loff: 278921216 [D 20:01:05.835341] segment exceeds byte limit [D 20:01:05.835365] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.835391] **********CALL***PROCESS********* [D 20:01:05.835414] segment of 10485760 sz 2097152 [D 20:01:05.835439] PINT_process_request [D 20:01:05.835463] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.835489] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.835514] to 10485760 ta 10485760 fi 12582912 [D 20:01:05.835538] basic type or contiguous data [D 20:01:05.835562] Request disp [D 20:01:05.835586] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.835611] process a segment [D 20:01:05.835635] of 10485760 sz 2097152 [D 20:01:05.835659] add a segment 0 [D 20:01:05.835683] retval = 2097152 [D 20:01:05.835707] exiting distribute returned less than expected [D 20:01:05.835731] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:05.835761] *****RETURN***FROM***PROCESS***** [D 20:01:05.835784] end iteration [D 20:01:05.835808] done with segments or bytes [D 20:01:05.835832] of 281018368 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.835858] next loff: 281018368 next poff: 281018368 [D 20:01:05.835883] return value 2097152 [D 20:01:05.835907] finished [D 20:01:05.835931] retval = 2097152 [D 20:01:05.835955] exiting distribute returned less than expected [D 20:01:05.835980] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 281018368 fo 301989888 eof 0 [D 20:01:05.836006] ========================================================= [D 20:01:05.836030] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:05.836055] element 0: offset: 0x92e41000, size: 2097152 [D 20:01:05.836085] post_sr: da13:3346 bh 44 len 48 wr_id 15e72fa0. [D 20:01:05.836155] openib_check_cq: completed send wr_id 15e72fa0. [D 20:01:05.839357] openib_check_cq: completed recv wr_id 15e73d88. [D 20:01:05.839385] openib_post_rr: da13:3346 bh 92 wr_id 15e73d88. [D 20:01:05.839418] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:05.839445] ========================================================= [D 20:01:05.839469] PINT_process_request [D 20:01:05.839493] tiling 12 copies [D 20:01:05.839518] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.839544] lvl 0 el 0 blk 0 by 343932928 [D 20:01:05.839570] to 343932928 ta 335544320 fi 369098752 [D 20:01:05.839594] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.839619] basic type or contiguous data [D 20:01:05.839643] Request disp [D 20:01:05.839667] contig_offset = 343932928 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.839693] calling distribute [D 20:01:05.839716] PINT_distribute [D 20:01:05.839740] of 343932928 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.839767] begin iteration loff: 343932928 [D 20:01:05.839792] segment exceeds byte limit [D 20:01:05.839817] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.839841] **********CALL***PROCESS********* [D 20:01:05.839865] segment of 8388608 sz 2097152 [D 20:01:05.839890] PINT_process_request [D 20:01:05.839914] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.839940] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.839964] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.839989] basic type or contiguous data [D 20:01:05.840013] Request disp [D 20:01:05.840037] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.840063] process a segment [D 20:01:05.840086] of 8388608 sz 2097152 [D 20:01:05.840111] add a segment 0 [D 20:01:05.840135] retval = 2097152 [D 20:01:05.840159] exiting distribute returned less than expected [D 20:01:05.840184] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.840212] *****RETURN***FROM***PROCESS***** [D 20:01:05.840238] end iteration [D 20:01:05.840262] done with segments or bytes [D 20:01:05.840286] of 346030080 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.840312] next loff: 346030080 next poff: 346030080 [D 20:01:05.840336] return value 2097152 [D 20:01:05.840360] finished [D 20:01:05.840384] retval = 2097152 [D 20:01:05.840409] exiting distribute returned less than expected [D 20:01:05.840433] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 346030080 fo 369098752 eof 0 [D 20:01:05.840459] ========================================================= [D 20:01:05.840483] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:05.840509] element 0: offset: 0x8ec3d000, size: 2097152 [D 20:01:05.840538] post_sr: da13:3346 bh 45 len 48 wr_id 15e72fb8. [D 20:01:05.840608] openib_check_cq: completed send wr_id 15e72fb8. [D 20:01:05.841585] openib_check_cq: completed recv wr_id 15e73da0. [D 20:01:05.841611] openib_post_rr: da13:3346 bh 93 wr_id 15e73da0. [D 20:01:05.841648] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:05.841673] ========================================================= [D 20:01:05.841698] PINT_process_request [D 20:01:05.841722] tiling 11 copies [D 20:01:05.841746] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.841773] lvl 0 el 0 blk 0 by 310378496 [D 20:01:05.841797] to 310378496 ta 301989888 fi 335544320 [D 20:01:05.841822] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.841847] basic type or contiguous data [D 20:01:05.841871] Request disp [D 20:01:05.841895] contig_offset = 310378496 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.841920] calling distribute [D 20:01:05.841944] PINT_distribute [D 20:01:05.841969] of 310378496 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.841995] begin iteration loff: 310378496 [D 20:01:05.842020] segment exceeds byte limit [D 20:01:05.842044] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.842069] **********CALL***PROCESS********* [D 20:01:05.842093] segment of 8388608 sz 2097152 [D 20:01:05.842118] PINT_process_request [D 20:01:05.842142] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.842168] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.842193] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.842222] basic type or contiguous data [D 20:01:05.842249] Request disp [D 20:01:05.842277] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.842305] process a segment [D 20:01:05.842332] of 8388608 sz 2097152 [D 20:01:05.842357] add a segment 0 [D 20:01:05.842381] retval = 2097152 [D 20:01:05.842405] exiting distribute returned less than expected [D 20:01:05.842429] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.842455] *****RETURN***FROM***PROCESS***** [D 20:01:05.842479] end iteration [D 20:01:05.842503] done with segments or bytes [D 20:01:05.842527] of 312475648 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.842553] next loff: 312475648 next poff: 312475648 [D 20:01:05.842577] return value 2097152 [D 20:01:05.842601] finished [D 20:01:05.842625] retval = 2097152 [D 20:01:05.842649] exiting distribute returned less than expected [D 20:01:05.842673] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 312475648 fo 335544320 eof 0 [D 20:01:05.842699] ========================================================= [D 20:01:05.842723] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:05.842749] element 0: offset: 0x90c3f000, size: 2097152 [D 20:01:05.842779] post_sr: da13:3346 bh 46 len 48 wr_id 15e72fd0. [D 20:01:05.842848] openib_check_cq: completed send wr_id 15e72fd0. [D 20:01:05.843807] openib_check_cq: completed recv wr_id 15e73db8. [D 20:01:05.843834] openib_post_rr: da13:3346 bh 94 wr_id 15e73db8. [D 20:01:05.843867] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:05.843894] ========================================================= [D 20:01:05.843917] PINT_process_request [D 20:01:05.843941] tiling 5 copies [D 20:01:05.843966] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.843992] lvl 0 el 0 blk 0 by 125829120 [D 20:01:05.844017] to 125829120 ta 100663296 fi 134217728 [D 20:01:05.844041] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:05.844066] basic type or contiguous data [D 20:01:05.844090] Request disp [D 20:01:05.844114] contig_offset = 125829120 contig_size = 41943040 lvl_flag = 1 [D 20:01:05.844140] calling distribute [D 20:01:05.844163] PINT_distribute [D 20:01:05.844187] of 125829120 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:05.844216] begin iteration loff: 125829120 [D 20:01:05.844242] segment exceeds byte limit [D 20:01:05.844265] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:05.844290] **********CALL***PROCESS********* [D 20:01:05.844315] segment of 25165824 sz 2097152 [D 20:01:05.844339] PINT_process_request [D 20:01:05.844366] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.844392] lvl 0 el 0 blk 0 by 25165824 [D 20:01:05.844417] to 25165824 ta 25165824 fi 27262976 [D 20:01:05.844442] basic type or contiguous data [D 20:01:05.844466] Request disp [D 20:01:05.844490] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:05.844514] process a segment [D 20:01:05.844538] of 25165824 sz 2097152 [D 20:01:05.844562] add a segment 0 [D 20:01:05.844586] retval = 2097152 [D 20:01:05.844611] exiting distribute returned less than expected [D 20:01:05.844635] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:05.844660] *****RETURN***FROM***PROCESS***** [D 20:01:05.844685] end iteration [D 20:01:05.844709] done with segments or bytes [D 20:01:05.844733] of 127926272 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.844758] next loff: 127926272 next poff: 127926272 [D 20:01:05.844783] return value 2097152 [D 20:01:05.844807] finished [D 20:01:05.844831] retval = 2097152 [D 20:01:05.844855] exiting distribute returned less than expected [D 20:01:05.844879] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 127926272 fo 134217728 eof 0 [D 20:01:05.844905] ========================================================= [D 20:01:05.844929] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:05.844955] element 0: offset: 0x9dc4b000, size: 2097152 [D 20:01:05.844985] post_sr: da13:3346 bh 47 len 48 wr_id 15e72fe8. [D 20:01:05.845013] PINT_client_state_machine_test id 76 [D 20:01:05.845056] openib_check_cq: completed send wr_id 15e72fe8. [D 20:01:05.846032] openib_check_cq: completed recv wr_id 15e73dd0. [D 20:01:05.846058] openib_post_rr: da13:3346 bh 95 wr_id 15e73dd0. [D 20:01:05.846085] openib_check_cq: completed recv wr_id 15e73de8. [D 20:01:05.846112] openib_post_rr: da13:3346 bh 96 wr_id 15e73de8. [D 20:01:05.846145] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:05.846171] ========================================================= [D 20:01:05.846196] PINT_process_request [D 20:01:05.846223] tiling 13 copies [D 20:01:05.846252] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.846282] lvl 0 el 0 blk 0 by 377487360 [D 20:01:05.846310] to 377487360 ta 369098752 fi 402653184 [D 20:01:05.846337] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.846363] basic type or contiguous data [D 20:01:05.846386] Request disp [D 20:01:05.846410] contig_offset = 377487360 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.846435] calling distribute [D 20:01:05.846459] PINT_distribute [D 20:01:05.846483] of 377487360 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.846509] begin iteration loff: 377487360 [D 20:01:05.846534] segment exceeds byte limit [D 20:01:05.846558] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.846583] **********CALL***PROCESS********* [D 20:01:05.846607] segment of 8388608 sz 2097152 [D 20:01:05.846632] PINT_process_request [D 20:01:05.846655] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.846682] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.846707] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.846732] basic type or contiguous data [D 20:01:05.846755] Request disp [D 20:01:05.846779] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.846804] process a segment [D 20:01:05.846828] of 8388608 sz 2097152 [D 20:01:05.846852] add a segment 0 [D 20:01:05.846877] retval = 2097152 [D 20:01:05.846900] exiting distribute returned less than expected [D 20:01:05.846924] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.846950] *****RETURN***FROM***PROCESS***** [D 20:01:05.846975] end iteration [D 20:01:05.846999] done with segments or bytes [D 20:01:05.847023] of 379584512 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.847048] next loff: 379584512 next poff: 379584512 [D 20:01:05.847076] return value 2097152 [D 20:01:05.847101] finished [D 20:01:05.847125] retval = 2097152 [D 20:01:05.847148] exiting distribute returned less than expected [D 20:01:05.847173] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 379584512 fo 402653184 eof 0 [D 20:01:05.847199] ========================================================= [D 20:01:05.847224] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:05.847249] element 0: offset: 0x8cc3b000, size: 2097152 [D 20:01:05.847279] post_sr: da13:3346 bh 48 len 48 wr_id 15e73000. [D 20:01:05.847306] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:05.847332] ========================================================= [D 20:01:05.847356] PINT_process_request [D 20:01:05.847380] tiling 14 copies [D 20:01:05.847404] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.847430] lvl 0 el 0 blk 0 by 411041792 [D 20:01:05.847455] to 411041792 ta 402653184 fi 436207616 [D 20:01:05.847480] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.847505] basic type or contiguous data [D 20:01:05.847529] Request disp [D 20:01:05.847553] contig_offset = 411041792 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.847578] calling distribute [D 20:01:05.847601] PINT_distribute [D 20:01:05.847625] of 411041792 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.847652] begin iteration loff: 411041792 [D 20:01:05.847677] segment exceeds byte limit [D 20:01:05.847701] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.847726] **********CALL***PROCESS********* [D 20:01:05.847750] segment of 8388608 sz 2097152 [D 20:01:05.847774] PINT_process_request [D 20:01:05.847799] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.847825] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.847849] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.847874] basic type or contiguous data [D 20:01:05.847898] Request disp [D 20:01:05.847921] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.847947] process a segment [D 20:01:05.847970] of 8388608 sz 2097152 [D 20:01:05.847994] add a segment 0 [D 20:01:05.848018] retval = 2097152 [D 20:01:05.848042] exiting distribute returned less than expected [D 20:01:05.848066] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.848093] *****RETURN***FROM***PROCESS***** [D 20:01:05.848116] end iteration [D 20:01:05.848140] done with segments or bytes [D 20:01:05.848164] of 413138944 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.848190] next loff: 413138944 next poff: 413138944 [D 20:01:05.848217] return value 2097152 [D 20:01:05.848241] finished [D 20:01:05.848265] retval = 2097152 [D 20:01:05.848289] exiting distribute returned less than expected [D 20:01:05.848314] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 413138944 fo 436207616 eof 0 [D 20:01:05.848340] ========================================================= [D 20:01:05.848364] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:05.848389] element 0: offset: 0x8ac39000, size: 2097152 [D 20:01:05.848416] openib_check_cq: completed send wr_id 15e73000. [D 20:01:05.848442] openib_check_cq: completed recv wr_id 15e73e00. [D 20:01:05.848468] openib_post_rr: da13:3346 bh 97 wr_id 15e73e00. [D 20:01:05.848494] openib_check_cq: completed recv wr_id 15e73e18. [D 20:01:05.848521] openib_post_rr: da13:3346 bh 98 wr_id 15e73e18. [D 20:01:05.848551] post_sr: da13:3346 bh 49 len 48 wr_id 15e73018. [D 20:01:05.848586] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:05.848612] ========================================================= [D 20:01:05.848636] PINT_process_request [D 20:01:05.848660] tiling 7 copies [D 20:01:05.848685] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.848711] lvl 0 el 0 blk 0 by 180355072 [D 20:01:05.848738] to 180355072 ta 167772160 fi 201326592 [D 20:01:05.848764] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:05.848789] basic type or contiguous data [D 20:01:05.848813] Request disp [D 20:01:05.848836] contig_offset = 180355072 contig_size = 54525952 lvl_flag = 1 [D 20:01:05.848861] calling distribute [D 20:01:05.848886] PINT_distribute [D 20:01:05.848909] of 180355072 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.848936] begin iteration loff: 180355072 [D 20:01:05.848960] segment exceeds byte limit [D 20:01:05.848985] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:05.849010] **********CALL***PROCESS********* [D 20:01:05.849033] segment of 12582912 sz 2097152 [D 20:01:05.849058] PINT_process_request [D 20:01:05.849081] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.849107] lvl 0 el 0 blk 0 by 12582912 [D 20:01:05.849133] to 12582912 ta 12582912 fi 14680064 [D 20:01:05.849157] basic type or contiguous data [D 20:01:05.849182] Request disp [D 20:01:05.849205] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:05.849230] process a segment [D 20:01:05.849254] of 12582912 sz 2097152 [D 20:01:05.849278] add a segment 0 [D 20:01:05.849302] retval = 2097152 [D 20:01:05.849326] exiting distribute returned less than expected [D 20:01:05.849350] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:05.849376] *****RETURN***FROM***PROCESS***** [D 20:01:05.849401] end iteration [D 20:01:05.849425] done with segments or bytes [D 20:01:05.849449] of 182452224 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.849474] next loff: 182452224 next poff: 182452224 [D 20:01:05.849498] return value 2097152 [D 20:01:05.849523] finished [D 20:01:05.849547] retval = 2097152 [D 20:01:05.849571] exiting distribute returned less than expected [D 20:01:05.849595] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 182452224 fo 201326592 eof 0 [D 20:01:05.849621] ========================================================= [D 20:01:05.849645] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:05.849671] element 0: offset: 0x99047000, size: 2097152 [D 20:01:05.849697] openib_check_cq: completed send wr_id 15e73018. [D 20:01:05.849725] post_sr: da13:3346 bh 50 len 48 wr_id 15e73030. [D 20:01:05.849753] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ee0210. [D 20:01:05.849778] ========================================================= [D 20:01:05.849802] PINT_process_request [D 20:01:05.849826] tiling 16 copies [D 20:01:05.849851] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.849877] lvl 0 el 0 blk 0 by 478150656 [D 20:01:05.849902] to 478150656 ta 469762048 fi 503316480 [D 20:01:05.849927] mto 8388608 mta 6291456 mfi 8388608 [D 20:01:05.849952] basic type or contiguous data [D 20:01:05.849976] Request disp [D 20:01:05.850000] contig_offset = 478150656 contig_size = 58720256 lvl_flag = 1 [D 20:01:05.850026] calling distribute [D 20:01:05.850049] PINT_distribute [D 20:01:05.850073] of 478150656 sz 25165824 ix 0 sm 64 by 0 bm 2097152 fsz 480247808 exfl 0 [D 20:01:05.850100] begin iteration loff: 478150656 [D 20:01:05.850125] segment exceeds byte limit [D 20:01:05.850149] client lstof 8388608 diff 0 sgof 8388608 [D 20:01:05.850173] **********CALL***PROCESS********* [D 20:01:05.850197] segment of 8388608 sz 2097152 [D 20:01:05.850226] PINT_process_request [D 20:01:05.850254] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.850284] lvl 0 el 0 blk 0 by 8388608 [D 20:01:05.850314] to 8388608 ta 8388608 fi 10485760 [D 20:01:05.850338] basic type or contiguous data [D 20:01:05.850363] Request disp [D 20:01:05.850387] contig_offset = 8388608 contig_size = 25165824 lvl_flag = 1 [D 20:01:05.850412] process a segment [D 20:01:05.850436] of 8388608 sz 2097152 [D 20:01:05.850460] add a segment 0 [D 20:01:05.850483] retval = 2097152 [D 20:01:05.850511] exiting distribute returned less than expected [D 20:01:05.850535] done sg 1 sm 64 by 2097152 bm 2097152 ta 8388608 to 10485760 fo 10485760 eof 0 [D 20:01:05.850561] *****RETURN***FROM***PROCESS***** [D 20:01:05.850585] end iteration [D 20:01:05.850609] done with segments or bytes [D 20:01:05.850633] of 480247808 sz 23068672 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.850659] next loff: 480247808 next poff: 480247808 [D 20:01:05.850684] [return value] 25165824 (EOF) [D 20:01:05.850708] return value 2097152 (EOF) [D 20:01:05.850733] finished [D 20:01:05.850756] retval = 2097152 [D 20:01:05.850781] exiting distribute returned less than expected [D 20:01:05.850805] done sg 1 sm 64 by 2097152 bm 2097152 ta 469762048 to 480247808 fo 503316480 eof 1 [D 20:01:05.850831] ========================================================= [D 20:01:05.850855] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 37 [D 20:01:05.850881] element 0: offset: 0x86c35000, size: 2097152 [D 20:01:05.850907] openib_check_cq: completed send wr_id 15e73030. [D 20:01:05.850933] openib_check_cq: completed recv wr_id 15e73e30. [D 20:01:05.850958] openib_post_rr: da13:3346 bh 99 wr_id 15e73e30. [D 20:01:05.850989] post_sr: da13:3346 bh 51 len 48 wr_id 15e73048. [D 20:01:05.851023] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:05.851049] ========================================================= [D 20:01:05.851074] PINT_process_request [D 20:01:05.851098] tiling 8 copies [D 20:01:05.851122] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.851149] lvl 0 el 0 blk 0 by 213909504 [D 20:01:05.851174] to 213909504 ta 201326592 fi 234881024 [D 20:01:05.851199] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:05.851223] basic type or contiguous data [D 20:01:05.851247] Request disp [D 20:01:05.851271] contig_offset = 213909504 contig_size = 54525952 lvl_flag = 1 [D 20:01:05.851297] calling distribute [D 20:01:05.851321] PINT_distribute [D 20:01:05.851345] of 213909504 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.851370] begin iteration loff: 213909504 [D 20:01:05.851396] segment exceeds byte limit [D 20:01:05.851420] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:05.851445] **********CALL***PROCESS********* [D 20:01:05.851469] segment of 12582912 sz 2097152 [D 20:01:05.851493] PINT_process_request [D 20:01:05.851517] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.851543] lvl 0 el 0 blk 0 by 12582912 [D 20:01:05.851568] to 12582912 ta 12582912 fi 14680064 [D 20:01:05.851593] basic type or contiguous data [D 20:01:05.851616] Request disp [D 20:01:05.851640] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:05.851666] process a segment [D 20:01:05.851690] of 12582912 sz 2097152 [D 20:01:05.851714] add a segment 0 [D 20:01:05.851738] retval = 2097152 [D 20:01:05.851762] exiting distribute returned less than expected [D 20:01:05.851786] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:05.851813] *****RETURN***FROM***PROCESS***** [D 20:01:05.851837] end iteration [D 20:01:05.851861] done with segments or bytes [D 20:01:05.851884] of 216006656 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.851910] next loff: 216006656 next poff: 216006656 [D 20:01:05.851935] return value 2097152 [D 20:01:05.851959] finished [D 20:01:05.851983] retval = 2097152 [D 20:01:05.852006] exiting distribute returned less than expected [D 20:01:05.852031] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 216006656 fo 234881024 eof 0 [D 20:01:05.852057] ========================================================= [D 20:01:05.852082] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:05.852108] element 0: offset: 0x97045000, size: 2097152 [D 20:01:05.852133] openib_check_cq: completed send wr_id 15e73048. [D 20:01:05.852162] post_sr: da13:3346 bh 52 len 48 wr_id 15e73060. [D 20:01:05.852234] openib_check_cq: completed send wr_id 15e73060. [D 20:01:05.852717] openib_check_cq: completed recv wr_id 15e734e8. [D 20:01:05.852744] openib_post_rr: da13:3346 bh 0 wr_id 15e734e8. [D 20:01:05.852775] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:05.852802] ========================================================= [D 20:01:05.852826] PINT_process_request [D 20:01:05.852849] tiling 6 copies [D 20:01:05.852874] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.852901] lvl 0 el 0 blk 0 by 150994944 [D 20:01:05.852926] to 150994944 ta 134217728 fi 167772160 [D 20:01:05.852951] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:05.852975] basic type or contiguous data [D 20:01:05.852999] Request disp [D 20:01:05.853023] contig_offset = 150994944 contig_size = 50331648 lvl_flag = 1 [D 20:01:05.853049] calling distribute [D 20:01:05.853073] PINT_distribute [D 20:01:05.853096] of 150994944 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.853123] begin iteration loff: 150994944 [D 20:01:05.853148] segment exceeds byte limit [D 20:01:05.853172] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:05.853197] **********CALL***PROCESS********* [D 20:01:05.853220] segment of 16777216 sz 2097152 [D 20:01:05.853245] PINT_process_request [D 20:01:05.853269] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.853296] lvl 0 el 0 blk 0 by 16777216 [D 20:01:05.853320] to 16777216 ta 16777216 fi 18874368 [D 20:01:05.853345] basic type or contiguous data [D 20:01:05.853369] Request disp [D 20:01:05.853393] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:05.853418] process a segment [D 20:01:05.853444] of 16777216 sz 2097152 [D 20:01:05.853468] add a segment 0 [D 20:01:05.853492] retval = 2097152 [D 20:01:05.853516] exiting distribute returned less than expected [D 20:01:05.853541] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:05.853567] *****RETURN***FROM***PROCESS***** [D 20:01:05.853591] end iteration [D 20:01:05.853615] done with segments or bytes [D 20:01:05.853639] of 153092096 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.853665] next loff: 153092096 next poff: 153092096 [D 20:01:05.853690] return value 2097152 [D 20:01:05.853714] finished [D 20:01:05.853738] retval = 2097152 [D 20:01:05.853762] exiting distribute returned less than expected [D 20:01:05.853786] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 153092096 fo 167772160 eof 0 [D 20:01:05.853813] ========================================================= [D 20:01:05.853837] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:05.853862] element 0: offset: 0x9b449000, size: 2097152 [D 20:01:05.853892] post_sr: da13:3346 bh 53 len 48 wr_id 15e73078. [D 20:01:05.853965] openib_check_cq: completed send wr_id 15e73078. [D 20:01:05.854940] openib_check_cq: completed recv wr_id 15e73500. [D 20:01:05.854968] openib_post_rr: da13:3346 bh 1 wr_id 15e73500. [D 20:01:05.854994] openib_check_cq: completed recv wr_id 15e73518. [D 20:01:05.855022] openib_post_rr: da13:3346 bh 2 wr_id 15e73518. [D 20:01:05.855048] openib_check_cq: completed recv wr_id 15e73530. [D 20:01:05.855075] openib_post_rr: da13:3346 bh 3 wr_id 15e73530. [D 20:01:05.855102] openib_check_cq: completed recv wr_id 15e73548. [D 20:01:05.855130] openib_post_rr: da13:3346 bh 4 wr_id 15e73548. [D 20:01:05.855156] openib_check_cq: completed recv wr_id 15e73560. [D 20:01:05.855184] openib_post_rr: da13:3346 bh 5 wr_id 15e73560. [D 20:01:05.855211] openib_check_cq: completed recv wr_id 15e73578. [D 20:01:05.855238] openib_post_rr: da13:3346 bh 6 wr_id 15e73578. [D 20:01:05.855264] openib_check_cq: completed recv wr_id 15e73590. [D 20:01:05.855293] openib_post_rr: da13:3346 bh 7 wr_id 15e73590. [D 20:01:05.855319] openib_check_cq: completed recv wr_id 15e735a8. [D 20:01:05.855347] openib_post_rr: da13:3346 bh 8 wr_id 15e735a8. [D 20:01:05.855376] openib_check_cq: completed recv wr_id 15e735c0. [D 20:01:05.855403] openib_post_rr: da13:3346 bh 9 wr_id 15e735c0. [D 20:01:05.855437] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:05.855463] ========================================================= [D 20:01:05.855486] PINT_process_request [D 20:01:05.855511] tiling 9 copies [D 20:01:05.855535] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.855562] lvl 0 el 0 blk 0 by 247463936 [D 20:01:05.855587] to 247463936 ta 234881024 fi 268435456 [D 20:01:05.855611] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:05.855636] basic type or contiguous data [D 20:01:05.855660] Request disp [D 20:01:05.855684] contig_offset = 247463936 contig_size = 54525952 lvl_flag = 1 [D 20:01:05.855709] calling distribute [D 20:01:05.855732] PINT_distribute [D 20:01:05.855756] of 247463936 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.855783] begin iteration loff: 247463936 [D 20:01:05.855808] segment exceeds byte limit [D 20:01:05.855832] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:05.855856] **********CALL***PROCESS********* [D 20:01:05.855880] segment of 12582912 sz 2097152 [D 20:01:05.855905] PINT_process_request [D 20:01:05.855929] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.855955] lvl 0 el 0 blk 0 by 12582912 [D 20:01:05.855980] to 12582912 ta 12582912 fi 14680064 [D 20:01:05.856005] basic type or contiguous data [D 20:01:05.856029] Request disp [D 20:01:05.856052] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:05.856078] process a segment [D 20:01:05.856101] of 12582912 sz 2097152 [D 20:01:05.856125] add a segment 0 [D 20:01:05.856150] retval = 2097152 [D 20:01:05.856174] exiting distribute returned less than expected [D 20:01:05.856198] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:05.856226] *****RETURN***FROM***PROCESS***** [D 20:01:05.856251] end iteration [D 20:01:05.856275] done with segments or bytes [D 20:01:05.856299] of 249561088 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.856325] next loff: 249561088 next poff: 249561088 [D 20:01:05.856349] return value 2097152 [D 20:01:05.856373] finished [D 20:01:05.856397] retval = 2097152 [D 20:01:05.856422] exiting distribute returned less than expected [D 20:01:05.856446] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 249561088 fo 268435456 eof 0 [D 20:01:05.856472] ========================================================= [D 20:01:05.856496] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:05.856521] element 0: offset: 0x95043000, size: 2097152 [D 20:01:05.856551] post_sr: da13:3346 bh 54 len 48 wr_id 15e73090. [D 20:01:05.856579] PINT_client_state_machine_test id 76 [D 20:01:05.856608] openib_check_cq: completed send wr_id 15e73090. [D 20:01:05.866613] PINT_client_state_machine_test id 76 [D 20:01:05.876644] PINT_client_state_machine_test id 76 [D 20:01:05.886677] PINT_client_state_machine_test id 76 [D 20:01:05.896709] PINT_client_state_machine_test id 76 [D 20:01:05.906743] PINT_client_state_machine_test id 76 [D 20:01:05.916769] PINT_client_state_machine_test id 76 [D 20:01:05.921867] openib_check_cq: completed recv wr_id 15e735d8. [D 20:01:05.921895] openib_post_rr: da13:3346 bh 10 wr_id 15e735d8. [D 20:01:05.921927] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:05.921953] ========================================================= [D 20:01:05.921977] PINT_process_request [D 20:01:05.922001] tiling 10 copies [D 20:01:05.922025] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.922052] lvl 0 el 0 blk 0 by 281018368 [D 20:01:05.922077] to 281018368 ta 268435456 fi 301989888 [D 20:01:05.922101] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:05.922126] basic type or contiguous data [D 20:01:05.922153] Request disp [D 20:01:05.922178] contig_offset = 281018368 contig_size = 54525952 lvl_flag = 1 [D 20:01:05.922203] calling distribute [D 20:01:05.922232] PINT_distribute [D 20:01:05.922259] of 281018368 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.922289] begin iteration loff: 281018368 [D 20:01:05.922318] segment exceeds byte limit [D 20:01:05.922343] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:05.922368] **********CALL***PROCESS********* [D 20:01:05.922391] segment of 12582912 sz 2097152 [D 20:01:05.922416] PINT_process_request [D 20:01:05.922441] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.922467] lvl 0 el 0 blk 0 by 12582912 [D 20:01:05.922492] to 12582912 ta 12582912 fi 14680064 [D 20:01:05.922516] basic type or contiguous data [D 20:01:05.922540] Request disp [D 20:01:05.922564] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:05.922590] process a segment [D 20:01:05.922613] of 12582912 sz 2097152 [D 20:01:05.922637] add a segment 0 [D 20:01:05.922661] retval = 2097152 [D 20:01:05.922686] exiting distribute returned less than expected [D 20:01:05.922710] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:05.922736] *****RETURN***FROM***PROCESS***** [D 20:01:05.922760] end iteration [D 20:01:05.922784] done with segments or bytes [D 20:01:05.922808] of 283115520 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.922834] next loff: 283115520 next poff: 283115520 [D 20:01:05.922859] return value 2097152 [D 20:01:05.922883] finished [D 20:01:05.922907] retval = 2097152 [D 20:01:05.922931] exiting distribute returned less than expected [D 20:01:05.922955] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 283115520 fo 301989888 eof 0 [D 20:01:05.922982] ========================================================= [D 20:01:05.923007] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:05.923031] element 0: offset: 0x93041000, size: 2097152 [D 20:01:05.923061] post_sr: da13:3346 bh 55 len 48 wr_id 15e730a8. [D 20:01:05.923130] openib_check_cq: completed send wr_id 15e730a8. [D 20:01:05.926315] openib_check_cq: completed recv wr_id 15e735f0. [D 20:01:05.926342] openib_post_rr: da13:3346 bh 11 wr_id 15e735f0. [D 20:01:05.926376] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:05.926402] ========================================================= [D 20:01:05.926426] PINT_process_request [D 20:01:05.926450] tiling 12 copies [D 20:01:05.926474] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.926501] lvl 0 el 0 blk 0 by 346030080 [D 20:01:05.926526] to 346030080 ta 335544320 fi 369098752 [D 20:01:05.926550] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:05.926576] basic type or contiguous data [D 20:01:05.926600] Request disp [D 20:01:05.926624] contig_offset = 346030080 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.926650] calling distribute [D 20:01:05.926673] PINT_distribute [D 20:01:05.926697] of 346030080 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.926724] begin iteration loff: 346030080 [D 20:01:05.926749] segment exceeds byte limit [D 20:01:05.926773] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.926797] **********CALL***PROCESS********* [D 20:01:05.926821] segment of 10485760 sz 2097152 [D 20:01:05.926846] PINT_process_request [D 20:01:05.926870] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.926897] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.926921] to 10485760 ta 10485760 fi 12582912 [D 20:01:05.926946] basic type or contiguous data [D 20:01:05.926970] Request disp [D 20:01:05.926994] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.927019] process a segment [D 20:01:05.927042] of 10485760 sz 2097152 [D 20:01:05.927066] add a segment 0 [D 20:01:05.927090] retval = 2097152 [D 20:01:05.927117] exiting distribute returned less than expected [D 20:01:05.927142] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:05.927167] *****RETURN***FROM***PROCESS***** [D 20:01:05.927192] end iteration [D 20:01:05.927216] done with segments or bytes [D 20:01:05.927240] of 348127232 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.927266] next loff: 348127232 next poff: 348127232 [D 20:01:05.927290] return value 2097152 [D 20:01:05.927315] finished [D 20:01:05.927339] retval = 2097152 [D 20:01:05.927363] exiting distribute returned less than expected [D 20:01:05.927388] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 348127232 fo 369098752 eof 0 [D 20:01:05.927413] ========================================================= [D 20:01:05.927438] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:05.927463] element 0: offset: 0x8ee3d000, size: 2097152 [D 20:01:05.927493] post_sr: da13:3346 bh 56 len 48 wr_id 15e730c0. [D 20:01:05.927521] PINT_client_state_machine_test id 76 [D 20:01:05.927563] openib_check_cq: completed send wr_id 15e730c0. [D 20:01:05.930756] openib_check_cq: completed recv wr_id 15e73608. [D 20:01:05.930784] openib_post_rr: da13:3346 bh 12 wr_id 15e73608. [D 20:01:05.930816] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:05.930842] ========================================================= [D 20:01:05.930867] PINT_process_request [D 20:01:05.930891] tiling 11 copies [D 20:01:05.930915] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.930942] lvl 0 el 0 blk 0 by 312475648 [D 20:01:05.930967] to 312475648 ta 301989888 fi 335544320 [D 20:01:05.930992] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:05.931016] basic type or contiguous data [D 20:01:05.931041] Request disp [D 20:01:05.931065] contig_offset = 312475648 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.931091] calling distribute [D 20:01:05.931115] PINT_distribute [D 20:01:05.931138] of 312475648 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.931165] begin iteration loff: 312475648 [D 20:01:05.931190] segment exceeds byte limit [D 20:01:05.931215] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.931239] **********CALL***PROCESS********* [D 20:01:05.931262] segment of 10485760 sz 2097152 [D 20:01:05.931287] PINT_process_request [D 20:01:05.931311] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.931338] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.931363] to 10485760 ta 10485760 fi 12582912 [D 20:01:05.931387] basic type or contiguous data [D 20:01:05.931411] Request disp [D 20:01:05.931435] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.931461] process a segment [D 20:01:05.931485] of 10485760 sz 2097152 [D 20:01:05.931509] add a segment 0 [D 20:01:05.931532] retval = 2097152 [D 20:01:05.931557] exiting distribute returned less than expected [D 20:01:05.931581] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:05.931608] *****RETURN***FROM***PROCESS***** [D 20:01:05.931632] end iteration [D 20:01:05.931656] done with segments or bytes [D 20:01:05.931680] of 314572800 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.931706] next loff: 314572800 next poff: 314572800 [D 20:01:05.931731] return value 2097152 [D 20:01:05.931756] finished [D 20:01:05.931779] retval = 2097152 [D 20:01:05.931803] exiting distribute returned less than expected [D 20:01:05.931827] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 314572800 fo 335544320 eof 0 [D 20:01:05.931854] ========================================================= [D 20:01:05.931878] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:05.931903] element 0: offset: 0x90e3f000, size: 2097152 [D 20:01:05.931933] post_sr: da13:3346 bh 57 len 48 wr_id 15e730d8. [D 20:01:05.932002] openib_check_cq: completed send wr_id 15e730d8. [D 20:01:05.932983] openib_check_cq: completed recv wr_id 15e73620. [D 20:01:05.933012] openib_post_rr: da13:3346 bh 13 wr_id 15e73620. [D 20:01:05.933044] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:05.933070] ========================================================= [D 20:01:05.933094] PINT_process_request [D 20:01:05.933118] tiling 5 copies [D 20:01:05.933142] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.933169] lvl 0 el 0 blk 0 by 127926272 [D 20:01:05.933194] to 127926272 ta 100663296 fi 134217728 [D 20:01:05.933218] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:05.933243] basic type or contiguous data [D 20:01:05.933267] Request disp [D 20:01:05.933291] contig_offset = 127926272 contig_size = 39845888 lvl_flag = 1 [D 20:01:05.933316] calling distribute [D 20:01:05.933340] PINT_distribute [D 20:01:05.933363] of 127926272 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:05.933390] begin iteration loff: 127926272 [D 20:01:05.933415] segment exceeds byte limit [D 20:01:05.933442] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:05.933466] **********CALL***PROCESS********* [D 20:01:05.933490] segment of 27262976 sz 2097152 [D 20:01:05.933515] PINT_process_request [D 20:01:05.933539] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.933566] lvl 0 el 0 blk 0 by 27262976 [D 20:01:05.933590] to 27262976 ta 27262976 fi 29360128 [D 20:01:05.933615] basic type or contiguous data [D 20:01:05.933639] Request disp [D 20:01:05.933663] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:05.933688] process a segment [D 20:01:05.933712] of 27262976 sz 2097152 [D 20:01:05.933735] add a segment 0 [D 20:01:05.933759] retval = 2097152 [D 20:01:05.933784] exiting distribute returned less than expected [D 20:01:05.933808] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:05.933834] *****RETURN***FROM***PROCESS***** [D 20:01:05.933858] end iteration [D 20:01:05.933882] done with segments or bytes [D 20:01:05.933906] of 130023424 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.933932] next loff: 130023424 next poff: 130023424 [D 20:01:05.933957] return value 2097152 [D 20:01:05.933980] finished [D 20:01:05.934004] retval = 2097152 [D 20:01:05.934029] exiting distribute returned less than expected [D 20:01:05.934053] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 130023424 fo 134217728 eof 0 [D 20:01:05.934079] ========================================================= [D 20:01:05.934103] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:05.934129] element 0: offset: 0x9de4b000, size: 2097152 [D 20:01:05.934159] post_sr: da13:3346 bh 58 len 48 wr_id 15e730f0. [D 20:01:05.934230] openib_check_cq: completed send wr_id 15e730f0. [D 20:01:05.935208] openib_check_cq: completed recv wr_id 15e73638. [D 20:01:05.935236] openib_post_rr: da13:3346 bh 14 wr_id 15e73638. [D 20:01:05.935263] openib_check_cq: completed recv wr_id 15e73650. [D 20:01:05.935292] openib_post_rr: da13:3346 bh 15 wr_id 15e73650. [D 20:01:05.935317] openib_check_cq: completed recv wr_id 15e73668. [D 20:01:05.935344] openib_post_rr: da13:3346 bh 16 wr_id 15e73668. [D 20:01:05.935377] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:05.935403] ========================================================= [D 20:01:05.935428] PINT_process_request [D 20:01:05.935452] tiling 13 copies [D 20:01:05.935476] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.935503] lvl 0 el 0 blk 0 by 379584512 [D 20:01:05.935527] to 379584512 ta 369098752 fi 402653184 [D 20:01:05.935552] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:05.935577] basic type or contiguous data [D 20:01:05.935602] Request disp [D 20:01:05.935626] contig_offset = 379584512 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.935650] calling distribute [D 20:01:05.935679] PINT_distribute [D 20:01:05.935704] of 379584512 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.935730] begin iteration loff: 379584512 [D 20:01:05.935755] segment exceeds byte limit [D 20:01:05.935779] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.935805] **********CALL***PROCESS********* [D 20:01:05.935829] segment of 10485760 sz 2097152 [D 20:01:05.935853] PINT_process_request [D 20:01:05.935877] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.935903] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.935928] to 10485760 ta 10485760 fi 12582912 [D 20:01:05.935953] basic type or contiguous data [D 20:01:05.935976] Request disp [D 20:01:05.936000] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.936026] process a segment [D 20:01:05.936050] of 10485760 sz 2097152 [D 20:01:05.936074] add a segment 0 [D 20:01:05.936099] retval = 2097152 [D 20:01:05.936122] exiting distribute returned less than expected [D 20:01:05.936146] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:05.936173] *****RETURN***FROM***PROCESS***** [D 20:01:05.936197] end iteration [D 20:01:05.936223] done with segments or bytes [D 20:01:05.936247] of 381681664 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.936273] next loff: 381681664 next poff: 381681664 [D 20:01:05.936298] return value 2097152 [D 20:01:05.936323] finished [D 20:01:05.936347] retval = 2097152 [D 20:01:05.936370] exiting distribute returned less than expected [D 20:01:05.936395] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 381681664 fo 402653184 eof 0 [D 20:01:05.936421] ========================================================= [D 20:01:05.936446] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:05.936471] element 0: offset: 0x8ce3b000, size: 2097152 [D 20:01:05.936500] post_sr: da13:3346 bh 59 len 48 wr_id 15e73108. [D 20:01:05.936570] openib_check_cq: completed send wr_id 15e73108. [D 20:01:05.937440] openib_check_cq: completed recv wr_id 15e73680. [D 20:01:05.937467] openib_post_rr: da13:3346 bh 17 wr_id 15e73680. [D 20:01:05.937494] openib_check_cq: completed recv wr_id 15e73698. [D 20:01:05.937521] openib_post_rr: da13:3346 bh 18 wr_id 15e73698. [D 20:01:05.937547] openib_check_cq: completed recv wr_id 15e736b0. [D 20:01:05.937574] openib_post_rr: da13:3346 bh 19 wr_id 15e736b0. [D 20:01:05.937606] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:05.937632] ========================================================= [D 20:01:05.937657] PINT_process_request [D 20:01:05.937681] tiling 7 copies [D 20:01:05.937705] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.937731] lvl 0 el 0 blk 0 by 182452224 [D 20:01:05.937756] to 182452224 ta 167772160 fi 201326592 [D 20:01:05.937781] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:05.937806] basic type or contiguous data [D 20:01:05.937830] Request disp [D 20:01:05.937854] contig_offset = 182452224 contig_size = 52428800 lvl_flag = 1 [D 20:01:05.937879] calling distribute [D 20:01:05.937903] PINT_distribute [D 20:01:05.937927] of 182452224 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.937953] begin iteration loff: 182452224 [D 20:01:05.937978] segment exceeds byte limit [D 20:01:05.938003] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:05.938028] **********CALL***PROCESS********* [D 20:01:05.938052] segment of 14680064 sz 2097152 [D 20:01:05.938076] PINT_process_request [D 20:01:05.938099] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.938126] lvl 0 el 0 blk 0 by 14680064 [D 20:01:05.938151] to 14680064 ta 14680064 fi 16777216 [D 20:01:05.938176] basic type or contiguous data [D 20:01:05.938199] Request disp [D 20:01:05.938227] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:05.938256] process a segment [D 20:01:05.938286] of 14680064 sz 2097152 [D 20:01:05.938315] add a segment 0 [D 20:01:05.938340] retval = 2097152 [D 20:01:05.938364] exiting distribute returned less than expected [D 20:01:05.938388] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:05.938414] *****RETURN***FROM***PROCESS***** [D 20:01:05.938439] end iteration [D 20:01:05.938463] done with segments or bytes [D 20:01:05.938487] of 184549376 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.938512] next loff: 184549376 next poff: 184549376 [D 20:01:05.938537] return value 2097152 [D 20:01:05.938561] finished [D 20:01:05.938585] retval = 2097152 [D 20:01:05.938609] exiting distribute returned less than expected [D 20:01:05.938632] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 184549376 fo 201326592 eof 0 [D 20:01:05.938659] ========================================================= [D 20:01:05.938684] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:05.938710] element 0: offset: 0x99247000, size: 2097152 [D 20:01:05.938738] post_sr: da13:3346 bh 60 len 48 wr_id 15e73120. [D 20:01:05.938765] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:05.938791] ========================================================= [D 20:01:05.938816] PINT_process_request [D 20:01:05.938839] tiling 14 copies [D 20:01:05.938863] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.938890] lvl 0 el 0 blk 0 by 413138944 [D 20:01:05.938915] to 413138944 ta 402653184 fi 436207616 [D 20:01:05.938940] mto 10485760 mta 8388608 mfi 10485760 [D 20:01:05.938964] basic type or contiguous data [D 20:01:05.938988] Request disp [D 20:01:05.939012] contig_offset = 413138944 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.939037] calling distribute [D 20:01:05.939062] PINT_distribute [D 20:01:05.939085] of 413138944 sz 23068672 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.939111] begin iteration loff: 413138944 [D 20:01:05.939136] segment exceeds byte limit [D 20:01:05.939160] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.939185] **********CALL***PROCESS********* [D 20:01:05.939208] segment of 10485760 sz 2097152 [D 20:01:05.939233] PINT_process_request [D 20:01:05.939257] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.939283] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.939309] to 10485760 ta 10485760 fi 12582912 [D 20:01:05.939333] basic type or contiguous data [D 20:01:05.939357] Request disp [D 20:01:05.939381] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.939406] process a segment [D 20:01:05.939430] of 10485760 sz 2097152 [D 20:01:05.939454] add a segment 0 [D 20:01:05.939478] retval = 2097152 [D 20:01:05.939502] exiting distribute returned less than expected [D 20:01:05.939526] done sg 1 sm 64 by 2097152 bm 2097152 ta 10485760 to 12582912 fo 12582912 eof 0 [D 20:01:05.939552] *****RETURN***FROM***PROCESS***** [D 20:01:05.939577] end iteration [D 20:01:05.939600] done with segments or bytes [D 20:01:05.939624] of 415236096 sz 20971520 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.939650] next loff: 415236096 next poff: 415236096 [D 20:01:05.939675] return value 2097152 [D 20:01:05.939699] finished [D 20:01:05.939723] retval = 2097152 [D 20:01:05.939746] exiting distribute returned less than expected [D 20:01:05.939770] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 415236096 fo 436207616 eof 0 [D 20:01:05.939797] ========================================================= [D 20:01:05.939821] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:05.939846] element 0: offset: 0x8ae39000, size: 2097152 [D 20:01:05.939872] openib_check_cq: completed send wr_id 15e73120. [D 20:01:05.939898] openib_check_cq: completed recv wr_id 15e736c8. [D 20:01:05.939925] openib_post_rr: da13:3346 bh 20 wr_id 15e736c8. [D 20:01:05.939950] openib_check_cq: completed recv wr_id 15e736e0. [D 20:01:05.939980] openib_post_rr: da13:3346 bh 21 wr_id 15e736e0. [D 20:01:05.940010] post_sr: da13:3346 bh 61 len 48 wr_id 15e73138. [D 20:01:05.940038] PINT_client_state_machine_test id 76 [D 20:01:05.940073] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:05.940098] ========================================================= [D 20:01:05.940123] PINT_process_request [D 20:01:05.940147] tiling 8 copies [D 20:01:05.940172] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.940197] lvl 0 el 0 blk 0 by 216006656 [D 20:01:05.940225] to 216006656 ta 201326592 fi 234881024 [D 20:01:05.940251] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:05.940276] basic type or contiguous data [D 20:01:05.940299] Request disp [D 20:01:05.940323] contig_offset = 216006656 contig_size = 52428800 lvl_flag = 1 [D 20:01:05.940349] calling distribute [D 20:01:05.940373] PINT_distribute [D 20:01:05.940397] of 216006656 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.940423] begin iteration loff: 216006656 [D 20:01:05.940448] segment exceeds byte limit [D 20:01:05.940472] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:05.940497] **********CALL***PROCESS********* [D 20:01:05.940521] segment of 14680064 sz 2097152 [D 20:01:05.940546] PINT_process_request [D 20:01:05.940569] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.940595] lvl 0 el 0 blk 0 by 14680064 [D 20:01:05.940620] to 14680064 ta 14680064 fi 16777216 [D 20:01:05.940645] basic type or contiguous data [D 20:01:05.940669] Request disp [D 20:01:05.940692] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:05.940718] process a segment [D 20:01:05.940742] of 14680064 sz 2097152 [D 20:01:05.940766] add a segment 0 [D 20:01:05.940790] retval = 2097152 [D 20:01:05.940814] exiting distribute returned less than expected [D 20:01:05.940837] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:05.940864] *****RETURN***FROM***PROCESS***** [D 20:01:05.940888] end iteration [D 20:01:05.940912] done with segments or bytes [D 20:01:05.940936] of 218103808 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.940961] next loff: 218103808 next poff: 218103808 [D 20:01:05.940986] return value 2097152 [D 20:01:05.941010] finished [D 20:01:05.941034] retval = 2097152 [D 20:01:05.941058] exiting distribute returned less than expected [D 20:01:05.941082] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 218103808 fo 234881024 eof 0 [D 20:01:05.941108] ========================================================= [D 20:01:05.941133] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:05.941159] element 0: offset: 0x97245000, size: 2097152 [D 20:01:05.941184] openib_check_cq: completed send wr_id 15e73138. [D 20:01:05.941213] post_sr: da13:3346 bh 62 len 48 wr_id 15e73150. [D 20:01:05.941241] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ee0210. [D 20:01:05.941267] flowproto completing 0x15ee0210 [D 20:01:05.941293] Job flows in progress (callback time): 10 [D 20:01:05.941321] sm control op 6 [D 20:01:05.941347] [SM Entering]: (0x15edf3f8) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:05.941372] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:05.941398] (0x15edf490) io_datafile_complete_operations (tag 2) [D 20:01:05.941423] matched completed flow for context 0x15edfc60 [D 20:01:05.941448] *** all operations completed (msgpairs, flows, write acks) [D 20:01:05.941473] [SM Exiting]: (0x15edf3f8) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:05.941499] [SM Entering]: (0x15edf3f8) client_io_sm:io_analyze_results (status: 2) [D 20:01:05.941525] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:05.941550] (0x15edf490) io state: io_analyze_results [D 20:01:05.941575] - io_check_context_status called [D 20:01:05.941602] 10485760 bytes read from context 0x15edfc60 [D 20:01:05.941628] [1/1] running size is 10485760 [D 20:01:05.941652] [2/1] running size is 10485760 [D 20:01:05.941681] no context errors found; using: Success [D 20:01:05.941706] total bytes transferred is 10485760 [D 20:01:05.941732] PINT_new_request_states n=1 [D 20:01:05.941757] PINT_new_request_states n=1 [D 20:01:05.941781] ========================================================= [D 20:01:05.941806] PINT_process_request [D 20:01:05.941830] tiling 2 copies [D 20:01:05.941854] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.941881] lvl 0 el 0 blk 0 by 0 [D 20:01:05.941905] to 0 ta 0 fi 33554432 [D 20:01:05.941929] mto 0 mta 0 mfi 33554432 [D 20:01:05.941955] basic type or contiguous data [D 20:01:05.941979] Request disp [D 20:01:05.942003] contig_offset = 0 contig_size = 67108864 lvl_flag = 1 [D 20:01:05.942028] calling distribute [D 20:01:05.942052] PINT_distribute [D 20:01:05.942076] of 0 sz 33554432 ix 0 sm 50 by 0 bm 33554432 fsz 0 exfl 1 [D 20:01:05.942103] begin iteration loff: 0 [D 20:01:05.942128] file being extended [D 20:01:05.942152] process a segment [D 20:01:05.942175] of 0 sz 33554432 [D 20:01:05.942199] add a segment 0 [D 20:01:05.942228] end iteration [D 20:01:05.942255] done with segments or bytes [D 20:01:05.942283] of 33554432 sz 0 sg 1 sm 50 by 33554432 bm 33554432 [D 20:01:05.942313] next loff: 33554432 next poff: 33554432 [D 20:01:05.942338] (return value) 33554432 [D 20:01:05.942363] finished [D 20:01:05.942387] retval = 33554432 [D 20:01:05.942412] exiting distribute returned less than expected [D 20:01:05.942436] done sg 1 sm 50 by 33554432 bm 33554432 ta 0 to 33554432 fo 33554432 eof 0 [D 20:01:05.942461] ========================================================= [D 20:01:05.942489] PINT_new_request_states n=1 [D 20:01:05.942514] PINT_new_request_states n=1 [D 20:01:05.942539] ========================================================= [D 20:01:05.942564] PINT_process_request [D 20:01:05.942587] tiling 15 copies [D 20:01:05.942611] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.942638] lvl 0 el 0 blk 0 by 0 [D 20:01:05.942663] to 0 ta 0 fi 480247808 [D 20:01:05.942687] mto 0 mta 0 mfi 33554432 [D 20:01:05.942712] basic type or contiguous data [D 20:01:05.942736] Request disp [D 20:01:05.942760] contig_offset = 0 contig_size = 503316480 lvl_flag = 1 [D 20:01:05.942785] calling distribute [D 20:01:05.942809] PINT_distribute [D 20:01:05.942832] of 0 sz 480247808 ix 0 sm 50 by 0 bm 480247808 fsz 480247808 exfl 0 [D 20:01:05.942859] begin iteration loff: 0 [D 20:01:05.942884] process a segment [D 20:01:05.942908] of 0 sz 480247808 [D 20:01:05.942933] add a segment 0 [D 20:01:05.942956] end iteration [D 20:01:05.942980] done with segments or bytes [D 20:01:05.943004] of 480247808 sz 0 sg 1 sm 50 by 480247808 bm 480247808 [D 20:01:05.943030] next loff: 480247808 next poff: 480247808 [D 20:01:05.943055] [return value] 480247808 (EOF) [D 20:01:05.943079] (return value) 480247808 (EOF) [D 20:01:05.943104] finished [D 20:01:05.943128] retval = 480247808 [D 20:01:05.943152] exiting distribute returned less than expected [D 20:01:05.943176] done sg 1 sm 50 by 480247808 bm 480247808 ta 0 to 480247808 fo 480247808 eof 1 [D 20:01:05.943202] ========================================================= [D 20:01:05.943228] PINT_new_request_states n=1 [D 20:01:05.943253] PINT_new_request_states n=1 [D 20:01:05.943280] ========================================================= [D 20:01:05.943305] PINT_process_request [D 20:01:05.943328] tiling 15 copies [D 20:01:05.943352] skipping ahead to target_offset [D 20:01:05.943376] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.943403] lvl 0 el 0 blk 0 by 0 [D 20:01:05.943428] to 0 ta 469762048 fi 480247808 [D 20:01:05.943454] mto 0 mta 0 mfi 33554432 [D 20:01:05.943479] basic type or contiguous data [D 20:01:05.943503] Request disp [D 20:01:05.943530] contig_offset = 0 contig_size = 503316480 lvl_flag = 1 [D 20:01:05.943555] process logical skip [D 20:01:05.943579] retval = 469762048 [D 20:01:05.943604] exiting logical skip because distribute indicates done [D 20:01:05.943628] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.943655] lvl 0 el 0 blk 0 by 469762048 [D 20:01:05.943679] to 469762048 ta 469762048 fi 480247808 [D 20:01:05.943704] mto 0 mta 0 mfi 33554432 [D 20:01:05.943729] basic type or contiguous data [D 20:01:05.943753] Request disp [D 20:01:05.943777] contig_offset = 469762048 contig_size = 33554432 lvl_flag = 1 [D 20:01:05.943801] calling distribute [D 20:01:05.943826] PINT_distribute [D 20:01:05.943850] of 469762048 sz 10485760 ix 0 sm 50 by 0 bm 33554432 fsz 480247808 exfl 0 [D 20:01:05.943877] begin iteration loff: 469762048 [D 20:01:05.943902] client lstof 0 diff 0 sgof 0 [D 20:01:05.943926] **********CALL***PROCESS********* [D 20:01:05.943950] segment of 0 sz 10485760 [D 20:01:05.943975] PINT_process_request [D 20:01:05.943999] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.944025] lvl 0 el 0 blk 0 by 0 [D 20:01:05.944049] to 0 ta 0 fi 10485760 [D 20:01:05.944074] basic type or contiguous data [D 20:01:05.944098] Request disp [D 20:01:05.944122] contig_offset = 0 contig_size = 33554432 lvl_flag = 1 [D 20:01:05.944148] process a segment [D 20:01:05.944170] of 0 sz 10485760 [D 20:01:05.944195] add a segment 0 [D 20:01:05.944222] retval = 10485760 [D 20:01:05.944246] exiting distribute returned less than expected [D 20:01:05.944271] done sg 1 sm 50 by 10485760 bm 33554432 ta 0 to 10485760 fo 10485760 eof 0 [D 20:01:05.944296] *****RETURN***FROM***PROCESS***** [D 20:01:05.944321] end iteration [D 20:01:05.944345] of 480247808 sz 0 sg 1 sm 50 by 10485760 bm 33554432 [D 20:01:05.944371] next loff: 480247808 next poff: 480247808 [D 20:01:05.944396] [return value] 10485760 (EOF) [D 20:01:05.944419] (return value) 10485760 (EOF) [D 20:01:05.944444] finished [D 20:01:05.944468] retval = 10485760 [D 20:01:05.944492] exiting distribute returned less than expected [D 20:01:05.944516] done sg 1 sm 50 by 10485760 bm 33554432 ta 469762048 to 480247808 fo 480247808 eof 1 [D 20:01:05.944542] ========================================================= [D 20:01:05.944567] ========================================================= [D 20:01:05.944591] PINT_process_request [D 20:01:05.944615] tiling 16 copies [D 20:01:05.944639] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.944664] lvl 0 el 0 blk 0 by 480247808 [D 20:01:05.944689] to 480247808 ta 469762048 fi 503316480 [D 20:01:05.944714] mto 10485760 mta 0 mfi 10485760 [D 20:01:05.944739] basic type or contiguous data [D 20:01:05.944763] Request disp [D 20:01:05.944787] contig_offset = 480247808 contig_size = 56623104 lvl_flag = 1 [D 20:01:05.944812] calling distribute [D 20:01:05.944836] PINT_distribute [D 20:01:05.944860] of 480247808 sz 23068672 ix 0 sm 50 by 0 bm 33554432 fsz 480247808 exfl 1 [D 20:01:05.944886] begin iteration loff: 480247808 [D 20:01:05.944911] file being extended [D 20:01:05.944935] client lstof 10485760 diff 0 sgof 10485760 [D 20:01:05.944960] **********CALL***PROCESS********* [D 20:01:05.944984] segment of 10485760 sz 23068672 [D 20:01:05.945008] PINT_process_request [D 20:01:05.945032] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.945058] lvl 0 el 0 blk 0 by 10485760 [D 20:01:05.945083] to 10485760 ta 10485760 fi 33554432 [D 20:01:05.945108] basic type or contiguous data [D 20:01:05.945132] Request disp [D 20:01:05.945156] contig_offset = 10485760 contig_size = 23068672 lvl_flag = 1 [D 20:01:05.945180] process a segment [D 20:01:05.945204] of 10485760 sz 23068672 [D 20:01:05.945229] add a segment 0 [D 20:01:05.945253] retval = 23068672 [D 20:01:05.945277] return from level -1 [D 20:01:05.945301] done sg 1 sm 50 by 23068672 bm 33554432 ta 10485760 to 33554432 fo 33554432 eof 0 [D 20:01:05.945330] *****RETURN***FROM***PROCESS***** [D 20:01:05.945354] end iteration [D 20:01:05.945378] of 503316480 sz 0 sg 1 sm 50 by 23068672 bm 33554432 [D 20:01:05.945404] next loff: 503316480 next poff: 503316480 [D 20:01:05.945428] (return value) 23068672 [D 20:01:05.945453] finished [D 20:01:05.945477] retval = 23068672 [D 20:01:05.945501] exiting distribute returned less than expected [D 20:01:05.945526] done sg 1 sm 50 by 23068672 bm 33554432 ta 469762048 to 503316480 fo 503316480 eof 0 [D 20:01:05.945551] ========================================================= [D 20:01:05.952719] [SM Exiting]: (0x15edf3f8) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:05.952770] [SM Entering]: (0x15edf3f8) client_io_sm:io_cleanup (status: 2) [D 20:01:05.952795] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:05.952821] (0x15edf490) io state: io_cleanup [D 20:01:05.952849] [SM Exiting]: (0x15edf3f8) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:05.952876] client_state_machine_terminate smcb 0x15edf3f8 [D 20:01:05.952900] add smcb 0x15edf3f8 to completion list [D 20:01:05.952927] PINT_client_state_machine_test id 76 [D 20:01:05.952959] openib_check_cq: completed send wr_id 15e73150. [D 20:01:05.952985] openib_check_cq: completed recv wr_id 15e736f8. [D 20:01:05.953013] openib_post_rr: da13:3346 bh 22 wr_id 15e736f8. [D 20:01:05.953040] openib_check_cq: completed recv wr_id 15e73710. [D 20:01:05.953066] openib_post_rr: da13:3346 bh 23 wr_id 15e73710. [D 20:01:05.953092] openib_check_cq: completed recv wr_id 15e73728. [D 20:01:05.953123] openib_post_rr: da13:3346 bh 24 wr_id 15e73728. [D 20:01:05.953149] openib_check_cq: completed recv wr_id 15e73740. [D 20:01:05.953176] openib_post_rr: da13:3346 bh 25 wr_id 15e73740. [D 20:01:05.953202] openib_check_cq: completed recv wr_id 15e73758. [D 20:01:05.953228] openib_post_rr: da13:3346 bh 26 wr_id 15e73758. [D 20:01:05.953255] openib_check_cq: completed recv wr_id 15e73770. [D 20:01:05.953281] openib_post_rr: da13:3346 bh 27 wr_id 15e73770. [D 20:01:05.953308] openib_check_cq: completed recv wr_id 15e73788. [D 20:01:05.953334] openib_post_rr: da13:3346 bh 28 wr_id 15e73788. [D 20:01:05.953360] openib_check_cq: completed recv wr_id 15e737a0. [D 20:01:05.953387] openib_post_rr: da13:3346 bh 29 wr_id 15e737a0. [D 20:01:05.953413] openib_check_cq: completed recv wr_id 15e737b8. [D 20:01:05.953442] openib_post_rr: da13:3346 bh 30 wr_id 15e737b8. [D 20:01:05.953469] openib_check_cq: completed recv wr_id 15e737d0. [D 20:01:05.953495] openib_post_rr: da13:3346 bh 31 wr_id 15e737d0. [D 20:01:05.953521] openib_check_cq: completed recv wr_id 15e737e8. [D 20:01:05.953547] openib_post_rr: da13:3346 bh 32 wr_id 15e737e8. [D 20:01:05.953583] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:05.953610] ========================================================= [D 20:01:05.953635] PINT_process_request [D 20:01:05.953660] tiling 7 copies [D 20:01:05.953683] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.953710] lvl 0 el 0 blk 0 by 184549376 [D 20:01:05.953735] to 184549376 ta 167772160 fi 201326592 [D 20:01:05.953760] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:05.953785] basic type or contiguous data [D 20:01:05.953809] Request disp [D 20:01:05.953833] contig_offset = 184549376 contig_size = 50331648 lvl_flag = 1 [D 20:01:05.953859] calling distribute [D 20:01:05.953883] PINT_distribute [D 20:01:05.953907] of 184549376 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.953934] begin iteration loff: 184549376 [D 20:01:05.953959] segment exceeds byte limit [D 20:01:05.953984] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:05.954009] **********CALL***PROCESS********* [D 20:01:05.954032] segment of 16777216 sz 2097152 [D 20:01:05.954056] PINT_process_request [D 20:01:05.954081] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.954112] lvl 0 el 0 blk 0 by 16777216 [D 20:01:05.954137] to 16777216 ta 16777216 fi 18874368 [D 20:01:05.954162] basic type or contiguous data [D 20:01:05.954186] Request disp [D 20:01:05.954213] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:05.954242] process a segment [D 20:01:05.954270] of 16777216 sz 2097152 [D 20:01:05.954298] add a segment 0 [D 20:01:05.954325] retval = 2097152 [D 20:01:05.954350] exiting distribute returned less than expected [D 20:01:05.954374] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:05.954401] *****RETURN***FROM***PROCESS***** [D 20:01:05.954425] end iteration [D 20:01:05.954449] done with segments or bytes [D 20:01:05.954473] of 186646528 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.954499] next loff: 186646528 next poff: 186646528 [D 20:01:05.954524] return value 2097152 [D 20:01:05.954548] finished [D 20:01:05.954572] retval = 2097152 [D 20:01:05.954597] exiting distribute returned less than expected [D 20:01:05.954621] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 186646528 fo 201326592 eof 0 [D 20:01:05.954648] ========================================================= [D 20:01:05.954672] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:05.954697] element 0: offset: 0x99447000, size: 2097152 [D 20:01:05.954728] post_sr: da13:3346 bh 63 len 48 wr_id 15e73168. [D 20:01:05.954756] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:05.954782] ========================================================= [D 20:01:05.954806] PINT_process_request [D 20:01:05.954830] tiling 8 copies [D 20:01:05.954854] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.954880] lvl 0 el 0 blk 0 by 218103808 [D 20:01:05.954906] to 218103808 ta 201326592 fi 234881024 [D 20:01:05.954931] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:05.954956] basic type or contiguous data [D 20:01:05.954979] Request disp [D 20:01:05.955003] contig_offset = 218103808 contig_size = 50331648 lvl_flag = 1 [D 20:01:05.955029] calling distribute [D 20:01:05.955053] PINT_distribute [D 20:01:05.955077] of 218103808 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.955103] begin iteration loff: 218103808 [D 20:01:05.955128] segment exceeds byte limit [D 20:01:05.955152] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:05.955177] **********CALL***PROCESS********* [D 20:01:05.955201] segment of 16777216 sz 2097152 [D 20:01:05.955225] PINT_process_request [D 20:01:05.955249] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.955276] lvl 0 el 0 blk 0 by 16777216 [D 20:01:05.955301] to 16777216 ta 16777216 fi 18874368 [D 20:01:05.955326] basic type or contiguous data [D 20:01:05.955349] Request disp [D 20:01:05.955372] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:05.955398] process a segment [D 20:01:05.955422] of 16777216 sz 2097152 [D 20:01:05.955446] add a segment 0 [D 20:01:05.955470] retval = 2097152 [D 20:01:05.955494] exiting distribute returned less than expected [D 20:01:05.955518] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:05.955544] *****RETURN***FROM***PROCESS***** [D 20:01:05.955569] end iteration [D 20:01:05.955593] done with segments or bytes [D 20:01:05.955616] of 220200960 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.955642] next loff: 220200960 next poff: 220200960 [D 20:01:05.955667] return value 2097152 [D 20:01:05.955691] finished [D 20:01:05.955715] retval = 2097152 [D 20:01:05.955739] exiting distribute returned less than expected [D 20:01:05.955763] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 220200960 fo 234881024 eof 0 [D 20:01:05.955789] ========================================================= [D 20:01:05.955814] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:05.955842] element 0: offset: 0x97445000, size: 2097152 [D 20:01:05.955868] openib_check_cq: completed send wr_id 15e73168. [D 20:01:05.955898] post_sr: da13:3346 bh 64 len 48 wr_id 15e73180. [D 20:01:05.955925] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:05.955951] ========================================================= [D 20:01:05.955976] PINT_process_request [D 20:01:05.956000] tiling 6 copies [D 20:01:05.956024] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.956050] lvl 0 el 0 blk 0 by 153092096 [D 20:01:05.956075] to 153092096 ta 134217728 fi 167772160 [D 20:01:05.956099] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:05.956124] basic type or contiguous data [D 20:01:05.956148] Request disp [D 20:01:05.956172] contig_offset = 153092096 contig_size = 48234496 lvl_flag = 1 [D 20:01:05.956197] calling distribute [D 20:01:05.956224] PINT_distribute [D 20:01:05.956248] of 153092096 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.956274] begin iteration loff: 153092096 [D 20:01:05.956299] segment exceeds byte limit [D 20:01:05.956324] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:05.956349] **********CALL***PROCESS********* [D 20:01:05.956373] segment of 18874368 sz 2097152 [D 20:01:05.956397] PINT_process_request [D 20:01:05.956420] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.956447] lvl 0 el 0 blk 0 by 18874368 [D 20:01:05.956471] to 18874368 ta 18874368 fi 20971520 [D 20:01:05.956496] basic type or contiguous data [D 20:01:05.956520] Request disp [D 20:01:05.956543] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:05.956568] process a segment [D 20:01:05.956592] of 18874368 sz 2097152 [D 20:01:05.956616] add a segment 0 [D 20:01:05.956640] retval = 2097152 [D 20:01:05.956664] exiting distribute returned less than expected [D 20:01:05.956687] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:05.956714] *****RETURN***FROM***PROCESS***** [D 20:01:05.956738] end iteration [D 20:01:05.956762] done with segments or bytes [D 20:01:05.956786] of 155189248 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.956811] next loff: 155189248 next poff: 155189248 [D 20:01:05.956836] return value 2097152 [D 20:01:05.956860] finished [D 20:01:05.956884] retval = 2097152 [D 20:01:05.956908] exiting distribute returned less than expected [D 20:01:05.956932] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 155189248 fo 167772160 eof 0 [D 20:01:05.956958] ========================================================= [D 20:01:05.956983] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:05.957008] element 0: offset: 0x9b649000, size: 2097152 [D 20:01:05.957034] openib_check_cq: completed send wr_id 15e73180. [D 20:01:05.957063] post_sr: da13:3346 bh 65 len 48 wr_id 15e73198. [D 20:01:05.957091] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:05.957117] ========================================================= [D 20:01:05.957141] PINT_process_request [D 20:01:05.957165] tiling 9 copies [D 20:01:05.957189] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.957216] lvl 0 el 0 blk 0 by 249561088 [D 20:01:05.957241] to 249561088 ta 234881024 fi 268435456 [D 20:01:05.957265] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:05.957290] basic type or contiguous data [D 20:01:05.957314] Request disp [D 20:01:05.957338] contig_offset = 249561088 contig_size = 52428800 lvl_flag = 1 [D 20:01:05.957364] calling distribute [D 20:01:05.957387] PINT_distribute [D 20:01:05.957411] of 249561088 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.957438] begin iteration loff: 249561088 [D 20:01:05.957463] segment exceeds byte limit [D 20:01:05.957488] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:05.957512] **********CALL***PROCESS********* [D 20:01:05.957539] segment of 14680064 sz 2097152 [D 20:01:05.957564] PINT_process_request [D 20:01:05.957588] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.957614] lvl 0 el 0 blk 0 by 14680064 [D 20:01:05.957639] to 14680064 ta 14680064 fi 16777216 [D 20:01:05.957664] basic type or contiguous data [D 20:01:05.957688] Request disp [D 20:01:05.957712] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:05.957737] process a segment [D 20:01:05.957761] of 14680064 sz 2097152 [D 20:01:05.957785] add a segment 0 [D 20:01:05.957810] retval = 2097152 [D 20:01:05.957834] exiting distribute returned less than expected [D 20:01:05.957859] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:05.957884] *****RETURN***FROM***PROCESS***** [D 20:01:05.957909] end iteration [D 20:01:05.957933] done with segments or bytes [D 20:01:05.957957] of 251658240 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.957983] next loff: 251658240 next poff: 251658240 [D 20:01:05.958007] return value 2097152 [D 20:01:05.958031] finished [D 20:01:05.958055] retval = 2097152 [D 20:01:05.958080] exiting distribute returned less than expected [D 20:01:05.958104] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 251658240 fo 268435456 eof 0 [D 20:01:05.958130] ========================================================= [D 20:01:05.958155] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:05.958180] element 0: offset: 0x95243000, size: 2097152 [D 20:01:05.958207] openib_check_cq: completed send wr_id 15e73198. [D 20:01:05.958242] post_sr: da13:3346 bh 66 len 48 wr_id 15e731b0. [D 20:01:05.958273] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:05.958302] ========================================================= [D 20:01:05.958330] PINT_process_request [D 20:01:05.958353] tiling 10 copies [D 20:01:05.958378] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.958404] lvl 0 el 0 blk 0 by 283115520 [D 20:01:05.958430] to 283115520 ta 268435456 fi 301989888 [D 20:01:05.958455] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:05.958479] basic type or contiguous data [D 20:01:05.958503] Request disp [D 20:01:05.958527] contig_offset = 283115520 contig_size = 52428800 lvl_flag = 1 [D 20:01:05.958552] calling distribute [D 20:01:05.958577] PINT_distribute [D 20:01:05.958600] of 283115520 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.958626] begin iteration loff: 283115520 [D 20:01:05.958651] segment exceeds byte limit [D 20:01:05.958676] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:05.958701] **********CALL***PROCESS********* [D 20:01:05.958724] segment of 14680064 sz 2097152 [D 20:01:05.958749] PINT_process_request [D 20:01:05.958773] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.958799] lvl 0 el 0 blk 0 by 14680064 [D 20:01:05.958824] to 14680064 ta 14680064 fi 16777216 [D 20:01:05.958848] basic type or contiguous data [D 20:01:05.958872] Request disp [D 20:01:05.958896] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:05.958922] process a segment [D 20:01:05.958946] of 14680064 sz 2097152 [D 20:01:05.958970] add a segment 0 [D 20:01:05.958993] retval = 2097152 [D 20:01:05.959018] exiting distribute returned less than expected [D 20:01:05.959042] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:05.959068] *****RETURN***FROM***PROCESS***** [D 20:01:05.959093] end iteration [D 20:01:05.959116] done with segments or bytes [D 20:01:05.959140] of 285212672 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.959166] next loff: 285212672 next poff: 285212672 [D 20:01:05.959191] return value 2097152 [D 20:01:05.959216] finished [D 20:01:05.959239] retval = 2097152 [D 20:01:05.959263] exiting distribute returned less than expected [D 20:01:05.959288] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 285212672 fo 301989888 eof 0 [D 20:01:05.959317] ========================================================= [D 20:01:05.959342] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:05.959368] element 0: offset: 0x93241000, size: 2097152 [D 20:01:05.959394] openib_check_cq: completed send wr_id 15e731b0. [D 20:01:05.959424] post_sr: da13:3346 bh 67 len 48 wr_id 15e731c8. [D 20:01:05.959460] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:05.959486] ========================================================= [D 20:01:05.959510] PINT_process_request [D 20:01:05.959534] tiling 12 copies [D 20:01:05.959558] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.959585] lvl 0 el 0 blk 0 by 348127232 [D 20:01:05.959609] to 348127232 ta 335544320 fi 369098752 [D 20:01:05.959634] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:05.959660] basic type or contiguous data [D 20:01:05.959684] Request disp [D 20:01:05.959708] contig_offset = 348127232 contig_size = 54525952 lvl_flag = 1 [D 20:01:05.959733] calling distribute [D 20:01:05.959757] PINT_distribute [D 20:01:05.959781] of 348127232 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.959808] begin iteration loff: 348127232 [D 20:01:05.959833] segment exceeds byte limit [D 20:01:05.959856] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:05.959881] **********CALL***PROCESS********* [D 20:01:05.959905] segment of 12582912 sz 2097152 [D 20:01:05.959930] PINT_process_request [D 20:01:05.959954] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.959980] lvl 0 el 0 blk 0 by 12582912 [D 20:01:05.960004] to 12582912 ta 12582912 fi 14680064 [D 20:01:05.960029] basic type or contiguous data [D 20:01:05.960053] Request disp [D 20:01:05.960077] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:05.960102] process a segment [D 20:01:05.960125] of 12582912 sz 2097152 [D 20:01:05.960149] add a segment 0 [D 20:01:05.960173] retval = 2097152 [D 20:01:05.960198] exiting distribute returned less than expected [D 20:01:05.960225] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:05.960251] *****RETURN***FROM***PROCESS***** [D 20:01:05.960275] end iteration [D 20:01:05.960299] done with segments or bytes [D 20:01:05.960323] of 350224384 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.960349] next loff: 350224384 next poff: 350224384 [D 20:01:05.960373] return value 2097152 [D 20:01:05.960398] finished [D 20:01:05.960421] retval = 2097152 [D 20:01:05.960445] exiting distribute returned less than expected [D 20:01:05.960470] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 350224384 fo 369098752 eof 0 [D 20:01:05.960495] ========================================================= [D 20:01:05.960520] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:05.960545] element 0: offset: 0x8f03d000, size: 2097152 [D 20:01:05.960571] openib_check_cq: completed send wr_id 15e731c8. [D 20:01:05.960600] post_sr: da13:3346 bh 68 len 48 wr_id 15e731e0. [D 20:01:05.960627] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:05.960654] ========================================================= [D 20:01:05.960678] PINT_process_request [D 20:01:05.960701] tiling 11 copies [D 20:01:05.960726] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.960753] lvl 0 el 0 blk 0 by 314572800 [D 20:01:05.960778] to 314572800 ta 301989888 fi 335544320 [D 20:01:05.960803] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:05.960827] basic type or contiguous data [D 20:01:05.960852] Request disp [D 20:01:05.960876] contig_offset = 314572800 contig_size = 54525952 lvl_flag = 1 [D 20:01:05.960901] calling distribute [D 20:01:05.960925] PINT_distribute [D 20:01:05.960948] of 314572800 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.960978] begin iteration loff: 314572800 [D 20:01:05.961004] segment exceeds byte limit [D 20:01:05.961028] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:05.961052] **********CALL***PROCESS********* [D 20:01:05.961076] segment of 12582912 sz 2097152 [D 20:01:05.961101] PINT_process_request [D 20:01:05.961125] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.961151] lvl 0 el 0 blk 0 by 12582912 [D 20:01:05.961175] to 12582912 ta 12582912 fi 14680064 [D 20:01:05.961200] basic type or contiguous data [D 20:01:05.961225] Request disp [D 20:01:05.961248] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:05.961274] process a segment [D 20:01:05.961297] of 12582912 sz 2097152 [D 20:01:05.961321] add a segment 0 [D 20:01:05.961345] retval = 2097152 [D 20:01:05.961369] exiting distribute returned less than expected [D 20:01:05.961393] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:05.961419] *****RETURN***FROM***PROCESS***** [D 20:01:05.961443] end iteration [D 20:01:05.961467] done with segments or bytes [D 20:01:05.961491] of 316669952 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.961517] next loff: 316669952 next poff: 316669952 [D 20:01:05.961542] return value 2097152 [D 20:01:05.961566] finished [D 20:01:05.961589] retval = 2097152 [D 20:01:05.961614] exiting distribute returned less than expected [D 20:01:05.961638] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 316669952 fo 335544320 eof 0 [D 20:01:05.961664] ========================================================= [D 20:01:05.961688] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:05.961713] element 0: offset: 0x9103f000, size: 2097152 [D 20:01:05.961740] openib_check_cq: completed send wr_id 15e731e0. [D 20:01:05.961766] openib_check_cq: completed recv wr_id 15e73800. [D 20:01:05.961791] openib_post_rr: da13:3346 bh 33 wr_id 15e73800. [D 20:01:05.961822] post_sr: da13:3346 bh 69 len 48 wr_id 15e731f8. [D 20:01:05.961849] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:05.961874] ========================================================= [D 20:01:05.961899] PINT_process_request [D 20:01:05.961923] tiling 13 copies [D 20:01:05.961947] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.961974] lvl 0 el 0 blk 0 by 381681664 [D 20:01:05.961998] to 381681664 ta 369098752 fi 402653184 [D 20:01:05.962023] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:05.962048] basic type or contiguous data [D 20:01:05.962072] Request disp [D 20:01:05.962097] contig_offset = 381681664 contig_size = 54525952 lvl_flag = 1 [D 20:01:05.962121] calling distribute [D 20:01:05.962145] PINT_distribute [D 20:01:05.962169] of 381681664 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.962196] begin iteration loff: 381681664 [D 20:01:05.962225] segment exceeds byte limit [D 20:01:05.962253] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:05.962282] **********CALL***PROCESS********* [D 20:01:05.962308] segment of 12582912 sz 2097152 [D 20:01:05.962336] PINT_process_request [D 20:01:05.962360] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.962387] lvl 0 el 0 blk 0 by 12582912 [D 20:01:05.962411] to 12582912 ta 12582912 fi 14680064 [D 20:01:05.962436] basic type or contiguous data [D 20:01:05.962460] Request disp [D 20:01:05.962484] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:05.962509] process a segment [D 20:01:05.962532] of 12582912 sz 2097152 [D 20:01:05.962557] add a segment 0 [D 20:01:05.962581] retval = 2097152 [D 20:01:05.962605] exiting distribute returned less than expected [D 20:01:05.962630] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:05.962656] *****RETURN***FROM***PROCESS***** [D 20:01:05.962680] end iteration [D 20:01:05.962707] done with segments or bytes [D 20:01:05.962731] of 383778816 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.962758] next loff: 383778816 next poff: 383778816 [D 20:01:05.962782] return value 2097152 [D 20:01:05.962807] finished [D 20:01:05.962831] retval = 2097152 [D 20:01:05.962855] exiting distribute returned less than expected [D 20:01:05.962879] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 383778816 fo 402653184 eof 0 [D 20:01:05.962905] ========================================================= [D 20:01:05.962930] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:05.962956] element 0: offset: 0x8d03b000, size: 2097152 [D 20:01:05.962982] openib_check_cq: completed send wr_id 15e731f8. [D 20:01:05.963010] post_sr: da13:3346 bh 70 len 48 wr_id 15e73210. [D 20:01:05.963038] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:05.963064] ========================================================= [D 20:01:05.963088] PINT_process_request [D 20:01:05.963112] tiling 14 copies [D 20:01:05.963136] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.963163] lvl 0 el 0 blk 0 by 415236096 [D 20:01:05.963188] to 415236096 ta 402653184 fi 436207616 [D 20:01:05.963212] mto 12582912 mta 10485760 mfi 12582912 [D 20:01:05.963237] basic type or contiguous data [D 20:01:05.963261] Request disp [D 20:01:05.963285] contig_offset = 415236096 contig_size = 54525952 lvl_flag = 1 [D 20:01:05.963311] calling distribute [D 20:01:05.963335] PINT_distribute [D 20:01:05.963358] of 415236096 sz 20971520 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.963385] begin iteration loff: 415236096 [D 20:01:05.963410] segment exceeds byte limit [D 20:01:05.963436] client lstof 12582912 diff 0 sgof 12582912 [D 20:01:05.963460] **********CALL***PROCESS********* [D 20:01:05.963485] segment of 12582912 sz 2097152 [D 20:01:05.963510] PINT_process_request [D 20:01:05.963534] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.963560] lvl 0 el 0 blk 0 by 12582912 [D 20:01:05.963584] to 12582912 ta 12582912 fi 14680064 [D 20:01:05.963609] basic type or contiguous data [D 20:01:05.963633] Request disp [D 20:01:05.963657] contig_offset = 12582912 contig_size = 20971520 lvl_flag = 1 [D 20:01:05.963682] process a segment [D 20:01:05.963706] of 12582912 sz 2097152 [D 20:01:05.963730] add a segment 0 [D 20:01:05.963754] retval = 2097152 [D 20:01:05.963778] exiting distribute returned less than expected [D 20:01:05.963803] done sg 1 sm 64 by 2097152 bm 2097152 ta 12582912 to 14680064 fo 14680064 eof 0 [D 20:01:05.963829] *****RETURN***FROM***PROCESS***** [D 20:01:05.963853] end iteration [D 20:01:05.963877] done with segments or bytes [D 20:01:05.963901] of 417333248 sz 18874368 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.963927] next loff: 417333248 next poff: 417333248 [D 20:01:05.963952] return value 2097152 [D 20:01:05.963975] finished [D 20:01:05.963999] retval = 2097152 [D 20:01:05.964023] exiting distribute returned less than expected [D 20:01:05.964048] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 417333248 fo 436207616 eof 0 [D 20:01:05.964074] ========================================================= [D 20:01:05.964097] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:05.964123] element 0: offset: 0x8b039000, size: 2097152 [D 20:01:05.964150] openib_check_cq: completed send wr_id 15e73210. [D 20:01:05.964176] openib_check_cq: completed recv wr_id 15e73818. [D 20:01:05.964202] openib_post_rr: da13:3346 bh 34 wr_id 15e73818. [D 20:01:05.964232] openib_check_cq: completed recv wr_id 15e73830. [D 20:01:05.964261] openib_post_rr: da13:3346 bh 35 wr_id 15e73830. [D 20:01:05.964286] openib_check_cq: completed recv wr_id 15e73848. [D 20:01:05.964315] openib_post_rr: da13:3346 bh 36 wr_id 15e73848. [D 20:01:05.964341] openib_check_cq: completed recv wr_id 15e73860. [D 20:01:05.964369] openib_post_rr: da13:3346 bh 37 wr_id 15e73860. [D 20:01:05.964399] openib_check_cq: completed recv wr_id 15e73878. [D 20:01:05.964426] openib_post_rr: da13:3346 bh 38 wr_id 15e73878. [D 20:01:05.964453] openib_check_cq: completed recv wr_id 15e73890. [D 20:01:05.964481] openib_post_rr: da13:3346 bh 39 wr_id 15e73890. [D 20:01:05.964508] openib_check_cq: completed recv wr_id 15e738a8. [D 20:01:05.964535] openib_post_rr: da13:3346 bh 40 wr_id 15e738a8. [D 20:01:05.964562] openib_check_cq: completed recv wr_id 15e738c0. [D 20:01:05.964591] openib_post_rr: da13:3346 bh 41 wr_id 15e738c0. [D 20:01:05.964616] openib_check_cq: completed recv wr_id 15e738d8. [D 20:01:05.964644] openib_post_rr: da13:3346 bh 42 wr_id 15e738d8. [D 20:01:05.964674] post_sr: da13:3346 bh 71 len 48 wr_id 15e73228. [D 20:01:05.964702] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:05.964728] ========================================================= [D 20:01:05.964753] PINT_process_request [D 20:01:05.964776] tiling 5 copies [D 20:01:05.964800] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.964827] lvl 0 el 0 blk 0 by 130023424 [D 20:01:05.964852] to 130023424 ta 100663296 fi 134217728 [D 20:01:05.964877] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:05.964901] basic type or contiguous data [D 20:01:05.964925] Request disp [D 20:01:05.964949] contig_offset = 130023424 contig_size = 37748736 lvl_flag = 1 [D 20:01:05.964975] calling distribute [D 20:01:05.964999] PINT_distribute [D 20:01:05.965022] of 130023424 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:05.965048] begin iteration loff: 130023424 [D 20:01:05.965073] segment exceeds byte limit [D 20:01:05.965098] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:05.965123] **********CALL***PROCESS********* [D 20:01:05.965147] segment of 29360128 sz 2097152 [D 20:01:05.965171] PINT_process_request [D 20:01:05.965195] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.965221] lvl 0 el 0 blk 0 by 29360128 [D 20:01:05.965246] to 29360128 ta 29360128 fi 31457280 [D 20:01:05.965270] basic type or contiguous data [D 20:01:05.965294] Request disp [D 20:01:05.965318] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:05.965344] process a segment [D 20:01:05.965368] of 29360128 sz 2097152 [D 20:01:05.965392] add a segment 0 [D 20:01:05.965416] retval = 2097152 [D 20:01:05.965440] exiting distribute returned less than expected [D 20:01:05.965464] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:05.965491] *****RETURN***FROM***PROCESS***** [D 20:01:05.965516] end iteration [D 20:01:05.965539] done with segments or bytes [D 20:01:05.965563] of 132120576 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.965589] next loff: 132120576 next poff: 132120576 [D 20:01:05.965614] return value 2097152 [D 20:01:05.965638] finished [D 20:01:05.965662] retval = 2097152 [D 20:01:05.965685] exiting distribute returned less than expected [D 20:01:05.965710] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 132120576 fo 134217728 eof 0 [D 20:01:05.965736] ========================================================= [D 20:01:05.965761] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:05.965785] element 0: offset: 0x9e04b000, size: 2097152 [D 20:01:05.965812] openib_check_cq: completed send wr_id 15e73228. [D 20:01:05.965841] post_sr: da13:3346 bh 72 len 48 wr_id 15e73240. [D 20:01:05.965869] PINT_client_state_machine_test id 76 [D 20:01:05.965904] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:05.965929] ========================================================= [D 20:01:05.965954] PINT_process_request [D 20:01:05.965978] tiling 7 copies [D 20:01:05.966002] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.966029] lvl 0 el 0 blk 0 by 186646528 [D 20:01:05.966053] to 186646528 ta 167772160 fi 201326592 [D 20:01:05.966089] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:05.966114] basic type or contiguous data [D 20:01:05.966138] Request disp [D 20:01:05.966162] contig_offset = 186646528 contig_size = 48234496 lvl_flag = 1 [D 20:01:05.966188] calling distribute [D 20:01:05.966215] PINT_distribute [D 20:01:05.966242] of 186646528 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.966273] begin iteration loff: 186646528 [D 20:01:05.966301] segment exceeds byte limit [D 20:01:05.966328] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:05.966353] **********CALL***PROCESS********* [D 20:01:05.966378] segment of 18874368 sz 2097152 [D 20:01:05.966402] PINT_process_request [D 20:01:05.966425] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.966452] lvl 0 el 0 blk 0 by 18874368 [D 20:01:05.966477] to 18874368 ta 18874368 fi 20971520 [D 20:01:05.966502] basic type or contiguous data [D 20:01:05.966526] Request disp [D 20:01:05.966549] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:05.966574] process a segment [D 20:01:05.966598] of 18874368 sz 2097152 [D 20:01:05.966622] add a segment 0 [D 20:01:05.966646] retval = 2097152 [D 20:01:05.966670] exiting distribute returned less than expected [D 20:01:05.966694] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:05.966720] *****RETURN***FROM***PROCESS***** [D 20:01:05.966745] end iteration [D 20:01:05.966769] done with segments or bytes [D 20:01:05.966793] of 188743680 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.966818] next loff: 188743680 next poff: 188743680 [D 20:01:05.966843] return value 2097152 [D 20:01:05.966868] finished [D 20:01:05.966891] retval = 2097152 [D 20:01:05.966916] exiting distribute returned less than expected [D 20:01:05.966939] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 188743680 fo 201326592 eof 0 [D 20:01:05.966965] ========================================================= [D 20:01:05.966990] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:05.967015] element 0: offset: 0x99647000, size: 2097152 [D 20:01:05.967041] openib_check_cq: completed send wr_id 15e73240. [D 20:01:05.967067] openib_check_cq: completed recv wr_id 15e738f0. [D 20:01:05.967094] openib_post_rr: da13:3346 bh 43 wr_id 15e738f0. [D 20:01:05.967123] post_sr: da13:3346 bh 73 len 48 wr_id 15e73258. [D 20:01:05.967158] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:05.967184] ========================================================= [D 20:01:05.967208] PINT_process_request [D 20:01:05.967232] tiling 8 copies [D 20:01:05.967256] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.967283] lvl 0 el 0 blk 0 by 220200960 [D 20:01:05.967308] to 220200960 ta 201326592 fi 234881024 [D 20:01:05.967332] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:05.967357] basic type or contiguous data [D 20:01:05.967381] Request disp [D 20:01:05.967405] contig_offset = 220200960 contig_size = 48234496 lvl_flag = 1 [D 20:01:05.967430] calling distribute [D 20:01:05.967453] PINT_distribute [D 20:01:05.967477] of 220200960 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.967504] begin iteration loff: 220200960 [D 20:01:05.967529] segment exceeds byte limit [D 20:01:05.967553] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:05.967577] **********CALL***PROCESS********* [D 20:01:05.967601] segment of 18874368 sz 2097152 [D 20:01:05.967626] PINT_process_request [D 20:01:05.967650] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.967676] lvl 0 el 0 blk 0 by 18874368 [D 20:01:05.967700] to 18874368 ta 18874368 fi 20971520 [D 20:01:05.967725] basic type or contiguous data [D 20:01:05.967749] Request disp [D 20:01:05.967773] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:05.967798] process a segment [D 20:01:05.967826] of 18874368 sz 2097152 [D 20:01:05.967851] add a segment 0 [D 20:01:05.967876] retval = 2097152 [D 20:01:05.967900] exiting distribute returned less than expected [D 20:01:05.967924] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:05.967950] *****RETURN***FROM***PROCESS***** [D 20:01:05.967974] end iteration [D 20:01:05.967998] done with segments or bytes [D 20:01:05.968022] of 222298112 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.968048] next loff: 222298112 next poff: 222298112 [D 20:01:05.968072] return value 2097152 [D 20:01:05.968097] finished [D 20:01:05.968121] retval = 2097152 [D 20:01:05.968145] exiting distribute returned less than expected [D 20:01:05.968169] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 222298112 fo 234881024 eof 0 [D 20:01:05.968194] ========================================================= [D 20:01:05.968221] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:05.968247] element 0: offset: 0x97645000, size: 2097152 [D 20:01:05.968274] openib_check_cq: completed send wr_id 15e73258. [D 20:01:05.968303] post_sr: da13:3346 bh 74 len 48 wr_id 15e73270. [D 20:01:05.968376] openib_check_cq: completed send wr_id 15e73270. [D 20:01:05.968401] openib_check_cq: completed recv wr_id 15e73908. [D 20:01:05.968428] openib_post_rr: da13:3346 bh 44 wr_id 15e73908. [D 20:01:05.968461] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:05.968486] ========================================================= [D 20:01:05.968511] PINT_process_request [D 20:01:05.968535] tiling 6 copies [D 20:01:05.968560] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.968585] lvl 0 el 0 blk 0 by 155189248 [D 20:01:05.968610] to 155189248 ta 134217728 fi 167772160 [D 20:01:05.968635] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:05.968660] basic type or contiguous data [D 20:01:05.968684] Request disp [D 20:01:05.968708] contig_offset = 155189248 contig_size = 46137344 lvl_flag = 1 [D 20:01:05.968733] calling distribute [D 20:01:05.968757] PINT_distribute [D 20:01:05.968781] of 155189248 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.968808] begin iteration loff: 155189248 [D 20:01:05.968832] segment exceeds byte limit [D 20:01:05.968856] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:05.968881] **********CALL***PROCESS********* [D 20:01:05.968905] segment of 20971520 sz 2097152 [D 20:01:05.968930] PINT_process_request [D 20:01:05.968954] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.968979] lvl 0 el 0 blk 0 by 20971520 [D 20:01:05.969005] to 20971520 ta 20971520 fi 23068672 [D 20:01:05.969030] basic type or contiguous data [D 20:01:05.969054] Request disp [D 20:01:05.969078] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:05.969102] process a segment [D 20:01:05.969126] of 20971520 sz 2097152 [D 20:01:05.969151] add a segment 0 [D 20:01:05.969175] retval = 2097152 [D 20:01:05.969199] exiting distribute returned less than expected [D 20:01:05.969223] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:05.969249] *****RETURN***FROM***PROCESS***** [D 20:01:05.969273] end iteration [D 20:01:05.969297] done with segments or bytes [D 20:01:05.969321] of 157286400 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.969347] next loff: 157286400 next poff: 157286400 [D 20:01:05.969371] return value 2097152 [D 20:01:05.969395] finished [D 20:01:05.969419] retval = 2097152 [D 20:01:05.969443] exiting distribute returned less than expected [D 20:01:05.969467] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 157286400 fo 167772160 eof 0 [D 20:01:05.969493] ========================================================= [D 20:01:05.969517] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:05.969543] element 0: offset: 0x9b849000, size: 2097152 [D 20:01:05.969574] post_sr: da13:3346 bh 75 len 48 wr_id 15e73288. [D 20:01:05.969646] openib_check_cq: completed send wr_id 15e73288. [D 20:01:05.972833] openib_check_cq: completed recv wr_id 15e73920. [D 20:01:05.972861] openib_post_rr: da13:3346 bh 45 wr_id 15e73920. [D 20:01:05.972893] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:05.972919] ========================================================= [D 20:01:05.972944] PINT_process_request [D 20:01:05.972968] tiling 9 copies [D 20:01:05.972991] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.973017] lvl 0 el 0 blk 0 by 251658240 [D 20:01:05.973043] to 251658240 ta 234881024 fi 268435456 [D 20:01:05.973068] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:05.973092] basic type or contiguous data [D 20:01:05.973115] Request disp [D 20:01:05.973140] contig_offset = 251658240 contig_size = 50331648 lvl_flag = 1 [D 20:01:05.973165] calling distribute [D 20:01:05.973189] PINT_distribute [D 20:01:05.973213] of 251658240 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.973238] begin iteration loff: 251658240 [D 20:01:05.973264] segment exceeds byte limit [D 20:01:05.973288] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:05.973313] **********CALL***PROCESS********* [D 20:01:05.973337] segment of 16777216 sz 2097152 [D 20:01:05.973361] PINT_process_request [D 20:01:05.973385] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.973412] lvl 0 el 0 blk 0 by 16777216 [D 20:01:05.973439] to 16777216 ta 16777216 fi 18874368 [D 20:01:05.973464] basic type or contiguous data [D 20:01:05.973487] Request disp [D 20:01:05.973511] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:05.973536] process a segment [D 20:01:05.973560] of 16777216 sz 2097152 [D 20:01:05.973585] add a segment 0 [D 20:01:05.973608] retval = 2097152 [D 20:01:05.973632] exiting distribute returned less than expected [D 20:01:05.973657] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:05.973683] *****RETURN***FROM***PROCESS***** [D 20:01:05.973708] end iteration [D 20:01:05.973731] done with segments or bytes [D 20:01:05.973755] of 253755392 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.973781] next loff: 253755392 next poff: 253755392 [D 20:01:05.973806] return value 2097152 [D 20:01:05.973830] finished [D 20:01:05.973854] retval = 2097152 [D 20:01:05.973878] exiting distribute returned less than expected [D 20:01:05.973902] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 253755392 fo 268435456 eof 0 [D 20:01:05.973928] ========================================================= [D 20:01:05.973953] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:05.973978] element 0: offset: 0x95443000, size: 2097152 [D 20:01:05.974007] post_sr: da13:3346 bh 76 len 48 wr_id 15e732a0. [D 20:01:05.974079] openib_check_cq: completed send wr_id 15e732a0. [D 20:01:05.975900] PINT_client_state_machine_test id 76 [D 20:01:05.977278] openib_check_cq: completed recv wr_id 15e73938. [D 20:01:05.977305] openib_post_rr: da13:3346 bh 46 wr_id 15e73938. [D 20:01:05.977337] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:05.977363] ========================================================= [D 20:01:05.977388] PINT_process_request [D 20:01:05.977412] tiling 10 copies [D 20:01:05.977436] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.977462] lvl 0 el 0 blk 0 by 285212672 [D 20:01:05.977487] to 285212672 ta 268435456 fi 301989888 [D 20:01:05.977512] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:05.977537] basic type or contiguous data [D 20:01:05.977560] Request disp [D 20:01:05.977584] contig_offset = 285212672 contig_size = 50331648 lvl_flag = 1 [D 20:01:05.977610] calling distribute [D 20:01:05.977634] PINT_distribute [D 20:01:05.977658] of 285212672 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.977686] begin iteration loff: 285212672 [D 20:01:05.977712] segment exceeds byte limit [D 20:01:05.977736] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:05.977761] **********CALL***PROCESS********* [D 20:01:05.977786] segment of 16777216 sz 2097152 [D 20:01:05.977810] PINT_process_request [D 20:01:05.977834] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.977860] lvl 0 el 0 blk 0 by 16777216 [D 20:01:05.977885] to 16777216 ta 16777216 fi 18874368 [D 20:01:05.977910] basic type or contiguous data [D 20:01:05.977933] Request disp [D 20:01:05.977957] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:05.977983] process a segment [D 20:01:05.978007] of 16777216 sz 2097152 [D 20:01:05.978031] add a segment 0 [D 20:01:05.978055] retval = 2097152 [D 20:01:05.978079] exiting distribute returned less than expected [D 20:01:05.978103] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:05.978129] *****RETURN***FROM***PROCESS***** [D 20:01:05.978154] end iteration [D 20:01:05.978178] done with segments or bytes [D 20:01:05.978200] of 287309824 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.978232] next loff: 287309824 next poff: 287309824 [D 20:01:05.978260] return value 2097152 [D 20:01:05.978287] finished [D 20:01:05.978315] retval = 2097152 [D 20:01:05.978340] exiting distribute returned less than expected [D 20:01:05.978364] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 287309824 fo 301989888 eof 0 [D 20:01:05.978390] ========================================================= [D 20:01:05.978415] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:05.978440] element 0: offset: 0x93441000, size: 2097152 [D 20:01:05.978469] post_sr: da13:3346 bh 77 len 48 wr_id 15e732b8. [D 20:01:05.978540] openib_check_cq: completed send wr_id 15e732b8. [D 20:01:05.981724] openib_check_cq: completed recv wr_id 15e73950. [D 20:01:05.981751] openib_post_rr: da13:3346 bh 47 wr_id 15e73950. [D 20:01:05.981778] openib_check_cq: completed recv wr_id 15e73968. [D 20:01:05.981805] openib_post_rr: da13:3346 bh 48 wr_id 15e73968. [D 20:01:05.981837] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:05.981863] ========================================================= [D 20:01:05.981887] PINT_process_request [D 20:01:05.981911] tiling 12 copies [D 20:01:05.981936] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.981963] lvl 0 el 0 blk 0 by 350224384 [D 20:01:05.981987] to 350224384 ta 335544320 fi 369098752 [D 20:01:05.982012] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:05.982037] basic type or contiguous data [D 20:01:05.982061] Request disp [D 20:01:05.982085] contig_offset = 350224384 contig_size = 52428800 lvl_flag = 1 [D 20:01:05.982110] calling distribute [D 20:01:05.982134] PINT_distribute [D 20:01:05.982158] of 350224384 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.982185] begin iteration loff: 350224384 [D 20:01:05.982213] segment exceeds byte limit [D 20:01:05.982241] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:05.982269] **********CALL***PROCESS********* [D 20:01:05.982296] segment of 14680064 sz 2097152 [D 20:01:05.982325] PINT_process_request [D 20:01:05.982349] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.982376] lvl 0 el 0 blk 0 by 14680064 [D 20:01:05.982400] to 14680064 ta 14680064 fi 16777216 [D 20:01:05.982425] basic type or contiguous data [D 20:01:05.982449] Request disp [D 20:01:05.982473] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:05.982498] process a segment [D 20:01:05.982522] of 14680064 sz 2097152 [D 20:01:05.982546] add a segment 0 [D 20:01:05.982570] retval = 2097152 [D 20:01:05.982595] exiting distribute returned less than expected [D 20:01:05.982619] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:05.982647] *****RETURN***FROM***PROCESS***** [D 20:01:05.982672] end iteration [D 20:01:05.982696] done with segments or bytes [D 20:01:05.982720] of 352321536 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.982746] next loff: 352321536 next poff: 352321536 [D 20:01:05.982770] return value 2097152 [D 20:01:05.982795] finished [D 20:01:05.982819] retval = 2097152 [D 20:01:05.982843] exiting distribute returned less than expected [D 20:01:05.982867] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 352321536 fo 369098752 eof 0 [D 20:01:05.982893] ========================================================= [D 20:01:05.982917] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:05.982943] element 0: offset: 0x8f23d000, size: 2097152 [D 20:01:05.982972] post_sr: da13:3346 bh 78 len 48 wr_id 15e732d0. [D 20:01:05.983042] openib_check_cq: completed send wr_id 15e732d0. [D 20:01:05.983956] openib_check_cq: completed recv wr_id 15e73980. [D 20:01:05.983983] openib_post_rr: da13:3346 bh 49 wr_id 15e73980. [D 20:01:05.984009] openib_check_cq: completed recv wr_id 15e73998. [D 20:01:05.984037] openib_post_rr: da13:3346 bh 50 wr_id 15e73998. [D 20:01:05.984064] openib_check_cq: completed recv wr_id 15e739b0. [D 20:01:05.984089] openib_post_rr: da13:3346 bh 51 wr_id 15e739b0. [D 20:01:05.984121] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:05.984147] ========================================================= [D 20:01:05.984171] PINT_process_request [D 20:01:05.984195] tiling 11 copies [D 20:01:05.984222] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.984249] lvl 0 el 0 blk 0 by 316669952 [D 20:01:05.984273] to 316669952 ta 301989888 fi 335544320 [D 20:01:05.984298] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:05.984323] basic type or contiguous data [D 20:01:05.984347] Request disp [D 20:01:05.984371] contig_offset = 316669952 contig_size = 52428800 lvl_flag = 1 [D 20:01:05.984396] calling distribute [D 20:01:05.984420] PINT_distribute [D 20:01:05.984444] of 316669952 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:05.984471] begin iteration loff: 316669952 [D 20:01:05.984496] segment exceeds byte limit [D 20:01:05.984519] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:05.984544] **********CALL***PROCESS********* [D 20:01:05.984568] segment of 14680064 sz 2097152 [D 20:01:05.984593] PINT_process_request [D 20:01:05.984617] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.984642] lvl 0 el 0 blk 0 by 14680064 [D 20:01:05.984667] to 14680064 ta 14680064 fi 16777216 [D 20:01:05.984692] basic type or contiguous data [D 20:01:05.984716] Request disp [D 20:01:05.984740] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:05.984765] process a segment [D 20:01:05.984788] of 14680064 sz 2097152 [D 20:01:05.984813] add a segment 0 [D 20:01:05.984837] retval = 2097152 [D 20:01:05.984861] exiting distribute returned less than expected [D 20:01:05.984885] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:05.984910] *****RETURN***FROM***PROCESS***** [D 20:01:05.984935] end iteration [D 20:01:05.984959] done with segments or bytes [D 20:01:05.984983] of 318767104 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.985009] next loff: 318767104 next poff: 318767104 [D 20:01:05.985033] return value 2097152 [D 20:01:05.985057] finished [D 20:01:05.985081] retval = 2097152 [D 20:01:05.985105] exiting distribute returned less than expected [D 20:01:05.985129] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 318767104 fo 335544320 eof 0 [D 20:01:05.985155] ========================================================= [D 20:01:05.985179] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:05.985204] element 0: offset: 0x9123f000, size: 2097152 [D 20:01:05.985237] post_sr: da13:3346 bh 79 len 48 wr_id 15e732e8. [D 20:01:05.985263] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:05.985289] ========================================================= [D 20:01:05.985314] PINT_process_request [D 20:01:05.985338] tiling 13 copies [D 20:01:05.985363] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.985388] lvl 0 el 0 blk 0 by 383778816 [D 20:01:05.985413] to 383778816 ta 369098752 fi 402653184 [D 20:01:05.985438] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:05.985463] basic type or contiguous data [D 20:01:05.985486] Request disp [D 20:01:05.985510] contig_offset = 383778816 contig_size = 52428800 lvl_flag = 1 [D 20:01:05.985536] calling distribute [D 20:01:05.985560] PINT_distribute [D 20:01:05.985584] of 383778816 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.985610] begin iteration loff: 383778816 [D 20:01:05.985635] segment exceeds byte limit [D 20:01:05.985659] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:05.985685] **********CALL***PROCESS********* [D 20:01:05.985709] segment of 14680064 sz 2097152 [D 20:01:05.985733] PINT_process_request [D 20:01:05.985757] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.985783] lvl 0 el 0 blk 0 by 14680064 [D 20:01:05.985808] to 14680064 ta 14680064 fi 16777216 [D 20:01:05.985833] basic type or contiguous data [D 20:01:05.985857] Request disp [D 20:01:05.985880] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:05.985905] process a segment [D 20:01:05.985929] of 14680064 sz 2097152 [D 20:01:05.985954] add a segment 0 [D 20:01:05.985978] retval = 2097152 [D 20:01:05.986001] exiting distribute returned less than expected [D 20:01:05.986025] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:05.986052] *****RETURN***FROM***PROCESS***** [D 20:01:05.986076] end iteration [D 20:01:05.986100] done with segments or bytes [D 20:01:05.986124] of 385875968 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.986149] next loff: 385875968 next poff: 385875968 [D 20:01:05.986174] return value 2097152 [D 20:01:05.986199] finished [D 20:01:05.986227] retval = 2097152 [D 20:01:05.986254] exiting distribute returned less than expected [D 20:01:05.986282] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 385875968 fo 402653184 eof 0 [D 20:01:05.986312] ========================================================= [D 20:01:05.986338] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:05.986364] element 0: offset: 0x8d23b000, size: 2097152 [D 20:01:05.986391] openib_check_cq: completed send wr_id 15e732e8. [D 20:01:05.986416] openib_check_cq: completed recv wr_id 15e739c8. [D 20:01:05.986442] openib_post_rr: da13:3346 bh 52 wr_id 15e739c8. [D 20:01:05.986469] openib_check_cq: completed recv wr_id 15e739e0. [D 20:01:05.986495] openib_post_rr: da13:3346 bh 53 wr_id 15e739e0. [D 20:01:05.986521] openib_check_cq: completed recv wr_id 15e739f8. [D 20:01:05.986549] openib_post_rr: da13:3346 bh 54 wr_id 15e739f8. [D 20:01:05.986576] openib_check_cq: completed recv wr_id 15e73a10. [D 20:01:05.986603] openib_post_rr: da13:3346 bh 55 wr_id 15e73a10. [D 20:01:05.986630] openib_check_cq: completed recv wr_id 15e73a28. [D 20:01:05.986658] openib_post_rr: da13:3346 bh 56 wr_id 15e73a28. [D 20:01:05.986684] openib_check_cq: completed recv wr_id 15e73a40. [D 20:01:05.986712] openib_post_rr: da13:3346 bh 57 wr_id 15e73a40. [D 20:01:05.986739] openib_check_cq: completed recv wr_id 15e73a58. [D 20:01:05.986766] openib_post_rr: da13:3346 bh 58 wr_id 15e73a58. [D 20:01:05.986792] openib_check_cq: completed recv wr_id 15e73a70. [D 20:01:05.986820] openib_post_rr: da13:3346 bh 59 wr_id 15e73a70. [D 20:01:05.986846] openib_check_cq: completed recv wr_id 15e73a88. [D 20:01:05.986874] openib_post_rr: da13:3346 bh 60 wr_id 15e73a88. [D 20:01:05.986901] openib_check_cq: completed recv wr_id 15e73aa0. [D 20:01:05.986931] openib_post_rr: da13:3346 bh 61 wr_id 15e73aa0. [D 20:01:05.986958] openib_check_cq: completed recv wr_id 15e73ab8. [D 20:01:05.986986] openib_post_rr: da13:3346 bh 62 wr_id 15e73ab8. [D 20:01:05.987012] openib_check_cq: completed recv wr_id 15e73ad0. [D 20:01:05.987040] openib_post_rr: da13:3346 bh 63 wr_id 15e73ad0. [D 20:01:05.987067] openib_check_cq: completed recv wr_id 15e73ae8. [D 20:01:05.987093] openib_post_rr: da13:3346 bh 64 wr_id 15e73ae8. [D 20:01:05.987120] openib_check_cq: completed recv wr_id 15e73b00. [D 20:01:05.987148] openib_post_rr: da13:3346 bh 65 wr_id 15e73b00. [D 20:01:05.987174] openib_check_cq: completed recv wr_id 15e73b18. [D 20:01:05.987201] openib_post_rr: da13:3346 bh 66 wr_id 15e73b18. [D 20:01:05.987228] openib_check_cq: completed recv wr_id 15e73b30. [D 20:01:05.987256] openib_post_rr: da13:3346 bh 67 wr_id 15e73b30. [D 20:01:05.987285] post_sr: da13:3346 bh 80 len 48 wr_id 15e73300. [D 20:01:05.987313] PINT_client_state_machine_test id 76 [D 20:01:05.987348] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:05.987374] ========================================================= [D 20:01:05.987399] PINT_process_request [D 20:01:05.987422] tiling 14 copies [D 20:01:05.987447] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.987473] lvl 0 el 0 blk 0 by 417333248 [D 20:01:05.987498] to 417333248 ta 402653184 fi 436207616 [D 20:01:05.987524] mto 14680064 mta 12582912 mfi 14680064 [D 20:01:05.987548] basic type or contiguous data [D 20:01:05.987572] Request disp [D 20:01:05.987596] contig_offset = 417333248 contig_size = 52428800 lvl_flag = 1 [D 20:01:05.987621] calling distribute [D 20:01:05.987646] PINT_distribute [D 20:01:05.987669] of 417333248 sz 18874368 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:05.987696] begin iteration loff: 417333248 [D 20:01:05.987721] segment exceeds byte limit [D 20:01:05.987745] client lstof 14680064 diff 0 sgof 14680064 [D 20:01:05.987770] **********CALL***PROCESS********* [D 20:01:05.987793] segment of 14680064 sz 2097152 [D 20:01:05.987818] PINT_process_request [D 20:01:05.987842] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.987868] lvl 0 el 0 blk 0 by 14680064 [D 20:01:05.987893] to 14680064 ta 14680064 fi 16777216 [D 20:01:05.987917] basic type or contiguous data [D 20:01:05.987941] Request disp [D 20:01:05.987965] contig_offset = 14680064 contig_size = 18874368 lvl_flag = 1 [D 20:01:05.987991] process a segment [D 20:01:05.988015] of 14680064 sz 2097152 [D 20:01:05.988039] add a segment 0 [D 20:01:05.988062] retval = 2097152 [D 20:01:05.988086] exiting distribute returned less than expected [D 20:01:05.988111] done sg 1 sm 64 by 2097152 bm 2097152 ta 14680064 to 16777216 fo 16777216 eof 0 [D 20:01:05.988138] *****RETURN***FROM***PROCESS***** [D 20:01:05.988162] end iteration [D 20:01:05.988185] done with segments or bytes [D 20:01:05.988212] of 419430400 sz 16777216 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.988238] next loff: 419430400 next poff: 419430400 [D 20:01:05.988263] return value 2097152 [D 20:01:05.988287] finished [D 20:01:05.988311] retval = 2097152 [D 20:01:05.988335] exiting distribute returned less than expected [D 20:01:05.988359] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 419430400 fo 436207616 eof 0 [D 20:01:05.988386] ========================================================= [D 20:01:05.988409] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:05.988435] element 0: offset: 0x8b239000, size: 2097152 [D 20:01:05.988461] openib_check_cq: completed send wr_id 15e73300. [D 20:01:05.988487] openib_check_cq: completed recv wr_id 15e73b48. [D 20:01:05.988513] openib_post_rr: da13:3346 bh 68 wr_id 15e73b48. [D 20:01:05.988540] openib_check_cq: completed recv wr_id 15e73b60. [D 20:01:05.988568] openib_post_rr: da13:3346 bh 69 wr_id 15e73b60. [D 20:01:05.988594] openib_check_cq: completed recv wr_id 15e73b78. [D 20:01:05.988623] openib_post_rr: da13:3346 bh 70 wr_id 15e73b78. [D 20:01:05.988653] post_sr: da13:3346 bh 81 len 48 wr_id 15e73318. [D 20:01:05.988680] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:05.988706] ========================================================= [D 20:01:05.988730] PINT_process_request [D 20:01:05.988754] tiling 5 copies [D 20:01:05.988779] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.988804] lvl 0 el 0 blk 0 by 132120576 [D 20:01:05.988829] to 132120576 ta 100663296 fi 134217728 [D 20:01:05.988855] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:05.988880] basic type or contiguous data [D 20:01:05.988903] Request disp [D 20:01:05.988927] contig_offset = 132120576 contig_size = 35651584 lvl_flag = 1 [D 20:01:05.988952] calling distribute [D 20:01:05.988977] PINT_distribute [D 20:01:05.989001] of 132120576 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 442499072 exfl 0 [D 20:01:05.989027] begin iteration loff: 132120576 [D 20:01:05.989052] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:05.989077] **********CALL***PROCESS********* [D 20:01:05.989101] segment of 31457280 sz 2097152 [D 20:01:05.989126] PINT_process_request [D 20:01:05.989150] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.989175] lvl 0 el 0 blk 0 by 31457280 [D 20:01:05.989201] to 31457280 ta 31457280 fi 33554432 [D 20:01:05.989226] basic type or contiguous data [D 20:01:05.989250] Request disp [D 20:01:05.989274] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:05.989299] process a segment [D 20:01:05.989323] of 31457280 sz 2097152 [D 20:01:05.989347] add a segment 0 [D 20:01:05.989371] retval = 2097152 [D 20:01:05.989396] return from level -1 [D 20:01:05.989419] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:05.989445] *****RETURN***FROM***PROCESS***** [D 20:01:05.989470] end iteration [D 20:01:05.989494] done with segments or bytes [D 20:01:05.989518] of 134217728 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.989542] next loff: 134217728 next poff: 134217728 [D 20:01:05.989567] (return value) 2097152 [D 20:01:05.989592] finished [D 20:01:05.989616] retval = 2097152 [D 20:01:05.989640] exiting distribute returned less than expected [D 20:01:05.989664] done sg 1 sm 64 by 2097152 bm 2097152 ta 100663296 to 134217728 fo 134217728 eof 0 [D 20:01:05.989689] ========================================================= [D 20:01:05.989714] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 26 [D 20:01:05.989739] element 0: offset: 0x9e24b000, size: 2097152 [D 20:01:05.989766] openib_check_cq: completed send wr_id 15e73318. [D 20:01:05.989794] post_sr: da13:3346 bh 82 len 48 wr_id 15e73330. [D 20:01:05.989829] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:05.989855] ========================================================= [D 20:01:05.989880] PINT_process_request [D 20:01:05.989904] tiling 7 copies [D 20:01:05.989928] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.989954] lvl 0 el 0 blk 0 by 188743680 [D 20:01:05.989980] to 188743680 ta 167772160 fi 201326592 [D 20:01:05.990005] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:05.990030] basic type or contiguous data [D 20:01:05.990054] Request disp [D 20:01:05.990077] contig_offset = 188743680 contig_size = 46137344 lvl_flag = 1 [D 20:01:05.990103] calling distribute [D 20:01:05.990127] PINT_distribute [D 20:01:05.990151] of 188743680 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.990178] begin iteration loff: 188743680 [D 20:01:05.990202] segment exceeds byte limit [D 20:01:05.990231] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:05.990260] **********CALL***PROCESS********* [D 20:01:05.990287] segment of 20971520 sz 2097152 [D 20:01:05.990316] PINT_process_request [D 20:01:05.990341] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.990369] lvl 0 el 0 blk 0 by 20971520 [D 20:01:05.990394] to 20971520 ta 20971520 fi 23068672 [D 20:01:05.990419] basic type or contiguous data [D 20:01:05.990444] Request disp [D 20:01:05.990468] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:05.990492] process a segment [D 20:01:05.990516] of 20971520 sz 2097152 [D 20:01:05.990541] add a segment 0 [D 20:01:05.990565] retval = 2097152 [D 20:01:05.990589] exiting distribute returned less than expected [D 20:01:05.990612] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:05.990639] *****RETURN***FROM***PROCESS***** [D 20:01:05.990664] end iteration [D 20:01:05.990688] done with segments or bytes [D 20:01:05.990712] of 190840832 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.990737] next loff: 190840832 next poff: 190840832 [D 20:01:05.990762] return value 2097152 [D 20:01:05.990787] finished [D 20:01:05.990811] retval = 2097152 [D 20:01:05.990835] exiting distribute returned less than expected [D 20:01:05.990859] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 190840832 fo 201326592 eof 0 [D 20:01:05.990885] ========================================================= [D 20:01:05.990909] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:05.990935] element 0: offset: 0x99847000, size: 2097152 [D 20:01:05.990962] openib_check_cq: completed send wr_id 15e73330. [D 20:01:05.990987] openib_check_cq: completed recv wr_id 15e73b90. [D 20:01:05.991013] openib_post_rr: da13:3346 bh 71 wr_id 15e73b90. [D 20:01:05.991044] post_sr: da13:3346 bh 83 len 48 wr_id 15e73348. [D 20:01:05.991070] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:05.991096] ========================================================= [D 20:01:05.991120] PINT_process_request [D 20:01:05.991144] tiling 8 copies [D 20:01:05.991168] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.991195] lvl 0 el 0 blk 0 by 222298112 [D 20:01:05.991220] to 222298112 ta 201326592 fi 234881024 [D 20:01:05.991245] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:05.991270] basic type or contiguous data [D 20:01:05.991293] Request disp [D 20:01:05.991317] contig_offset = 222298112 contig_size = 46137344 lvl_flag = 1 [D 20:01:05.991343] calling distribute [D 20:01:05.991367] PINT_distribute [D 20:01:05.991391] of 222298112 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.991417] begin iteration loff: 222298112 [D 20:01:05.991442] segment exceeds byte limit [D 20:01:05.991466] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:05.991491] **********CALL***PROCESS********* [D 20:01:05.991515] segment of 20971520 sz 2097152 [D 20:01:05.991539] PINT_process_request [D 20:01:05.991563] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.991589] lvl 0 el 0 blk 0 by 20971520 [D 20:01:05.991614] to 20971520 ta 20971520 fi 23068672 [D 20:01:05.991640] basic type or contiguous data [D 20:01:05.991663] Request disp [D 20:01:05.991687] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:05.991712] process a segment [D 20:01:05.991736] of 20971520 sz 2097152 [D 20:01:05.991760] add a segment 0 [D 20:01:05.991784] retval = 2097152 [D 20:01:05.991808] exiting distribute returned less than expected [D 20:01:05.991832] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:05.991858] *****RETURN***FROM***PROCESS***** [D 20:01:05.991883] end iteration [D 20:01:05.991907] done with segments or bytes [D 20:01:05.991930] of 224395264 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.991956] next loff: 224395264 next poff: 224395264 [D 20:01:05.991981] return value 2097152 [D 20:01:05.992005] finished [D 20:01:05.992029] retval = 2097152 [D 20:01:05.992053] exiting distribute returned less than expected [D 20:01:05.992079] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 224395264 fo 234881024 eof 0 [D 20:01:05.992106] ========================================================= [D 20:01:05.992131] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:05.992156] element 0: offset: 0x97845000, size: 2097152 [D 20:01:05.992182] openib_check_cq: completed send wr_id 15e73348. [D 20:01:05.992214] post_sr: da13:3346 bh 84 len 48 wr_id 15e73360. [D 20:01:05.992249] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:05.992275] ========================================================= [D 20:01:05.992300] PINT_process_request [D 20:01:05.992323] tiling 6 copies [D 20:01:05.992347] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.992374] lvl 0 el 0 blk 0 by 157286400 [D 20:01:05.992399] to 157286400 ta 134217728 fi 167772160 [D 20:01:05.992424] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:05.992449] basic type or contiguous data [D 20:01:05.992473] Request disp [D 20:01:05.992497] contig_offset = 157286400 contig_size = 44040192 lvl_flag = 1 [D 20:01:05.992523] calling distribute [D 20:01:05.992547] PINT_distribute [D 20:01:05.992570] of 157286400 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:05.992597] begin iteration loff: 157286400 [D 20:01:05.992622] segment exceeds byte limit [D 20:01:05.992646] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:05.992671] **********CALL***PROCESS********* [D 20:01:05.992695] segment of 23068672 sz 2097152 [D 20:01:05.992720] PINT_process_request [D 20:01:05.992744] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:05.992770] lvl 0 el 0 blk 0 by 23068672 [D 20:01:05.992794] to 23068672 ta 23068672 fi 25165824 [D 20:01:05.992819] basic type or contiguous data [D 20:01:05.992843] Request disp [D 20:01:05.992867] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:05.992892] process a segment [D 20:01:05.992916] of 23068672 sz 2097152 [D 20:01:05.992940] add a segment 0 [D 20:01:05.992964] retval = 2097152 [D 20:01:05.992988] exiting distribute returned less than expected [D 20:01:05.993012] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:05.993038] *****RETURN***FROM***PROCESS***** [D 20:01:05.993062] end iteration [D 20:01:05.993086] done with segments or bytes [D 20:01:05.993110] of 159383552 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:05.993136] next loff: 159383552 next poff: 159383552 [D 20:01:05.993161] return value 2097152 [D 20:01:05.993185] finished [D 20:01:05.993208] retval = 2097152 [D 20:01:05.993232] exiting distribute returned less than expected [D 20:01:05.993256] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 159383552 fo 167772160 eof 0 [D 20:01:05.993283] ========================================================= [D 20:01:05.993308] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:05.993332] element 0: offset: 0x9ba49000, size: 2097152 [D 20:01:05.993359] openib_check_cq: completed send wr_id 15e73360. [D 20:01:05.993389] post_sr: da13:3346 bh 85 len 48 wr_id 15e73378. [D 20:01:05.993424] openib_check_cq: completed send wr_id 15e73378. [D 20:01:05.997344] PINT_client_state_machine_test id 76 [D 20:01:06.007376] PINT_client_state_machine_test id 76 [D 20:01:06.017408] PINT_client_state_machine_test id 76 [D 20:01:06.027441] PINT_client_state_machine_test id 76 [D 20:01:06.030580] openib_check_cq: completed recv wr_id 15e73ba8. [D 20:01:06.030610] openib_post_rr: da13:3346 bh 72 wr_id 15e73ba8. [D 20:01:06.033175] openib_check_cq: completed recv wr_id 15e73bc0. [D 20:01:06.033202] openib_post_rr: da13:3346 bh 73 wr_id 15e73bc0. [D 20:01:06.033234] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:06.033260] ========================================================= [D 20:01:06.033285] PINT_process_request [D 20:01:06.033309] tiling 9 copies [D 20:01:06.033336] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.033363] lvl 0 el 0 blk 0 by 253755392 [D 20:01:06.033388] to 253755392 ta 234881024 fi 268435456 [D 20:01:06.033412] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:06.033440] basic type or contiguous data [D 20:01:06.033464] Request disp [D 20:01:06.033488] contig_offset = 253755392 contig_size = 48234496 lvl_flag = 1 [D 20:01:06.033513] calling distribute [D 20:01:06.033537] PINT_distribute [D 20:01:06.033561] of 253755392 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.033588] begin iteration loff: 253755392 [D 20:01:06.033613] segment exceeds byte limit [D 20:01:06.033637] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:06.033661] **********CALL***PROCESS********* [D 20:01:06.033685] segment of 18874368 sz 2097152 [D 20:01:06.033710] PINT_process_request [D 20:01:06.033734] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.033760] lvl 0 el 0 blk 0 by 18874368 [D 20:01:06.033785] to 18874368 ta 18874368 fi 20971520 [D 20:01:06.033810] basic type or contiguous data [D 20:01:06.033834] Request disp [D 20:01:06.033858] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:06.033883] process a segment [D 20:01:06.033906] of 18874368 sz 2097152 [D 20:01:06.033931] add a segment 0 [D 20:01:06.033955] retval = 2097152 [D 20:01:06.033980] exiting distribute returned less than expected [D 20:01:06.034004] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:06.034029] *****RETURN***FROM***PROCESS***** [D 20:01:06.034054] end iteration [D 20:01:06.034078] done with segments or bytes [D 20:01:06.034102] of 255852544 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.034128] next loff: 255852544 next poff: 255852544 [D 20:01:06.034152] return value 2097152 [D 20:01:06.034177] finished [D 20:01:06.034201] retval = 2097152 [D 20:01:06.034230] exiting distribute returned less than expected [D 20:01:06.034257] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 255852544 fo 268435456 eof 0 [D 20:01:06.034288] ========================================================= [D 20:01:06.034317] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:06.034342] element 0: offset: 0x95643000, size: 2097152 [D 20:01:06.034372] post_sr: da13:3346 bh 86 len 48 wr_id 15e73390. [D 20:01:06.034440] openib_check_cq: completed send wr_id 15e73390. [D 20:01:06.035405] openib_check_cq: completed recv wr_id 15e73bd8. [D 20:01:06.035431] openib_post_rr: da13:3346 bh 74 wr_id 15e73bd8. [D 20:01:06.035464] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:06.035490] ========================================================= [D 20:01:06.035514] PINT_process_request [D 20:01:06.035538] tiling 10 copies [D 20:01:06.035562] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.035589] lvl 0 el 0 blk 0 by 287309824 [D 20:01:06.035614] to 287309824 ta 268435456 fi 301989888 [D 20:01:06.035638] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:06.035664] basic type or contiguous data [D 20:01:06.035688] Request disp [D 20:01:06.035712] contig_offset = 287309824 contig_size = 48234496 lvl_flag = 1 [D 20:01:06.035737] calling distribute [D 20:01:06.035760] PINT_distribute [D 20:01:06.035784] of 287309824 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.035811] begin iteration loff: 287309824 [D 20:01:06.035836] segment exceeds byte limit [D 20:01:06.035860] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:06.035884] **********CALL***PROCESS********* [D 20:01:06.035909] segment of 18874368 sz 2097152 [D 20:01:06.035933] PINT_process_request [D 20:01:06.035958] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.035984] lvl 0 el 0 blk 0 by 18874368 [D 20:01:06.036008] to 18874368 ta 18874368 fi 20971520 [D 20:01:06.036036] basic type or contiguous data [D 20:01:06.036061] Request disp [D 20:01:06.036085] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:06.036109] process a segment [D 20:01:06.036133] of 18874368 sz 2097152 [D 20:01:06.036158] add a segment 0 [D 20:01:06.036182] retval = 2097152 [D 20:01:06.036206] exiting distribute returned less than expected [D 20:01:06.036232] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:06.036259] *****RETURN***FROM***PROCESS***** [D 20:01:06.036283] end iteration [D 20:01:06.036307] done with segments or bytes [D 20:01:06.036331] of 289406976 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.036356] next loff: 289406976 next poff: 289406976 [D 20:01:06.036381] return value 2097152 [D 20:01:06.036406] finished [D 20:01:06.036430] retval = 2097152 [D 20:01:06.036454] exiting distribute returned less than expected [D 20:01:06.036478] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 289406976 fo 301989888 eof 0 [D 20:01:06.036504] ========================================================= [D 20:01:06.036529] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:06.036554] element 0: offset: 0x93641000, size: 2097152 [D 20:01:06.036583] post_sr: da13:3346 bh 87 len 48 wr_id 15e733a8. [D 20:01:06.036657] openib_check_cq: completed send wr_id 15e733a8. [D 20:01:06.037476] PINT_client_state_machine_test id 76 [D 20:01:06.039848] openib_check_cq: completed recv wr_id 15e73bf0. [D 20:01:06.039875] openib_post_rr: da13:3346 bh 75 wr_id 15e73bf0. [D 20:01:06.039908] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:06.039935] ========================================================= [D 20:01:06.039958] PINT_process_request [D 20:01:06.039983] tiling 12 copies [D 20:01:06.040008] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.040035] lvl 0 el 0 blk 0 by 352321536 [D 20:01:06.040060] to 352321536 ta 335544320 fi 369098752 [D 20:01:06.040084] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:06.040110] basic type or contiguous data [D 20:01:06.040134] Request disp [D 20:01:06.040159] contig_offset = 352321536 contig_size = 50331648 lvl_flag = 1 [D 20:01:06.040184] calling distribute [D 20:01:06.040211] PINT_distribute [D 20:01:06.040236] of 352321536 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.040263] begin iteration loff: 352321536 [D 20:01:06.040288] segment exceeds byte limit [D 20:01:06.040312] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:06.040337] **********CALL***PROCESS********* [D 20:01:06.040361] segment of 16777216 sz 2097152 [D 20:01:06.040386] PINT_process_request [D 20:01:06.040410] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.040436] lvl 0 el 0 blk 0 by 16777216 [D 20:01:06.040461] to 16777216 ta 16777216 fi 18874368 [D 20:01:06.040486] basic type or contiguous data [D 20:01:06.040511] Request disp [D 20:01:06.040534] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:06.040559] process a segment [D 20:01:06.040584] of 16777216 sz 2097152 [D 20:01:06.040608] add a segment 0 [D 20:01:06.040633] retval = 2097152 [D 20:01:06.040657] exiting distribute returned less than expected [D 20:01:06.040681] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:06.040707] *****RETURN***FROM***PROCESS***** [D 20:01:06.040732] end iteration [D 20:01:06.040756] done with segments or bytes [D 20:01:06.040780] of 354418688 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.040805] next loff: 354418688 next poff: 354418688 [D 20:01:06.040830] return value 2097152 [D 20:01:06.040855] finished [D 20:01:06.040879] retval = 2097152 [D 20:01:06.040903] exiting distribute returned less than expected [D 20:01:06.040927] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 354418688 fo 369098752 eof 0 [D 20:01:06.040953] ========================================================= [D 20:01:06.040981] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:06.041007] element 0: offset: 0x8f43d000, size: 2097152 [D 20:01:06.041036] post_sr: da13:3346 bh 88 len 48 wr_id 15e733c0. [D 20:01:06.041109] openib_check_cq: completed send wr_id 15e733c0. [D 20:01:06.046516] openib_check_cq: completed recv wr_id 15e73c08. [D 20:01:06.046544] openib_post_rr: da13:3346 bh 76 wr_id 15e73c08. [D 20:01:06.046575] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:06.046601] ========================================================= [D 20:01:06.046626] PINT_process_request [D 20:01:06.046649] tiling 11 copies [D 20:01:06.046674] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.046701] lvl 0 el 0 blk 0 by 318767104 [D 20:01:06.046726] to 318767104 ta 301989888 fi 335544320 [D 20:01:06.046751] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:06.046776] basic type or contiguous data [D 20:01:06.046800] Request disp [D 20:01:06.046824] contig_offset = 318767104 contig_size = 50331648 lvl_flag = 1 [D 20:01:06.046849] calling distribute [D 20:01:06.046874] PINT_distribute [D 20:01:06.046897] of 318767104 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.046924] begin iteration loff: 318767104 [D 20:01:06.046949] segment exceeds byte limit [D 20:01:06.046973] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:06.046998] **********CALL***PROCESS********* [D 20:01:06.047022] segment of 16777216 sz 2097152 [D 20:01:06.047047] PINT_process_request [D 20:01:06.047071] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.047097] lvl 0 el 0 blk 0 by 16777216 [D 20:01:06.047122] to 16777216 ta 16777216 fi 18874368 [D 20:01:06.047147] basic type or contiguous data [D 20:01:06.047171] Request disp [D 20:01:06.047195] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:06.047221] process a segment [D 20:01:06.047245] of 16777216 sz 2097152 [D 20:01:06.047269] add a segment 0 [D 20:01:06.047293] retval = 2097152 [D 20:01:06.047317] exiting distribute returned less than expected [D 20:01:06.047342] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:06.047369] *****RETURN***FROM***PROCESS***** [D 20:01:06.047392] end iteration [D 20:01:06.047417] done with segments or bytes [D 20:01:06.047441] of 320864256 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.047467] next loff: 320864256 next poff: 320864256 [D 20:01:06.047492] return value 2097152 [D 20:01:06.047515] finished [D 20:01:06.047539] retval = 2097152 [D 20:01:06.047564] exiting distribute returned less than expected [D 20:01:06.047588] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 320864256 fo 335544320 eof 0 [D 20:01:06.047615] ========================================================= [D 20:01:06.047638] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:06.047664] element 0: offset: 0x9143f000, size: 2097152 [D 20:01:06.047693] post_sr: da13:3346 bh 89 len 48 wr_id 15e733d8. [D 20:01:06.047721] PINT_client_state_machine_test id 76 [D 20:01:06.047762] openib_check_cq: completed send wr_id 15e733d8. [D 20:01:06.048742] openib_check_cq: completed recv wr_id 15e73c20. [D 20:01:06.048768] openib_post_rr: da13:3346 bh 77 wr_id 15e73c20. [D 20:01:06.048801] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:06.048827] ========================================================= [D 20:01:06.048851] PINT_process_request [D 20:01:06.048875] tiling 13 copies [D 20:01:06.048899] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.048925] lvl 0 el 0 blk 0 by 385875968 [D 20:01:06.048951] to 385875968 ta 369098752 fi 402653184 [D 20:01:06.048975] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:06.049000] basic type or contiguous data [D 20:01:06.049024] Request disp [D 20:01:06.049051] contig_offset = 385875968 contig_size = 50331648 lvl_flag = 1 [D 20:01:06.049077] calling distribute [D 20:01:06.049101] PINT_distribute [D 20:01:06.049125] of 385875968 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.049152] begin iteration loff: 385875968 [D 20:01:06.049177] segment exceeds byte limit [D 20:01:06.049201] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:06.049226] **********CALL***PROCESS********* [D 20:01:06.049250] segment of 16777216 sz 2097152 [D 20:01:06.049275] PINT_process_request [D 20:01:06.049300] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.049325] lvl 0 el 0 blk 0 by 16777216 [D 20:01:06.049351] to 16777216 ta 16777216 fi 18874368 [D 20:01:06.049376] basic type or contiguous data [D 20:01:06.049400] Request disp [D 20:01:06.049424] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:06.049449] process a segment [D 20:01:06.049473] of 16777216 sz 2097152 [D 20:01:06.049497] add a segment 0 [D 20:01:06.049522] retval = 2097152 [D 20:01:06.049546] exiting distribute returned less than expected [D 20:01:06.049571] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:06.049596] *****RETURN***FROM***PROCESS***** [D 20:01:06.049621] end iteration [D 20:01:06.049645] done with segments or bytes [D 20:01:06.049669] of 387973120 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.049695] next loff: 387973120 next poff: 387973120 [D 20:01:06.049719] return value 2097152 [D 20:01:06.049744] finished [D 20:01:06.049768] retval = 2097152 [D 20:01:06.049792] exiting distribute returned less than expected [D 20:01:06.049816] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 387973120 fo 402653184 eof 0 [D 20:01:06.049842] ========================================================= [D 20:01:06.049866] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:06.049892] element 0: offset: 0x8d43b000, size: 2097152 [D 20:01:06.049921] post_sr: da13:3346 bh 90 len 48 wr_id 15e733f0. [D 20:01:06.049993] openib_check_cq: completed send wr_id 15e733f0. [D 20:01:06.050970] openib_check_cq: completed recv wr_id 15e73c38. [D 20:01:06.050997] openib_post_rr: da13:3346 bh 78 wr_id 15e73c38. [D 20:01:06.051024] openib_check_cq: completed recv wr_id 15e73c50. [D 20:01:06.051052] openib_post_rr: da13:3346 bh 79 wr_id 15e73c50. [D 20:01:06.051078] openib_check_cq: completed recv wr_id 15e73c68. [D 20:01:06.051106] openib_post_rr: da13:3346 bh 80 wr_id 15e73c68. [D 20:01:06.051133] openib_check_cq: completed recv wr_id 15e73c80. [D 20:01:06.051161] openib_post_rr: da13:3346 bh 81 wr_id 15e73c80. [D 20:01:06.051187] openib_check_cq: completed recv wr_id 15e73c98. [D 20:01:06.051214] openib_post_rr: da13:3346 bh 82 wr_id 15e73c98. [D 20:01:06.051246] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:06.051273] ========================================================= [D 20:01:06.051298] PINT_process_request [D 20:01:06.051321] tiling 14 copies [D 20:01:06.051345] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.051372] lvl 0 el 0 blk 0 by 419430400 [D 20:01:06.051397] to 419430400 ta 402653184 fi 436207616 [D 20:01:06.051422] mto 16777216 mta 14680064 mfi 16777216 [D 20:01:06.051446] basic type or contiguous data [D 20:01:06.051470] Request disp [D 20:01:06.051494] contig_offset = 419430400 contig_size = 50331648 lvl_flag = 1 [D 20:01:06.051520] calling distribute [D 20:01:06.051544] PINT_distribute [D 20:01:06.051568] of 419430400 sz 16777216 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.051594] begin iteration loff: 419430400 [D 20:01:06.051619] segment exceeds byte limit [D 20:01:06.051643] client lstof 16777216 diff 0 sgof 16777216 [D 20:01:06.051668] **********CALL***PROCESS********* [D 20:01:06.051692] segment of 16777216 sz 2097152 [D 20:01:06.051716] PINT_process_request [D 20:01:06.051740] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.051770] lvl 0 el 0 blk 0 by 16777216 [D 20:01:06.051795] to 16777216 ta 16777216 fi 18874368 [D 20:01:06.051819] basic type or contiguous data [D 20:01:06.051844] Request disp [D 20:01:06.051868] contig_offset = 16777216 contig_size = 16777216 lvl_flag = 1 [D 20:01:06.051893] process a segment [D 20:01:06.051917] of 16777216 sz 2097152 [D 20:01:06.051940] add a segment 0 [D 20:01:06.051965] retval = 2097152 [D 20:01:06.051989] exiting distribute returned less than expected [D 20:01:06.052013] done sg 1 sm 64 by 2097152 bm 2097152 ta 16777216 to 18874368 fo 18874368 eof 0 [D 20:01:06.052040] *****RETURN***FROM***PROCESS***** [D 20:01:06.052063] end iteration [D 20:01:06.052087] done with segments or bytes [D 20:01:06.052111] of 421527552 sz 14680064 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.052137] next loff: 421527552 next poff: 421527552 [D 20:01:06.052162] return value 2097152 [D 20:01:06.052186] finished [D 20:01:06.052212] retval = 2097152 [D 20:01:06.052237] exiting distribute returned less than expected [D 20:01:06.052261] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 421527552 fo 436207616 eof 0 [D 20:01:06.052288] ========================================================= [D 20:01:06.052312] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:06.052337] element 0: offset: 0x8b439000, size: 2097152 [D 20:01:06.052367] post_sr: da13:3346 bh 91 len 48 wr_id 15e73408. [D 20:01:06.052393] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebbdc0. [D 20:01:06.052419] flowproto completing 0x15ebbdc0 [D 20:01:06.052446] Job flows in progress (callback time): 9 [D 20:01:06.052474] sm control op 6 [D 20:01:06.052498] [SM Entering]: (0x15ebafa8) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:06.052525] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.052550] (0x15ebb040) io_datafile_complete_operations (tag 2) [D 20:01:06.052576] matched completed flow for context 0x15ebb810 [D 20:01:06.052600] *** all operations completed (msgpairs, flows, write acks) [D 20:01:06.052625] [SM Exiting]: (0x15ebafa8) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:06.052652] [SM Entering]: (0x15ebafa8) client_io_sm:io_analyze_results (status: 2) [D 20:01:06.052678] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.052702] (0x15ebb040) io state: io_analyze_results [D 20:01:06.052727] - io_check_context_status called [D 20:01:06.052752] 33554432 bytes read from context 0x15ebb810 [D 20:01:06.052777] [1/1] running size is 33554432 [D 20:01:06.052802] [2/1] running size is 33554432 [D 20:01:06.052829] no context errors found; using: Success [D 20:01:06.052855] total bytes transferred is 33554432 [D 20:01:06.052879] [SM Exiting]: (0x15ebafa8) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:06.052906] [SM Entering]: (0x15ebafa8) client_io_sm:io_cleanup (status: 2) [D 20:01:06.052930] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.052956] (0x15ebb040) io state: io_cleanup [D 20:01:06.052982] [SM Exiting]: (0x15ebafa8) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:06.053009] client_state_machine_terminate smcb 0x15ebafa8 [D 20:01:06.053033] add smcb 0x15ebafa8 to completion list [D 20:01:06.053058] [SM frame get]: (0x15ebafa8) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 20 get_next_full_readbuf: enter userFilePos 134217728, read_ahead_offset 503316480 -- buffer 0x15e32848 curbuf 21 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 21 aio_req 0x15e74680 b->offset 134217728 b->size 33554432 [D 20:01:06.411648] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.411706] PINT_client_state_machine_test id 79 [D 20:01:06.411741] openib_check_cq: completed send wr_id 15e73408. [D 20:01:06.411780] openib_check_cq: completed recv wr_id 15e73cb0. [D 20:01:06.411810] openib_post_rr: da13:3346 bh 83 wr_id 15e73cb0. [D 20:01:06.411838] openib_check_cq: completed recv wr_id 15e73cc8. [D 20:01:06.411870] openib_post_rr: da13:3346 bh 84 wr_id 15e73cc8. [D 20:01:06.411896] openib_check_cq: completed recv wr_id 15e73ce0. [D 20:01:06.411923] openib_post_rr: da13:3346 bh 85 wr_id 15e73ce0. [D 20:01:06.411950] openib_check_cq: completed recv wr_id 15e73cf8. [D 20:01:06.411976] openib_post_rr: da13:3346 bh 86 wr_id 15e73cf8. [D 20:01:06.412002] openib_check_cq: completed recv wr_id 15e73d10. [D 20:01:06.412028] openib_post_rr: da13:3346 bh 87 wr_id 15e73d10. [D 20:01:06.412054] openib_check_cq: completed recv wr_id 15e73d28. [D 20:01:06.412081] openib_post_rr: da13:3346 bh 88 wr_id 15e73d28. [D 20:01:06.412108] openib_check_cq: completed recv wr_id 15e73d40. [D 20:01:06.412134] openib_post_rr: da13:3346 bh 89 wr_id 15e73d40. [D 20:01:06.412159] openib_check_cq: completed recv wr_id 15e73d58. [D 20:01:06.412186] openib_post_rr: da13:3346 bh 90 wr_id 15e73d58. [D 20:01:06.412214] openib_check_cq: completed recv wr_id 15e73d70. [D 20:01:06.412243] openib_post_rr: da13:3346 bh 91 wr_id 15e73d70. [D 20:01:06.412269] openib_check_cq: completed recv wr_id 15e73d88. [D 20:01:06.412298] openib_post_rr: da13:3346 bh 92 wr_id 15e73d88. [D 20:01:06.412323] openib_check_cq: completed recv wr_id 15e73da0. [D 20:01:06.412350] openib_post_rr: da13:3346 bh 93 wr_id 15e73da0. [D 20:01:06.412377] openib_check_cq: completed recv wr_id 15e73db8. [D 20:01:06.412403] openib_post_rr: da13:3346 bh 94 wr_id 15e73db8. [D 20:01:06.412428] openib_check_cq: completed recv wr_id 15e73dd0. [D 20:01:06.412455] openib_post_rr: da13:3346 bh 95 wr_id 15e73dd0. [D 20:01:06.412490] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:06.412517] ========================================================= [D 20:01:06.412542] PINT_process_request [D 20:01:06.412567] tiling 12 copies [D 20:01:06.412590] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.412617] lvl 0 el 0 blk 0 by 354418688 [D 20:01:06.412642] to 354418688 ta 335544320 fi 369098752 [D 20:01:06.412667] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:06.412693] basic type or contiguous data [D 20:01:06.412716] Request disp [D 20:01:06.412740] contig_offset = 354418688 contig_size = 48234496 lvl_flag = 1 [D 20:01:06.412766] calling distribute [D 20:01:06.412790] PINT_distribute [D 20:01:06.412814] of 354418688 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.412841] begin iteration loff: 354418688 [D 20:01:06.412866] segment exceeds byte limit [D 20:01:06.412891] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:06.412916] **********CALL***PROCESS********* [D 20:01:06.412940] segment of 18874368 sz 2097152 [D 20:01:06.412964] PINT_process_request [D 20:01:06.412988] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.413014] lvl 0 el 0 blk 0 by 18874368 [D 20:01:06.413039] to 18874368 ta 18874368 fi 20971520 [D 20:01:06.413064] basic type or contiguous data [D 20:01:06.413087] Request disp [D 20:01:06.413111] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:06.413136] process a segment [D 20:01:06.413160] of 18874368 sz 2097152 [D 20:01:06.413185] add a segment 0 [D 20:01:06.413208] retval = 2097152 [D 20:01:06.413232] exiting distribute returned less than expected [D 20:01:06.413256] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:06.413282] *****RETURN***FROM***PROCESS***** [D 20:01:06.413307] end iteration [D 20:01:06.413331] done with segments or bytes [D 20:01:06.413354] of 356515840 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.413380] next loff: 356515840 next poff: 356515840 [D 20:01:06.413407] return value 2097152 [D 20:01:06.413432] finished [D 20:01:06.413455] retval = 2097152 [D 20:01:06.413479] exiting distribute returned less than expected [D 20:01:06.413507] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 356515840 fo 369098752 eof 0 [D 20:01:06.413534] ========================================================= [D 20:01:06.413558] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:06.413584] element 0: offset: 0x8f63d000, size: 2097152 [D 20:01:06.413615] post_sr: da13:3346 bh 92 len 48 wr_id 15e73420. [D 20:01:06.413642] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:06.413668] ========================================================= [D 20:01:06.413693] PINT_process_request [D 20:01:06.413717] tiling 13 copies [D 20:01:06.413742] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.413768] lvl 0 el 0 blk 0 by 387973120 [D 20:01:06.413793] to 387973120 ta 369098752 fi 402653184 [D 20:01:06.413818] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:06.413843] basic type or contiguous data [D 20:01:06.413867] Request disp [D 20:01:06.413891] contig_offset = 387973120 contig_size = 48234496 lvl_flag = 1 [D 20:01:06.413916] calling distribute [D 20:01:06.413940] PINT_distribute [D 20:01:06.413964] of 387973120 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.413990] begin iteration loff: 387973120 [D 20:01:06.414016] segment exceeds byte limit [D 20:01:06.414040] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:06.414065] **********CALL***PROCESS********* [D 20:01:06.414089] segment of 18874368 sz 2097152 [D 20:01:06.414113] PINT_process_request [D 20:01:06.414137] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.414163] lvl 0 el 0 blk 0 by 18874368 [D 20:01:06.414188] to 18874368 ta 18874368 fi 20971520 [D 20:01:06.414216] basic type or contiguous data [D 20:01:06.414244] Request disp [D 20:01:06.414271] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:06.414299] process a segment [D 20:01:06.414327] of 18874368 sz 2097152 [D 20:01:06.414352] add a segment 0 [D 20:01:06.414376] retval = 2097152 [D 20:01:06.414400] exiting distribute returned less than expected [D 20:01:06.414424] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:06.414450] *****RETURN***FROM***PROCESS***** [D 20:01:06.414475] end iteration [D 20:01:06.414500] done with segments or bytes [D 20:01:06.414524] of 390070272 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.414549] next loff: 390070272 next poff: 390070272 [D 20:01:06.414574] return value 2097152 [D 20:01:06.414599] finished [D 20:01:06.414623] retval = 2097152 [D 20:01:06.414647] exiting distribute returned less than expected [D 20:01:06.414671] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 390070272 fo 402653184 eof 0 [D 20:01:06.414697] ========================================================= [D 20:01:06.414722] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:06.414747] element 0: offset: 0x8d63b000, size: 2097152 [D 20:01:06.414773] openib_check_cq: completed send wr_id 15e73420. [D 20:01:06.414802] post_sr: da13:3346 bh 93 len 48 wr_id 15e73438. [D 20:01:06.414830] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:06.414855] ========================================================= [D 20:01:06.414880] PINT_process_request [D 20:01:06.414904] tiling 11 copies [D 20:01:06.414928] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.414955] lvl 0 el 0 blk 0 by 320864256 [D 20:01:06.414979] to 320864256 ta 301989888 fi 335544320 [D 20:01:06.415004] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:06.415029] basic type or contiguous data [D 20:01:06.415053] Request disp [D 20:01:06.415077] contig_offset = 320864256 contig_size = 48234496 lvl_flag = 1 [D 20:01:06.415102] calling distribute [D 20:01:06.415125] PINT_distribute [D 20:01:06.415149] of 320864256 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.415179] begin iteration loff: 320864256 [D 20:01:06.415204] segment exceeds byte limit [D 20:01:06.415227] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:06.415252] **********CALL***PROCESS********* [D 20:01:06.415276] segment of 18874368 sz 2097152 [D 20:01:06.415301] PINT_process_request [D 20:01:06.415325] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.415350] lvl 0 el 0 blk 0 by 18874368 [D 20:01:06.415375] to 18874368 ta 18874368 fi 20971520 [D 20:01:06.415400] basic type or contiguous data [D 20:01:06.415424] Request disp [D 20:01:06.415448] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:06.415473] process a segment [D 20:01:06.415497] of 18874368 sz 2097152 [D 20:01:06.415521] add a segment 0 [D 20:01:06.415546] retval = 2097152 [D 20:01:06.415570] exiting distribute returned less than expected [D 20:01:06.415594] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:06.415619] *****RETURN***FROM***PROCESS***** [D 20:01:06.415644] end iteration [D 20:01:06.415668] done with segments or bytes [D 20:01:06.415692] of 322961408 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.415718] next loff: 322961408 next poff: 322961408 [D 20:01:06.415741] return value 2097152 [D 20:01:06.415766] finished [D 20:01:06.415790] retval = 2097152 [D 20:01:06.415814] exiting distribute returned less than expected [D 20:01:06.415838] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 322961408 fo 335544320 eof 0 [D 20:01:06.415864] ========================================================= [D 20:01:06.415888] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:06.415914] element 0: offset: 0x9163f000, size: 2097152 [D 20:01:06.415940] openib_check_cq: completed send wr_id 15e73438. [D 20:01:06.415968] post_sr: da13:3346 bh 94 len 48 wr_id 15e73450. [D 20:01:06.415996] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:06.416022] ========================================================= [D 20:01:06.416046] PINT_process_request [D 20:01:06.416069] tiling 14 copies [D 20:01:06.416094] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.416120] lvl 0 el 0 blk 0 by 421527552 [D 20:01:06.416145] to 421527552 ta 402653184 fi 436207616 [D 20:01:06.416170] mto 18874368 mta 16777216 mfi 18874368 [D 20:01:06.416194] basic type or contiguous data [D 20:01:06.416222] Request disp [D 20:01:06.416246] contig_offset = 421527552 contig_size = 48234496 lvl_flag = 1 [D 20:01:06.416271] calling distribute [D 20:01:06.416295] PINT_distribute [D 20:01:06.416318] of 421527552 sz 14680064 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.416345] begin iteration loff: 421527552 [D 20:01:06.416370] segment exceeds byte limit [D 20:01:06.416395] client lstof 18874368 diff 0 sgof 18874368 [D 20:01:06.416419] **********CALL***PROCESS********* [D 20:01:06.416443] segment of 18874368 sz 2097152 [D 20:01:06.416468] PINT_process_request [D 20:01:06.416492] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.416518] lvl 0 el 0 blk 0 by 18874368 [D 20:01:06.416543] to 18874368 ta 18874368 fi 20971520 [D 20:01:06.416567] basic type or contiguous data [D 20:01:06.416591] Request disp [D 20:01:06.416615] contig_offset = 18874368 contig_size = 14680064 lvl_flag = 1 [D 20:01:06.416640] process a segment [D 20:01:06.416664] of 18874368 sz 2097152 [D 20:01:06.416687] add a segment 0 [D 20:01:06.416712] retval = 2097152 [D 20:01:06.416736] exiting distribute returned less than expected [D 20:01:06.416760] done sg 1 sm 64 by 2097152 bm 2097152 ta 18874368 to 20971520 fo 20971520 eof 0 [D 20:01:06.416787] *****RETURN***FROM***PROCESS***** [D 20:01:06.416810] end iteration [D 20:01:06.416834] done with segments or bytes [D 20:01:06.416858] of 423624704 sz 12582912 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.416884] next loff: 423624704 next poff: 423624704 [D 20:01:06.416912] return value 2097152 [D 20:01:06.416936] finished [D 20:01:06.416960] retval = 2097152 [D 20:01:06.416984] exiting distribute returned less than expected [D 20:01:06.417008] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 423624704 fo 436207616 eof 0 [D 20:01:06.417035] ========================================================= [D 20:01:06.417059] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:06.417084] element 0: offset: 0x8b639000, size: 2097152 [D 20:01:06.417111] openib_check_cq: completed send wr_id 15e73450. [D 20:01:06.417140] post_sr: da13:3346 bh 95 len 48 wr_id 15e73468. [D 20:01:06.417167] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:06.417193] ========================================================= [D 20:01:06.417218] PINT_process_request [D 20:01:06.417242] tiling 9 copies [D 20:01:06.417265] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.417292] lvl 0 el 0 blk 0 by 255852544 [D 20:01:06.417317] to 255852544 ta 234881024 fi 268435456 [D 20:01:06.417342] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:06.417366] basic type or contiguous data [D 20:01:06.417390] Request disp [D 20:01:06.417414] contig_offset = 255852544 contig_size = 46137344 lvl_flag = 1 [D 20:01:06.417439] calling distribute [D 20:01:06.417463] PINT_distribute [D 20:01:06.417487] of 255852544 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.417513] begin iteration loff: 255852544 [D 20:01:06.417538] segment exceeds byte limit [D 20:01:06.417562] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:06.417588] **********CALL***PROCESS********* [D 20:01:06.417612] segment of 20971520 sz 2097152 [D 20:01:06.417635] PINT_process_request [D 20:01:06.417659] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.417686] lvl 0 el 0 blk 0 by 20971520 [D 20:01:06.417711] to 20971520 ta 20971520 fi 23068672 [D 20:01:06.417736] basic type or contiguous data [D 20:01:06.417759] Request disp [D 20:01:06.417783] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:06.417809] process a segment [D 20:01:06.417833] of 20971520 sz 2097152 [D 20:01:06.417857] add a segment 0 [D 20:01:06.417881] retval = 2097152 [D 20:01:06.417905] exiting distribute returned less than expected [D 20:01:06.417929] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:06.417955] *****RETURN***FROM***PROCESS***** [D 20:01:06.417980] end iteration [D 20:01:06.418004] done with segments or bytes [D 20:01:06.418027] of 257949696 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.418053] next loff: 257949696 next poff: 257949696 [D 20:01:06.418078] return value 2097152 [D 20:01:06.418102] finished [D 20:01:06.418126] retval = 2097152 [D 20:01:06.418149] exiting distribute returned less than expected [D 20:01:06.418173] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 257949696 fo 268435456 eof 0 [D 20:01:06.418200] ========================================================= [D 20:01:06.418229] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:06.418258] element 0: offset: 0x95843000, size: 2097152 [D 20:01:06.418288] openib_check_cq: completed send wr_id 15e73468. [D 20:01:06.418317] openib_check_cq: completed recv wr_id 15e73de8. [D 20:01:06.418344] openib_post_rr: da13:3346 bh 96 wr_id 15e73de8. [D 20:01:06.418375] post_sr: da13:3346 bh 96 len 48 wr_id 15e73480. [D 20:01:06.418411] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:06.418437] ========================================================= [D 20:01:06.418461] PINT_process_request [D 20:01:06.418486] tiling 10 copies [D 20:01:06.418510] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.418537] lvl 0 el 0 blk 0 by 289406976 [D 20:01:06.418561] to 289406976 ta 268435456 fi 301989888 [D 20:01:06.418590] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:06.418615] basic type or contiguous data [D 20:01:06.418639] Request disp [D 20:01:06.418663] contig_offset = 289406976 contig_size = 46137344 lvl_flag = 1 [D 20:01:06.418688] calling distribute [D 20:01:06.418712] PINT_distribute [D 20:01:06.418736] of 289406976 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.418763] begin iteration loff: 289406976 [D 20:01:06.418787] segment exceeds byte limit [D 20:01:06.418811] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:06.418836] **********CALL***PROCESS********* [D 20:01:06.418860] segment of 20971520 sz 2097152 [D 20:01:06.418885] PINT_process_request [D 20:01:06.418909] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.418934] lvl 0 el 0 blk 0 by 20971520 [D 20:01:06.418959] to 20971520 ta 20971520 fi 23068672 [D 20:01:06.418984] basic type or contiguous data [D 20:01:06.419009] Request disp [D 20:01:06.419032] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:06.419057] process a segment [D 20:01:06.419081] of 20971520 sz 2097152 [D 20:01:06.419105] add a segment 0 [D 20:01:06.419130] retval = 2097152 [D 20:01:06.419154] exiting distribute returned less than expected [D 20:01:06.419178] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:06.419204] *****RETURN***FROM***PROCESS***** [D 20:01:06.419229] end iteration [D 20:01:06.419253] done with segments or bytes [D 20:01:06.419277] of 291504128 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.419302] next loff: 291504128 next poff: 291504128 [D 20:01:06.419327] return value 2097152 [D 20:01:06.419351] finished [D 20:01:06.419375] retval = 2097152 [D 20:01:06.419399] exiting distribute returned less than expected [D 20:01:06.419423] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 291504128 fo 301989888 eof 0 [D 20:01:06.419449] ========================================================= [D 20:01:06.419473] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:06.419499] element 0: offset: 0x93841000, size: 2097152 [D 20:01:06.419525] openib_check_cq: completed send wr_id 15e73480. [D 20:01:06.419554] post_sr: da13:3346 bh 97 len 48 wr_id 15e73498. [D 20:01:06.419581] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:06.419607] ========================================================= [D 20:01:06.419631] PINT_process_request [D 20:01:06.419654] tiling 8 copies [D 20:01:06.419679] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.419705] lvl 0 el 0 blk 0 by 224395264 [D 20:01:06.419730] to 224395264 ta 201326592 fi 234881024 [D 20:01:06.419754] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:06.419779] basic type or contiguous data [D 20:01:06.419803] Request disp [D 20:01:06.419827] contig_offset = 224395264 contig_size = 44040192 lvl_flag = 1 [D 20:01:06.419852] calling distribute [D 20:01:06.419877] PINT_distribute [D 20:01:06.419900] of 224395264 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.419926] begin iteration loff: 224395264 [D 20:01:06.419952] segment exceeds byte limit [D 20:01:06.419976] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:06.420001] **********CALL***PROCESS********* [D 20:01:06.420023] segment of 23068672 sz 2097152 [D 20:01:06.420048] PINT_process_request [D 20:01:06.420072] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.420098] lvl 0 el 0 blk 0 by 23068672 [D 20:01:06.420124] to 23068672 ta 23068672 fi 25165824 [D 20:01:06.420148] basic type or contiguous data [D 20:01:06.420172] Request disp [D 20:01:06.420196] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:06.420225] process a segment [D 20:01:06.420248] of 23068672 sz 2097152 [D 20:01:06.420273] add a segment 0 [D 20:01:06.420297] retval = 2097152 [D 20:01:06.420321] exiting distribute returned less than expected [D 20:01:06.420348] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:06.420374] *****RETURN***FROM***PROCESS***** [D 20:01:06.420399] end iteration [D 20:01:06.420423] done with segments or bytes [D 20:01:06.420447] of 226492416 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.420473] next loff: 226492416 next poff: 226492416 [D 20:01:06.420496] return value 2097152 [D 20:01:06.420521] finished [D 20:01:06.420545] retval = 2097152 [D 20:01:06.420569] exiting distribute returned less than expected [D 20:01:06.420593] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 226492416 fo 234881024 eof 0 [D 20:01:06.420619] ========================================================= [D 20:01:06.420643] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:06.420668] element 0: offset: 0x97a45000, size: 2097152 [D 20:01:06.420694] openib_check_cq: completed send wr_id 15e73498. [D 20:01:06.420723] post_sr: da13:3346 bh 98 len 48 wr_id 15e734b0. [D 20:01:06.420750] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:06.420775] ========================================================= [D 20:01:06.420800] PINT_process_request [D 20:01:06.420824] tiling 12 copies [D 20:01:06.420848] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.420874] lvl 0 el 0 blk 0 by 356515840 [D 20:01:06.420899] to 356515840 ta 335544320 fi 369098752 [D 20:01:06.420924] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:06.420948] basic type or contiguous data [D 20:01:06.420972] Request disp [D 20:01:06.420996] contig_offset = 356515840 contig_size = 46137344 lvl_flag = 1 [D 20:01:06.421022] calling distribute [D 20:01:06.421046] PINT_distribute [D 20:01:06.421070] of 356515840 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.421095] begin iteration loff: 356515840 [D 20:01:06.421120] segment exceeds byte limit [D 20:01:06.421145] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:06.421170] **********CALL***PROCESS********* [D 20:01:06.421194] segment of 20971520 sz 2097152 [D 20:01:06.421218] PINT_process_request [D 20:01:06.421242] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.421268] lvl 0 el 0 blk 0 by 20971520 [D 20:01:06.421293] to 20971520 ta 20971520 fi 23068672 [D 20:01:06.421318] basic type or contiguous data [D 20:01:06.421342] Request disp [D 20:01:06.421366] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:06.421391] process a segment [D 20:01:06.421415] of 20971520 sz 2097152 [D 20:01:06.421440] add a segment 0 [D 20:01:06.421463] retval = 2097152 [D 20:01:06.421487] exiting distribute returned less than expected [D 20:01:06.421512] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:06.421538] *****RETURN***FROM***PROCESS***** [D 20:01:06.421563] end iteration [D 20:01:06.421586] done with segments or bytes [D 20:01:06.421610] of 358612992 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.421636] next loff: 358612992 next poff: 358612992 [D 20:01:06.421661] return value 2097152 [D 20:01:06.421685] finished [D 20:01:06.421709] retval = 2097152 [D 20:01:06.421732] exiting distribute returned less than expected [D 20:01:06.421756] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 358612992 fo 369098752 eof 0 [D 20:01:06.421783] ========================================================= [D 20:01:06.421807] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:06.421832] element 0: offset: 0x8f83d000, size: 2097152 [D 20:01:06.421858] openib_check_cq: completed send wr_id 15e734b0. [D 20:01:06.421888] post_sr: da13:3346 bh 99 len 48 wr_id 15e734c8. [D 20:01:06.421916] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:06.421941] ========================================================= [D 20:01:06.421965] PINT_process_request [D 20:01:06.421993] tiling 7 copies [D 20:01:06.422018] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.422043] lvl 0 el 0 blk 0 by 190840832 [D 20:01:06.422068] to 190840832 ta 167772160 fi 201326592 [D 20:01:06.422093] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:06.422119] basic type or contiguous data [D 20:01:06.422142] Request disp [D 20:01:06.422165] contig_offset = 190840832 contig_size = 44040192 lvl_flag = 1 [D 20:01:06.422191] calling distribute [D 20:01:06.422218] PINT_distribute [D 20:01:06.422245] of 190840832 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.422276] begin iteration loff: 190840832 [D 20:01:06.422305] segment exceeds byte limit [D 20:01:06.422331] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:06.422356] **********CALL***PROCESS********* [D 20:01:06.422380] segment of 23068672 sz 2097152 [D 20:01:06.422405] PINT_process_request [D 20:01:06.422429] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.422455] lvl 0 el 0 blk 0 by 23068672 [D 20:01:06.422480] to 23068672 ta 23068672 fi 25165824 [D 20:01:06.422505] basic type or contiguous data [D 20:01:06.422530] Request disp [D 20:01:06.422553] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:06.422578] process a segment [D 20:01:06.422602] of 23068672 sz 2097152 [D 20:01:06.422626] add a segment 0 [D 20:01:06.422651] retval = 2097152 [D 20:01:06.422675] exiting distribute returned less than expected [D 20:01:06.422698] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:06.422724] *****RETURN***FROM***PROCESS***** [D 20:01:06.422749] end iteration [D 20:01:06.422773] done with segments or bytes [D 20:01:06.422797] of 192937984 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.422822] next loff: 192937984 next poff: 192937984 [D 20:01:06.422847] return value 2097152 [D 20:01:06.422872] finished [D 20:01:06.422895] retval = 2097152 [D 20:01:06.422920] exiting distribute returned less than expected [D 20:01:06.422944] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 192937984 fo 201326592 eof 0 [D 20:01:06.422969] ========================================================= [D 20:01:06.422994] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:06.423019] element 0: offset: 0x99a47000, size: 2097152 [D 20:01:06.423045] openib_check_cq: completed send wr_id 15e734c8. [D 20:01:06.423070] openib_check_cq: completed recv wr_id 15e73e00. [D 20:01:06.423097] openib_post_rr: da13:3346 bh 97 wr_id 15e73e00. [D 20:01:06.423127] post_sr: da13:3346 bh 0 len 48 wr_id 15e72b80. [D 20:01:06.423153] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:06.423179] ========================================================= [D 20:01:06.423204] PINT_process_request [D 20:01:06.423228] tiling 6 copies [D 20:01:06.423252] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.423278] lvl 0 el 0 blk 0 by 159383552 [D 20:01:06.423303] to 159383552 ta 134217728 fi 167772160 [D 20:01:06.423328] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:06.423353] basic type or contiguous data [D 20:01:06.423377] Request disp [D 20:01:06.423402] contig_offset = 159383552 contig_size = 41943040 lvl_flag = 1 [D 20:01:06.423428] calling distribute [D 20:01:06.423452] PINT_distribute [D 20:01:06.423476] of 159383552 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.423502] begin iteration loff: 159383552 [D 20:01:06.423527] segment exceeds byte limit [D 20:01:06.423551] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:06.423576] **********CALL***PROCESS********* [D 20:01:06.423600] segment of 25165824 sz 2097152 [D 20:01:06.423624] PINT_process_request [D 20:01:06.423648] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.423674] lvl 0 el 0 blk 0 by 25165824 [D 20:01:06.423700] to 25165824 ta 25165824 fi 27262976 [D 20:01:06.423727] basic type or contiguous data [D 20:01:06.423751] Request disp [D 20:01:06.423775] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:06.423801] process a segment [D 20:01:06.423825] of 25165824 sz 2097152 [D 20:01:06.423849] add a segment 0 [D 20:01:06.423872] retval = 2097152 [D 20:01:06.423896] exiting distribute returned less than expected [D 20:01:06.423921] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:06.423947] *****RETURN***FROM***PROCESS***** [D 20:01:06.423972] end iteration [D 20:01:06.423995] done with segments or bytes [D 20:01:06.424018] of 161480704 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.424044] next loff: 161480704 next poff: 161480704 [D 20:01:06.424069] return value 2097152 [D 20:01:06.424094] finished [D 20:01:06.424118] retval = 2097152 [D 20:01:06.424141] exiting distribute returned less than expected [D 20:01:06.424165] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 161480704 fo 167772160 eof 0 [D 20:01:06.424192] ========================================================= [D 20:01:06.424220] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:06.424245] element 0: offset: 0x9bc49000, size: 2097152 [D 20:01:06.424271] openib_check_cq: completed send wr_id 15e72b80. [D 20:01:06.424301] post_sr: da13:3346 bh 1 len 48 wr_id 15e72b98. [D 20:01:06.424328] PINT_client_state_machine_test id 79 [D 20:01:06.424362] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:06.424387] ========================================================= [D 20:01:06.424412] PINT_process_request [D 20:01:06.424436] tiling 13 copies [D 20:01:06.424460] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.424487] lvl 0 el 0 blk 0 by 390070272 [D 20:01:06.424511] to 390070272 ta 369098752 fi 402653184 [D 20:01:06.424536] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:06.424561] basic type or contiguous data [D 20:01:06.424585] Request disp [D 20:01:06.424609] contig_offset = 390070272 contig_size = 46137344 lvl_flag = 1 [D 20:01:06.424633] calling distribute [D 20:01:06.424657] PINT_distribute [D 20:01:06.424681] of 390070272 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.424708] begin iteration loff: 390070272 [D 20:01:06.424734] segment exceeds byte limit [D 20:01:06.424757] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:06.424782] **********CALL***PROCESS********* [D 20:01:06.424805] segment of 20971520 sz 2097152 [D 20:01:06.424830] PINT_process_request [D 20:01:06.424854] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.424880] lvl 0 el 0 blk 0 by 20971520 [D 20:01:06.424904] to 20971520 ta 20971520 fi 23068672 [D 20:01:06.424930] basic type or contiguous data [D 20:01:06.424954] Request disp [D 20:01:06.424978] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:06.425003] process a segment [D 20:01:06.425026] of 20971520 sz 2097152 [D 20:01:06.425050] add a segment 0 [D 20:01:06.425074] retval = 2097152 [D 20:01:06.425098] exiting distribute returned less than expected [D 20:01:06.425123] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:06.425148] *****RETURN***FROM***PROCESS***** [D 20:01:06.425173] end iteration [D 20:01:06.425197] done with segments or bytes [D 20:01:06.425221] of 392167424 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.425247] next loff: 392167424 next poff: 392167424 [D 20:01:06.425271] return value 2097152 [D 20:01:06.425295] finished [D 20:01:06.425319] retval = 2097152 [D 20:01:06.425343] exiting distribute returned less than expected [D 20:01:06.425367] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 392167424 fo 402653184 eof 0 [D 20:01:06.425394] ========================================================= [D 20:01:06.425417] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:06.425446] element 0: offset: 0x8d83b000, size: 2097152 [D 20:01:06.425472] openib_check_cq: completed send wr_id 15e72b98. [D 20:01:06.425502] post_sr: da13:3346 bh 2 len 48 wr_id 15e72bb0. [D 20:01:06.425571] openib_check_cq: completed send wr_id 15e72bb0. [D 20:01:06.427049] openib_check_cq: completed recv wr_id 15e73e18. [D 20:01:06.427076] openib_post_rr: da13:3346 bh 98 wr_id 15e73e18. [D 20:01:06.427108] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:06.427134] ========================================================= [D 20:01:06.427159] PINT_process_request [D 20:01:06.427183] tiling 11 copies [D 20:01:06.427207] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.427233] lvl 0 el 0 blk 0 by 322961408 [D 20:01:06.427258] to 322961408 ta 301989888 fi 335544320 [D 20:01:06.427283] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:06.427308] basic type or contiguous data [D 20:01:06.427331] Request disp [D 20:01:06.427355] contig_offset = 322961408 contig_size = 46137344 lvl_flag = 1 [D 20:01:06.427381] calling distribute [D 20:01:06.427405] PINT_distribute [D 20:01:06.427428] of 322961408 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.427454] begin iteration loff: 322961408 [D 20:01:06.427479] segment exceeds byte limit [D 20:01:06.427503] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:06.427528] **********CALL***PROCESS********* [D 20:01:06.427552] segment of 20971520 sz 2097152 [D 20:01:06.427576] PINT_process_request [D 20:01:06.427599] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.427625] lvl 0 el 0 blk 0 by 20971520 [D 20:01:06.427650] to 20971520 ta 20971520 fi 23068672 [D 20:01:06.427675] basic type or contiguous data [D 20:01:06.427699] Request disp [D 20:01:06.427722] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:06.427747] process a segment [D 20:01:06.427771] of 20971520 sz 2097152 [D 20:01:06.427795] add a segment 0 [D 20:01:06.427819] retval = 2097152 [D 20:01:06.427843] exiting distribute returned less than expected [D 20:01:06.427866] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:06.427893] *****RETURN***FROM***PROCESS***** [D 20:01:06.427918] end iteration [D 20:01:06.427942] done with segments or bytes [D 20:01:06.427966] of 325058560 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.427990] next loff: 325058560 next poff: 325058560 [D 20:01:06.428015] return value 2097152 [D 20:01:06.428040] finished [D 20:01:06.428063] retval = 2097152 [D 20:01:06.428088] exiting distribute returned less than expected [D 20:01:06.428112] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 325058560 fo 335544320 eof 0 [D 20:01:06.428137] ========================================================= [D 20:01:06.428161] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:06.428187] element 0: offset: 0x9183f000, size: 2097152 [D 20:01:06.428217] post_sr: da13:3346 bh 3 len 48 wr_id 15e72bc8. [D 20:01:06.428286] openib_check_cq: completed send wr_id 15e72bc8. [D 20:01:06.431496] openib_check_cq: completed recv wr_id 15e73e30. [D 20:01:06.431524] openib_post_rr: da13:3346 bh 99 wr_id 15e73e30. [D 20:01:06.431555] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:06.431581] ========================================================= [D 20:01:06.431606] PINT_process_request [D 20:01:06.431630] tiling 14 copies [D 20:01:06.431653] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.431680] lvl 0 el 0 blk 0 by 423624704 [D 20:01:06.431705] to 423624704 ta 402653184 fi 436207616 [D 20:01:06.431730] mto 20971520 mta 18874368 mfi 20971520 [D 20:01:06.431755] basic type or contiguous data [D 20:01:06.431778] Request disp [D 20:01:06.431802] contig_offset = 423624704 contig_size = 46137344 lvl_flag = 1 [D 20:01:06.431831] calling distribute [D 20:01:06.431856] PINT_distribute [D 20:01:06.431879] of 423624704 sz 12582912 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.431905] begin iteration loff: 423624704 [D 20:01:06.431931] segment exceeds byte limit [D 20:01:06.431955] client lstof 20971520 diff 0 sgof 20971520 [D 20:01:06.431980] **********CALL***PROCESS********* [D 20:01:06.432003] segment of 20971520 sz 2097152 [D 20:01:06.432027] PINT_process_request [D 20:01:06.432052] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.432078] lvl 0 el 0 blk 0 by 20971520 [D 20:01:06.432103] to 20971520 ta 20971520 fi 23068672 [D 20:01:06.432127] basic type or contiguous data [D 20:01:06.432151] Request disp [D 20:01:06.432175] contig_offset = 20971520 contig_size = 12582912 lvl_flag = 1 [D 20:01:06.432201] process a segment [D 20:01:06.432227] of 20971520 sz 2097152 [D 20:01:06.432251] add a segment 0 [D 20:01:06.432275] retval = 2097152 [D 20:01:06.432299] exiting distribute returned less than expected [D 20:01:06.432323] done sg 1 sm 64 by 2097152 bm 2097152 ta 20971520 to 23068672 fo 23068672 eof 0 [D 20:01:06.432350] *****RETURN***FROM***PROCESS***** [D 20:01:06.432374] end iteration [D 20:01:06.432397] done with segments or bytes [D 20:01:06.432421] of 425721856 sz 10485760 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.432447] next loff: 425721856 next poff: 425721856 [D 20:01:06.432472] return value 2097152 [D 20:01:06.432497] finished [D 20:01:06.432519] retval = 2097152 [D 20:01:06.432543] exiting distribute returned less than expected [D 20:01:06.432567] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 425721856 fo 436207616 eof 0 [D 20:01:06.432594] ========================================================= [D 20:01:06.432618] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:06.432643] element 0: offset: 0x8b839000, size: 2097152 [D 20:01:06.432672] post_sr: da13:3346 bh 4 len 48 wr_id 15e72be0. [D 20:01:06.432741] openib_check_cq: completed send wr_id 15e72be0. [D 20:01:06.434356] PINT_client_state_machine_test id 79 [D 20:01:06.435939] openib_check_cq: completed recv wr_id 15e734e8. [D 20:01:06.435967] openib_post_rr: da13:3346 bh 0 wr_id 15e734e8. [D 20:01:06.435998] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:06.436024] ========================================================= [D 20:01:06.436048] PINT_process_request [D 20:01:06.436073] tiling 9 copies [D 20:01:06.436097] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.436122] lvl 0 el 0 blk 0 by 257949696 [D 20:01:06.436148] to 257949696 ta 234881024 fi 268435456 [D 20:01:06.436173] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:06.436198] basic type or contiguous data [D 20:01:06.436224] Request disp [D 20:01:06.436248] contig_offset = 257949696 contig_size = 44040192 lvl_flag = 1 [D 20:01:06.436274] calling distribute [D 20:01:06.436298] PINT_distribute [D 20:01:06.436322] of 257949696 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.436348] begin iteration loff: 257949696 [D 20:01:06.436373] segment exceeds byte limit [D 20:01:06.436398] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:06.436423] **********CALL***PROCESS********* [D 20:01:06.436447] segment of 23068672 sz 2097152 [D 20:01:06.436470] PINT_process_request [D 20:01:06.436495] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.436521] lvl 0 el 0 blk 0 by 23068672 [D 20:01:06.436546] to 23068672 ta 23068672 fi 25165824 [D 20:01:06.436571] basic type or contiguous data [D 20:01:06.436594] Request disp [D 20:01:06.436618] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:06.436644] process a segment [D 20:01:06.436668] of 23068672 sz 2097152 [D 20:01:06.436692] add a segment 0 [D 20:01:06.436716] retval = 2097152 [D 20:01:06.436739] exiting distribute returned less than expected [D 20:01:06.436764] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:06.436793] *****RETURN***FROM***PROCESS***** [D 20:01:06.436818] end iteration [D 20:01:06.436841] done with segments or bytes [D 20:01:06.436865] of 260046848 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.436891] next loff: 260046848 next poff: 260046848 [D 20:01:06.436916] return value 2097152 [D 20:01:06.436941] finished [D 20:01:06.436964] retval = 2097152 [D 20:01:06.436988] exiting distribute returned less than expected [D 20:01:06.437012] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 260046848 fo 268435456 eof 0 [D 20:01:06.437039] ========================================================= [D 20:01:06.437063] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:06.437088] element 0: offset: 0x95a43000, size: 2097152 [D 20:01:06.437118] post_sr: da13:3346 bh 5 len 48 wr_id 15e72bf8. [D 20:01:06.437191] openib_check_cq: completed send wr_id 15e72bf8. [D 20:01:06.438169] openib_check_cq: completed recv wr_id 15e73500. [D 20:01:06.438197] openib_post_rr: da13:3346 bh 1 wr_id 15e73500. [D 20:01:06.438233] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:06.438262] ========================================================= [D 20:01:06.438290] PINT_process_request [D 20:01:06.438318] tiling 10 copies [D 20:01:06.438343] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.438369] lvl 0 el 0 blk 0 by 291504128 [D 20:01:06.438394] to 291504128 ta 268435456 fi 301989888 [D 20:01:06.438419] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:06.438444] basic type or contiguous data [D 20:01:06.438467] Request disp [D 20:01:06.438491] contig_offset = 291504128 contig_size = 44040192 lvl_flag = 1 [D 20:01:06.438517] calling distribute [D 20:01:06.438541] PINT_distribute [D 20:01:06.438565] of 291504128 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.438591] begin iteration loff: 291504128 [D 20:01:06.438615] segment exceeds byte limit [D 20:01:06.438640] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:06.438665] **********CALL***PROCESS********* [D 20:01:06.438689] segment of 23068672 sz 2097152 [D 20:01:06.438713] PINT_process_request [D 20:01:06.438736] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.438763] lvl 0 el 0 blk 0 by 23068672 [D 20:01:06.438787] to 23068672 ta 23068672 fi 25165824 [D 20:01:06.438813] basic type or contiguous data [D 20:01:06.438837] Request disp [D 20:01:06.438860] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:06.438885] process a segment [D 20:01:06.438909] of 23068672 sz 2097152 [D 20:01:06.438933] add a segment 0 [D 20:01:06.438957] retval = 2097152 [D 20:01:06.438981] exiting distribute returned less than expected [D 20:01:06.439004] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:06.439031] *****RETURN***FROM***PROCESS***** [D 20:01:06.439055] end iteration [D 20:01:06.439079] done with segments or bytes [D 20:01:06.439103] of 293601280 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.439128] next loff: 293601280 next poff: 293601280 [D 20:01:06.439153] return value 2097152 [D 20:01:06.439177] finished [D 20:01:06.439201] retval = 2097152 [D 20:01:06.439226] exiting distribute returned less than expected [D 20:01:06.439249] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 293601280 fo 301989888 eof 0 [D 20:01:06.439275] ========================================================= [D 20:01:06.439300] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:06.439325] element 0: offset: 0x93a41000, size: 2097152 [D 20:01:06.439354] post_sr: da13:3346 bh 6 len 48 wr_id 15e72c10. [D 20:01:06.439426] openib_check_cq: completed send wr_id 15e72c10. [D 20:01:06.440391] openib_check_cq: completed recv wr_id 15e73518. [D 20:01:06.440418] openib_post_rr: da13:3346 bh 2 wr_id 15e73518. [D 20:01:06.440447] openib_check_cq: completed recv wr_id 15e73530. [D 20:01:06.440474] openib_post_rr: da13:3346 bh 3 wr_id 15e73530. [D 20:01:06.440505] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:06.440530] ========================================================= [D 20:01:06.440555] PINT_process_request [D 20:01:06.440579] tiling 8 copies [D 20:01:06.440603] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.440629] lvl 0 el 0 blk 0 by 226492416 [D 20:01:06.440654] to 226492416 ta 201326592 fi 234881024 [D 20:01:06.440679] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:06.440703] basic type or contiguous data [D 20:01:06.440727] Request disp [D 20:01:06.440751] contig_offset = 226492416 contig_size = 41943040 lvl_flag = 1 [D 20:01:06.440775] calling distribute [D 20:01:06.440800] PINT_distribute [D 20:01:06.440824] of 226492416 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.440851] begin iteration loff: 226492416 [D 20:01:06.440876] segment exceeds byte limit [D 20:01:06.440899] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:06.440923] **********CALL***PROCESS********* [D 20:01:06.440947] segment of 25165824 sz 2097152 [D 20:01:06.440972] PINT_process_request [D 20:01:06.440996] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.441021] lvl 0 el 0 blk 0 by 25165824 [D 20:01:06.441046] to 25165824 ta 25165824 fi 27262976 [D 20:01:06.441071] basic type or contiguous data [D 20:01:06.441095] Request disp [D 20:01:06.441119] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:06.441143] process a segment [D 20:01:06.441167] of 25165824 sz 2097152 [D 20:01:06.441191] add a segment 0 [D 20:01:06.441215] retval = 2097152 [D 20:01:06.441239] exiting distribute returned less than expected [D 20:01:06.441264] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:06.441289] *****RETURN***FROM***PROCESS***** [D 20:01:06.441313] end iteration [D 20:01:06.441337] done with segments or bytes [D 20:01:06.441361] of 228589568 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.441387] next loff: 228589568 next poff: 228589568 [D 20:01:06.441411] return value 2097152 [D 20:01:06.441435] finished [D 20:01:06.441459] retval = 2097152 [D 20:01:06.441483] exiting distribute returned less than expected [D 20:01:06.441507] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 228589568 fo 234881024 eof 0 [D 20:01:06.441534] ========================================================= [D 20:01:06.441557] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:06.441583] element 0: offset: 0x97c45000, size: 2097152 [D 20:01:06.441612] post_sr: da13:3346 bh 7 len 48 wr_id 15e72c28. [D 20:01:06.441639] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:06.441664] ========================================================= [D 20:01:06.441689] PINT_process_request [D 20:01:06.441713] tiling 12 copies [D 20:01:06.441737] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.441762] lvl 0 el 0 blk 0 by 358612992 [D 20:01:06.441787] to 358612992 ta 335544320 fi 369098752 [D 20:01:06.441812] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:06.441837] basic type or contiguous data [D 20:01:06.441861] Request disp [D 20:01:06.441884] contig_offset = 358612992 contig_size = 44040192 lvl_flag = 1 [D 20:01:06.441910] calling distribute [D 20:01:06.441934] PINT_distribute [D 20:01:06.441958] of 358612992 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.441984] begin iteration loff: 358612992 [D 20:01:06.442008] segment exceeds byte limit [D 20:01:06.442032] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:06.442057] **********CALL***PROCESS********* [D 20:01:06.442080] segment of 23068672 sz 2097152 [D 20:01:06.442105] PINT_process_request [D 20:01:06.442129] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.442157] lvl 0 el 0 blk 0 by 23068672 [D 20:01:06.442183] to 23068672 ta 23068672 fi 25165824 [D 20:01:06.442208] basic type or contiguous data [D 20:01:06.442237] Request disp [D 20:01:06.442265] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:06.442294] process a segment [D 20:01:06.442321] of 23068672 sz 2097152 [D 20:01:06.442346] add a segment 0 [D 20:01:06.442370] retval = 2097152 [D 20:01:06.442394] exiting distribute returned less than expected [D 20:01:06.442418] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:06.442444] *****RETURN***FROM***PROCESS***** [D 20:01:06.442469] end iteration [D 20:01:06.442492] done with segments or bytes [D 20:01:06.442516] of 360710144 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.442542] next loff: 360710144 next poff: 360710144 [D 20:01:06.442566] return value 2097152 [D 20:01:06.442591] finished [D 20:01:06.442615] retval = 2097152 [D 20:01:06.442639] exiting distribute returned less than expected [D 20:01:06.442663] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 360710144 fo 369098752 eof 0 [D 20:01:06.442689] ========================================================= [D 20:01:06.442713] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:06.442739] element 0: offset: 0x8fa3d000, size: 2097152 [D 20:01:06.442765] openib_check_cq: completed send wr_id 15e72c28. [D 20:01:06.442790] openib_check_cq: completed recv wr_id 15e73548. [D 20:01:06.442817] openib_post_rr: da13:3346 bh 4 wr_id 15e73548. [D 20:01:06.442843] openib_check_cq: completed recv wr_id 15e73560. [D 20:01:06.442869] openib_post_rr: da13:3346 bh 5 wr_id 15e73560. [D 20:01:06.442899] post_sr: da13:3346 bh 8 len 48 wr_id 15e72c40. [D 20:01:06.442933] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:06.442958] ========================================================= [D 20:01:06.442983] PINT_process_request [D 20:01:06.443007] tiling 6 copies [D 20:01:06.443032] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.443058] lvl 0 el 0 blk 0 by 161480704 [D 20:01:06.443082] to 161480704 ta 134217728 fi 167772160 [D 20:01:06.443107] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:06.443132] basic type or contiguous data [D 20:01:06.443156] Request disp [D 20:01:06.443180] contig_offset = 161480704 contig_size = 39845888 lvl_flag = 1 [D 20:01:06.443204] calling distribute [D 20:01:06.443229] PINT_distribute [D 20:01:06.443253] of 161480704 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.443280] begin iteration loff: 161480704 [D 20:01:06.443305] segment exceeds byte limit [D 20:01:06.443328] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:06.443353] **********CALL***PROCESS********* [D 20:01:06.443377] segment of 27262976 sz 2097152 [D 20:01:06.443404] PINT_process_request [D 20:01:06.443428] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.443454] lvl 0 el 0 blk 0 by 27262976 [D 20:01:06.443479] to 27262976 ta 27262976 fi 29360128 [D 20:01:06.443504] basic type or contiguous data [D 20:01:06.443528] Request disp [D 20:01:06.443552] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:06.443577] process a segment [D 20:01:06.443600] of 27262976 sz 2097152 [D 20:01:06.443625] add a segment 0 [D 20:01:06.443649] retval = 2097152 [D 20:01:06.443673] exiting distribute returned less than expected [D 20:01:06.443696] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:06.443723] *****RETURN***FROM***PROCESS***** [D 20:01:06.443748] end iteration [D 20:01:06.443771] done with segments or bytes [D 20:01:06.443795] of 163577856 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.443820] next loff: 163577856 next poff: 163577856 [D 20:01:06.443845] return value 2097152 [D 20:01:06.443870] finished [D 20:01:06.443897] retval = 2097152 [D 20:01:06.443921] exiting distribute returned less than expected [D 20:01:06.443944] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 163577856 fo 167772160 eof 0 [D 20:01:06.443971] ========================================================= [D 20:01:06.443995] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:06.444020] element 0: offset: 0x9be49000, size: 2097152 [D 20:01:06.444046] openib_check_cq: completed send wr_id 15e72c40. [D 20:01:06.444075] post_sr: da13:3346 bh 9 len 48 wr_id 15e72c58. [D 20:01:06.444103] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:06.444128] ========================================================= [D 20:01:06.444152] PINT_process_request [D 20:01:06.444177] tiling 7 copies [D 20:01:06.444201] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.444230] lvl 0 el 0 blk 0 by 192937984 [D 20:01:06.444255] to 192937984 ta 167772160 fi 201326592 [D 20:01:06.444280] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:06.444305] basic type or contiguous data [D 20:01:06.444329] Request disp [D 20:01:06.444353] contig_offset = 192937984 contig_size = 41943040 lvl_flag = 1 [D 20:01:06.444377] calling distribute [D 20:01:06.444401] PINT_distribute [D 20:01:06.444425] of 192937984 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.444452] begin iteration loff: 192937984 [D 20:01:06.444477] segment exceeds byte limit [D 20:01:06.444500] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:06.444525] **********CALL***PROCESS********* [D 20:01:06.444549] segment of 25165824 sz 2097152 [D 20:01:06.444573] PINT_process_request [D 20:01:06.444597] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.444623] lvl 0 el 0 blk 0 by 25165824 [D 20:01:06.444648] to 25165824 ta 25165824 fi 27262976 [D 20:01:06.444673] basic type or contiguous data [D 20:01:06.444697] Request disp [D 20:01:06.444721] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:06.444746] process a segment [D 20:01:06.444769] of 25165824 sz 2097152 [D 20:01:06.444794] add a segment 0 [D 20:01:06.444818] retval = 2097152 [D 20:01:06.444842] exiting distribute returned less than expected [D 20:01:06.444866] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:06.444892] *****RETURN***FROM***PROCESS***** [D 20:01:06.444916] end iteration [D 20:01:06.444940] done with segments or bytes [D 20:01:06.444964] of 195035136 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.444990] next loff: 195035136 next poff: 195035136 [D 20:01:06.445014] return value 2097152 [D 20:01:06.445038] finished [D 20:01:06.445062] retval = 2097152 [D 20:01:06.445086] exiting distribute returned less than expected [D 20:01:06.445110] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 195035136 fo 201326592 eof 0 [D 20:01:06.445137] ========================================================= [D 20:01:06.445160] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:06.445186] element 0: offset: 0x99c47000, size: 2097152 [D 20:01:06.445212] openib_check_cq: completed send wr_id 15e72c58. [D 20:01:06.445238] openib_check_cq: completed recv wr_id 15e73578. [D 20:01:06.445263] openib_post_rr: da13:3346 bh 6 wr_id 15e73578. [D 20:01:06.445294] post_sr: da13:3346 bh 10 len 48 wr_id 15e72c70. [D 20:01:06.445322] PINT_client_state_machine_test id 79 [D 20:01:06.445355] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:06.445381] ========================================================= [D 20:01:06.445405] PINT_process_request [D 20:01:06.445429] tiling 13 copies [D 20:01:06.445453] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.445480] lvl 0 el 0 blk 0 by 392167424 [D 20:01:06.445505] to 392167424 ta 369098752 fi 402653184 [D 20:01:06.445529] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:06.445559] basic type or contiguous data [D 20:01:06.445584] Request disp [D 20:01:06.445608] contig_offset = 392167424 contig_size = 44040192 lvl_flag = 1 [D 20:01:06.445633] calling distribute [D 20:01:06.445657] PINT_distribute [D 20:01:06.445681] of 392167424 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.445708] begin iteration loff: 392167424 [D 20:01:06.445733] segment exceeds byte limit [D 20:01:06.445756] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:06.445781] **********CALL***PROCESS********* [D 20:01:06.445805] segment of 23068672 sz 2097152 [D 20:01:06.445829] PINT_process_request [D 20:01:06.445853] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.445879] lvl 0 el 0 blk 0 by 23068672 [D 20:01:06.445904] to 23068672 ta 23068672 fi 25165824 [D 20:01:06.445929] basic type or contiguous data [D 20:01:06.445953] Request disp [D 20:01:06.445977] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:06.446002] process a segment [D 20:01:06.446025] of 23068672 sz 2097152 [D 20:01:06.446049] add a segment 0 [D 20:01:06.446073] retval = 2097152 [D 20:01:06.446098] exiting distribute returned less than expected [D 20:01:06.446122] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:06.446147] *****RETURN***FROM***PROCESS***** [D 20:01:06.446172] end iteration [D 20:01:06.446196] done with segments or bytes [D 20:01:06.446224] of 394264576 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.446253] next loff: 394264576 next poff: 394264576 [D 20:01:06.446282] return value 2097152 [D 20:01:06.446310] finished [D 20:01:06.446335] retval = 2097152 [D 20:01:06.446360] exiting distribute returned less than expected [D 20:01:06.446384] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 394264576 fo 402653184 eof 0 [D 20:01:06.446411] ========================================================= [D 20:01:06.446434] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:06.446460] element 0: offset: 0x8da3b000, size: 2097152 [D 20:01:06.446486] openib_check_cq: completed send wr_id 15e72c70. [D 20:01:06.446516] post_sr: da13:3346 bh 11 len 48 wr_id 15e72c88. [D 20:01:06.446585] openib_check_cq: completed send wr_id 15e72c88. [D 20:01:06.447075] openib_check_cq: completed recv wr_id 15e73590. [D 20:01:06.447102] openib_post_rr: da13:3346 bh 7 wr_id 15e73590. [D 20:01:06.447133] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:06.447158] ========================================================= [D 20:01:06.447183] PINT_process_request [D 20:01:06.447207] tiling 11 copies [D 20:01:06.447232] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.447257] lvl 0 el 0 blk 0 by 325058560 [D 20:01:06.447282] to 325058560 ta 301989888 fi 335544320 [D 20:01:06.447307] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:06.447332] basic type or contiguous data [D 20:01:06.447357] Request disp [D 20:01:06.447381] contig_offset = 325058560 contig_size = 44040192 lvl_flag = 1 [D 20:01:06.447405] calling distribute [D 20:01:06.447429] PINT_distribute [D 20:01:06.447453] of 325058560 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.447480] begin iteration loff: 325058560 [D 20:01:06.447504] segment exceeds byte limit [D 20:01:06.447528] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:06.447553] **********CALL***PROCESS********* [D 20:01:06.447577] segment of 23068672 sz 2097152 [D 20:01:06.447602] PINT_process_request [D 20:01:06.447626] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.447651] lvl 0 el 0 blk 0 by 23068672 [D 20:01:06.447676] to 23068672 ta 23068672 fi 25165824 [D 20:01:06.447701] basic type or contiguous data [D 20:01:06.447725] Request disp [D 20:01:06.447749] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:06.447773] process a segment [D 20:01:06.447800] of 23068672 sz 2097152 [D 20:01:06.447825] add a segment 0 [D 20:01:06.447849] retval = 2097152 [D 20:01:06.447873] exiting distribute returned less than expected [D 20:01:06.447896] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:06.447923] *****RETURN***FROM***PROCESS***** [D 20:01:06.447947] end iteration [D 20:01:06.447971] done with segments or bytes [D 20:01:06.447996] of 327155712 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.448020] next loff: 327155712 next poff: 327155712 [D 20:01:06.448045] return value 2097152 [D 20:01:06.448070] finished [D 20:01:06.448093] retval = 2097152 [D 20:01:06.448117] exiting distribute returned less than expected [D 20:01:06.448141] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 327155712 fo 335544320 eof 0 [D 20:01:06.448167] ========================================================= [D 20:01:06.448191] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:06.448219] element 0: offset: 0x91a3f000, size: 2097152 [D 20:01:06.448249] post_sr: da13:3346 bh 12 len 48 wr_id 15e72ca0. [D 20:01:06.448323] openib_check_cq: completed send wr_id 15e72ca0. [D 20:01:06.449302] openib_check_cq: completed recv wr_id 15e735a8. [D 20:01:06.449329] openib_post_rr: da13:3346 bh 8 wr_id 15e735a8. [D 20:01:06.449359] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:06.449385] ========================================================= [D 20:01:06.449410] PINT_process_request [D 20:01:06.449434] tiling 14 copies [D 20:01:06.449458] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.449484] lvl 0 el 0 blk 0 by 425721856 [D 20:01:06.449509] to 425721856 ta 402653184 fi 436207616 [D 20:01:06.449534] mto 23068672 mta 20971520 mfi 23068672 [D 20:01:06.449559] basic type or contiguous data [D 20:01:06.449583] Request disp [D 20:01:06.449606] contig_offset = 425721856 contig_size = 44040192 lvl_flag = 1 [D 20:01:06.449631] calling distribute [D 20:01:06.449656] PINT_distribute [D 20:01:06.449679] of 425721856 sz 10485760 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.449706] begin iteration loff: 425721856 [D 20:01:06.449730] segment exceeds byte limit [D 20:01:06.449754] client lstof 23068672 diff 0 sgof 23068672 [D 20:01:06.449779] **********CALL***PROCESS********* [D 20:01:06.449803] segment of 23068672 sz 2097152 [D 20:01:06.449827] PINT_process_request [D 20:01:06.449851] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.449877] lvl 0 el 0 blk 0 by 23068672 [D 20:01:06.449902] to 23068672 ta 23068672 fi 25165824 [D 20:01:06.449926] basic type or contiguous data [D 20:01:06.449950] Request disp [D 20:01:06.449974] contig_offset = 23068672 contig_size = 10485760 lvl_flag = 1 [D 20:01:06.449999] process a segment [D 20:01:06.450023] of 23068672 sz 2097152 [D 20:01:06.450047] add a segment 0 [D 20:01:06.450071] retval = 2097152 [D 20:01:06.450095] exiting distribute returned less than expected [D 20:01:06.450119] done sg 1 sm 64 by 2097152 bm 2097152 ta 23068672 to 25165824 fo 25165824 eof 0 [D 20:01:06.450145] *****RETURN***FROM***PROCESS***** [D 20:01:06.450170] end iteration [D 20:01:06.450194] done with segments or bytes [D 20:01:06.450221] of 427819008 sz 8388608 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.450251] next loff: 427819008 next poff: 427819008 [D 20:01:06.450280] return value 2097152 [D 20:01:06.450307] finished [D 20:01:06.450333] retval = 2097152 [D 20:01:06.450358] exiting distribute returned less than expected [D 20:01:06.450382] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 427819008 fo 436207616 eof 0 [D 20:01:06.450407] ========================================================= [D 20:01:06.450432] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:06.450457] element 0: offset: 0x8ba39000, size: 2097152 [D 20:01:06.450487] post_sr: da13:3346 bh 13 len 48 wr_id 15e72cb8. [D 20:01:06.450557] openib_check_cq: completed send wr_id 15e72cb8. [D 20:01:06.453750] openib_check_cq: completed recv wr_id 15e735c0. [D 20:01:06.453777] openib_post_rr: da13:3346 bh 9 wr_id 15e735c0. [D 20:01:06.453808] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:06.453834] ========================================================= [D 20:01:06.453859] PINT_process_request [D 20:01:06.453883] tiling 9 copies [D 20:01:06.453907] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.453932] lvl 0 el 0 blk 0 by 260046848 [D 20:01:06.453957] to 260046848 ta 234881024 fi 268435456 [D 20:01:06.453982] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:06.454007] basic type or contiguous data [D 20:01:06.454030] Request disp [D 20:01:06.454054] contig_offset = 260046848 contig_size = 41943040 lvl_flag = 1 [D 20:01:06.454080] calling distribute [D 20:01:06.454104] PINT_distribute [D 20:01:06.454128] of 260046848 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.454154] begin iteration loff: 260046848 [D 20:01:06.454178] segment exceeds byte limit [D 20:01:06.454202] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:06.454232] **********CALL***PROCESS********* [D 20:01:06.454258] segment of 25165824 sz 2097152 [D 20:01:06.454287] PINT_process_request [D 20:01:06.454314] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.454341] lvl 0 el 0 blk 0 by 25165824 [D 20:01:06.454367] to 25165824 ta 25165824 fi 27262976 [D 20:01:06.454391] basic type or contiguous data [D 20:01:06.454415] Request disp [D 20:01:06.454439] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:06.454463] process a segment [D 20:01:06.454487] of 25165824 sz 2097152 [D 20:01:06.454512] add a segment 0 [D 20:01:06.454536] retval = 2097152 [D 20:01:06.454560] exiting distribute returned less than expected [D 20:01:06.454584] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:06.454610] *****RETURN***FROM***PROCESS***** [D 20:01:06.454634] end iteration [D 20:01:06.454658] done with segments or bytes [D 20:01:06.454682] of 262144000 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.454708] next loff: 262144000 next poff: 262144000 [D 20:01:06.454732] return value 2097152 [D 20:01:06.454756] finished [D 20:01:06.454780] retval = 2097152 [D 20:01:06.454805] exiting distribute returned less than expected [D 20:01:06.454829] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 262144000 fo 268435456 eof 0 [D 20:01:06.454854] ========================================================= [D 20:01:06.454879] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:06.454904] element 0: offset: 0x95c43000, size: 2097152 [D 20:01:06.454934] post_sr: da13:3346 bh 14 len 48 wr_id 15e72cd0. [D 20:01:06.455006] openib_check_cq: completed send wr_id 15e72cd0. [D 20:01:06.455348] PINT_client_state_machine_test id 79 [D 20:01:06.458196] openib_check_cq: completed recv wr_id 15e735d8. [D 20:01:06.458228] openib_post_rr: da13:3346 bh 10 wr_id 15e735d8. [D 20:01:06.458262] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:06.458292] ========================================================= [D 20:01:06.458320] PINT_process_request [D 20:01:06.458344] tiling 10 copies [D 20:01:06.458369] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.458395] lvl 0 el 0 blk 0 by 293601280 [D 20:01:06.458420] to 293601280 ta 268435456 fi 301989888 [D 20:01:06.458445] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:06.458470] basic type or contiguous data [D 20:01:06.458494] Request disp [D 20:01:06.458518] contig_offset = 293601280 contig_size = 41943040 lvl_flag = 1 [D 20:01:06.458542] calling distribute [D 20:01:06.458566] PINT_distribute [D 20:01:06.458590] of 293601280 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.458620] begin iteration loff: 293601280 [D 20:01:06.458645] segment exceeds byte limit [D 20:01:06.458669] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:06.458694] **********CALL***PROCESS********* [D 20:01:06.458719] segment of 25165824 sz 2097152 [D 20:01:06.458744] PINT_process_request [D 20:01:06.458767] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.458793] lvl 0 el 0 blk 0 by 25165824 [D 20:01:06.458818] to 25165824 ta 25165824 fi 27262976 [D 20:01:06.458843] basic type or contiguous data [D 20:01:06.458867] Request disp [D 20:01:06.458890] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:06.458915] process a segment [D 20:01:06.458939] of 25165824 sz 2097152 [D 20:01:06.458964] add a segment 0 [D 20:01:06.458988] retval = 2097152 [D 20:01:06.459012] exiting distribute returned less than expected [D 20:01:06.459035] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:06.459061] *****RETURN***FROM***PROCESS***** [D 20:01:06.459086] end iteration [D 20:01:06.459110] done with segments or bytes [D 20:01:06.459134] of 295698432 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.459159] next loff: 295698432 next poff: 295698432 [D 20:01:06.459184] return value 2097152 [D 20:01:06.459208] finished [D 20:01:06.459232] retval = 2097152 [D 20:01:06.459257] exiting distribute returned less than expected [D 20:01:06.459280] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 295698432 fo 301989888 eof 0 [D 20:01:06.459307] ========================================================= [D 20:01:06.459331] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:06.459357] element 0: offset: 0x93c41000, size: 2097152 [D 20:01:06.459386] post_sr: da13:3346 bh 15 len 48 wr_id 15e72ce8. [D 20:01:06.459454] openib_check_cq: completed send wr_id 15e72ce8. [D 20:01:06.460422] openib_check_cq: completed recv wr_id 15e735f0. [D 20:01:06.460449] openib_post_rr: da13:3346 bh 11 wr_id 15e735f0. [D 20:01:06.460480] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:06.460506] ========================================================= [D 20:01:06.460531] PINT_process_request [D 20:01:06.460555] tiling 8 copies [D 20:01:06.460579] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.460606] lvl 0 el 0 blk 0 by 228589568 [D 20:01:06.460631] to 228589568 ta 201326592 fi 234881024 [D 20:01:06.460656] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:06.460680] basic type or contiguous data [D 20:01:06.460704] Request disp [D 20:01:06.460728] contig_offset = 228589568 contig_size = 39845888 lvl_flag = 1 [D 20:01:06.460754] calling distribute [D 20:01:06.460778] PINT_distribute [D 20:01:06.460803] of 228589568 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.460828] begin iteration loff: 228589568 [D 20:01:06.460854] segment exceeds byte limit [D 20:01:06.460878] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:06.460903] **********CALL***PROCESS********* [D 20:01:06.460927] segment of 27262976 sz 2097152 [D 20:01:06.460951] PINT_process_request [D 20:01:06.460975] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.461001] lvl 0 el 0 blk 0 by 27262976 [D 20:01:06.461026] to 27262976 ta 27262976 fi 29360128 [D 20:01:06.461050] basic type or contiguous data [D 20:01:06.461073] Request disp [D 20:01:06.461097] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:06.461122] process a segment [D 20:01:06.461146] of 27262976 sz 2097152 [D 20:01:06.461171] add a segment 0 [D 20:01:06.461194] retval = 2097152 [D 20:01:06.461218] exiting distribute returned less than expected [D 20:01:06.461242] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:06.461268] *****RETURN***FROM***PROCESS***** [D 20:01:06.461293] end iteration [D 20:01:06.461317] done with segments or bytes [D 20:01:06.461342] of 230686720 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.461369] next loff: 230686720 next poff: 230686720 [D 20:01:06.461394] return value 2097152 [D 20:01:06.461419] finished [D 20:01:06.461442] retval = 2097152 [D 20:01:06.461466] exiting distribute returned less than expected [D 20:01:06.461490] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 230686720 fo 234881024 eof 0 [D 20:01:06.461516] ========================================================= [D 20:01:06.461541] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:06.461566] element 0: offset: 0x97e45000, size: 2097152 [D 20:01:06.461596] post_sr: da13:3346 bh 16 len 48 wr_id 15e72d00. [D 20:01:06.461667] openib_check_cq: completed send wr_id 15e72d00. [D 20:01:06.462648] openib_check_cq: completed recv wr_id 15e73608. [D 20:01:06.462676] openib_post_rr: da13:3346 bh 12 wr_id 15e73608. [D 20:01:06.462702] openib_check_cq: completed recv wr_id 15e73620. [D 20:01:06.462728] openib_post_rr: da13:3346 bh 13 wr_id 15e73620. [D 20:01:06.462759] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:06.462784] ========================================================= [D 20:01:06.462809] PINT_process_request [D 20:01:06.462834] tiling 6 copies [D 20:01:06.462858] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.462883] lvl 0 el 0 blk 0 by 163577856 [D 20:01:06.462909] to 163577856 ta 134217728 fi 167772160 [D 20:01:06.462934] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:06.462959] basic type or contiguous data [D 20:01:06.462983] Request disp [D 20:01:06.463006] contig_offset = 163577856 contig_size = 37748736 lvl_flag = 1 [D 20:01:06.463031] calling distribute [D 20:01:06.463056] PINT_distribute [D 20:01:06.463079] of 163577856 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.463106] begin iteration loff: 163577856 [D 20:01:06.463130] segment exceeds byte limit [D 20:01:06.463154] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:06.463179] **********CALL***PROCESS********* [D 20:01:06.463203] segment of 29360128 sz 2097152 [D 20:01:06.463228] PINT_process_request [D 20:01:06.463252] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.463277] lvl 0 el 0 blk 0 by 29360128 [D 20:01:06.463302] to 29360128 ta 29360128 fi 31457280 [D 20:01:06.463327] basic type or contiguous data [D 20:01:06.463351] Request disp [D 20:01:06.463375] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:06.463401] process a segment [D 20:01:06.463426] of 29360128 sz 2097152 [D 20:01:06.463450] add a segment 0 [D 20:01:06.463474] retval = 2097152 [D 20:01:06.463498] exiting distribute returned less than expected [D 20:01:06.463521] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:06.463548] *****RETURN***FROM***PROCESS***** [D 20:01:06.463572] end iteration [D 20:01:06.463596] done with segments or bytes [D 20:01:06.463620] of 165675008 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.463645] next loff: 165675008 next poff: 165675008 [D 20:01:06.463670] return value 2097152 [D 20:01:06.463694] finished [D 20:01:06.463718] retval = 2097152 [D 20:01:06.463743] exiting distribute returned less than expected [D 20:01:06.463767] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 165675008 fo 167772160 eof 0 [D 20:01:06.463792] ========================================================= [D 20:01:06.463816] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:06.463842] element 0: offset: 0x9c049000, size: 2097152 [D 20:01:06.463871] post_sr: da13:3346 bh 17 len 48 wr_id 15e72d18. [D 20:01:06.463898] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:06.463924] ========================================================= [D 20:01:06.463949] PINT_process_request [D 20:01:06.463973] tiling 12 copies [D 20:01:06.464008] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.464035] lvl 0 el 0 blk 0 by 360710144 [D 20:01:06.464059] to 360710144 ta 335544320 fi 369098752 [D 20:01:06.464084] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:06.464109] basic type or contiguous data [D 20:01:06.464134] Request disp [D 20:01:06.464158] contig_offset = 360710144 contig_size = 41943040 lvl_flag = 1 [D 20:01:06.464182] calling distribute [D 20:01:06.464206] PINT_distribute [D 20:01:06.464233] of 360710144 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.464260] begin iteration loff: 360710144 [D 20:01:06.464284] segment exceeds byte limit [D 20:01:06.464309] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:06.464334] **********CALL***PROCESS********* [D 20:01:06.464358] segment of 25165824 sz 2097152 [D 20:01:06.464383] PINT_process_request [D 20:01:06.464406] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.464432] lvl 0 el 0 blk 0 by 25165824 [D 20:01:06.464457] to 25165824 ta 25165824 fi 27262976 [D 20:01:06.464482] basic type or contiguous data [D 20:01:06.464506] Request disp [D 20:01:06.464529] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:06.464554] process a segment [D 20:01:06.464578] of 25165824 sz 2097152 [D 20:01:06.464603] add a segment 0 [D 20:01:06.464627] retval = 2097152 [D 20:01:06.464651] exiting distribute returned less than expected [D 20:01:06.464674] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:06.464701] *****RETURN***FROM***PROCESS***** [D 20:01:06.464725] end iteration [D 20:01:06.464749] done with segments or bytes [D 20:01:06.464773] of 362807296 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.464798] next loff: 362807296 next poff: 362807296 [D 20:01:06.464823] return value 2097152 [D 20:01:06.464848] finished [D 20:01:06.464872] retval = 2097152 [D 20:01:06.464896] exiting distribute returned less than expected [D 20:01:06.464920] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 362807296 fo 369098752 eof 0 [D 20:01:06.464946] ========================================================= [D 20:01:06.464970] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:06.464996] element 0: offset: 0x8fc3d000, size: 2097152 [D 20:01:06.465023] openib_check_cq: completed send wr_id 15e72d18. [D 20:01:06.465048] openib_check_cq: completed recv wr_id 15e73638. [D 20:01:06.465074] openib_post_rr: da13:3346 bh 14 wr_id 15e73638. [D 20:01:06.465100] openib_check_cq: completed recv wr_id 15e73650. [D 20:01:06.465126] openib_post_rr: da13:3346 bh 15 wr_id 15e73650. [D 20:01:06.465156] post_sr: da13:3346 bh 18 len 48 wr_id 15e72d30. [D 20:01:06.465188] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:06.465215] ========================================================= [D 20:01:06.465239] PINT_process_request [D 20:01:06.465264] tiling 7 copies [D 20:01:06.465288] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.465314] lvl 0 el 0 blk 0 by 195035136 [D 20:01:06.465339] to 195035136 ta 167772160 fi 201326592 [D 20:01:06.465364] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:06.465389] basic type or contiguous data [D 20:01:06.465413] Request disp [D 20:01:06.465436] contig_offset = 195035136 contig_size = 39845888 lvl_flag = 1 [D 20:01:06.465461] calling distribute [D 20:01:06.465485] PINT_distribute [D 20:01:06.465509] of 195035136 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.465536] begin iteration loff: 195035136 [D 20:01:06.465560] segment exceeds byte limit [D 20:01:06.465584] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:06.465609] **********CALL***PROCESS********* [D 20:01:06.465633] segment of 27262976 sz 2097152 [D 20:01:06.465658] PINT_process_request [D 20:01:06.465682] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.465710] lvl 0 el 0 blk 0 by 27262976 [D 20:01:06.465736] to 27262976 ta 27262976 fi 29360128 [D 20:01:06.465761] basic type or contiguous data [D 20:01:06.465785] Request disp [D 20:01:06.465808] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:06.465834] process a segment [D 20:01:06.465858] of 27262976 sz 2097152 [D 20:01:06.465882] add a segment 0 [D 20:01:06.465907] retval = 2097152 [D 20:01:06.465930] exiting distribute returned less than expected [D 20:01:06.465954] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:06.465981] *****RETURN***FROM***PROCESS***** [D 20:01:06.466005] end iteration [D 20:01:06.466029] done with segments or bytes [D 20:01:06.466053] of 197132288 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.466078] next loff: 197132288 next poff: 197132288 [D 20:01:06.466103] return value 2097152 [D 20:01:06.466128] finished [D 20:01:06.466152] retval = 2097152 [D 20:01:06.466176] exiting distribute returned less than expected [D 20:01:06.466199] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 197132288 fo 201326592 eof 0 [D 20:01:06.466230] ========================================================= [D 20:01:06.466259] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:06.466288] element 0: offset: 0x99e47000, size: 2097152 [D 20:01:06.466318] openib_check_cq: completed send wr_id 15e72d30. [D 20:01:06.466347] post_sr: da13:3346 bh 19 len 48 wr_id 15e72d48. [D 20:01:06.466375] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:06.466401] ========================================================= [D 20:01:06.466424] PINT_process_request [D 20:01:06.466448] tiling 13 copies [D 20:01:06.466473] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.466499] lvl 0 el 0 blk 0 by 394264576 [D 20:01:06.466524] to 394264576 ta 369098752 fi 402653184 [D 20:01:06.466549] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:06.466574] basic type or contiguous data [D 20:01:06.466598] Request disp [D 20:01:06.466622] contig_offset = 394264576 contig_size = 41943040 lvl_flag = 1 [D 20:01:06.466647] calling distribute [D 20:01:06.466670] PINT_distribute [D 20:01:06.466694] of 394264576 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.466721] begin iteration loff: 394264576 [D 20:01:06.466746] segment exceeds byte limit [D 20:01:06.466771] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:06.466795] **********CALL***PROCESS********* [D 20:01:06.466819] segment of 25165824 sz 2097152 [D 20:01:06.466844] PINT_process_request [D 20:01:06.466868] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.466894] lvl 0 el 0 blk 0 by 25165824 [D 20:01:06.466918] to 25165824 ta 25165824 fi 27262976 [D 20:01:06.466943] basic type or contiguous data [D 20:01:06.466967] Request disp [D 20:01:06.466991] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:06.467017] process a segment [D 20:01:06.467041] of 25165824 sz 2097152 [D 20:01:06.467064] add a segment 0 [D 20:01:06.467088] retval = 2097152 [D 20:01:06.467113] exiting distribute returned less than expected [D 20:01:06.467137] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:06.467163] *****RETURN***FROM***PROCESS***** [D 20:01:06.467187] end iteration [D 20:01:06.467211] done with segments or bytes [D 20:01:06.467235] of 396361728 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.467261] next loff: 396361728 next poff: 396361728 [D 20:01:06.467286] return value 2097152 [D 20:01:06.467309] finished [D 20:01:06.467333] retval = 2097152 [D 20:01:06.467357] exiting distribute returned less than expected [D 20:01:06.467381] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 396361728 fo 402653184 eof 0 [D 20:01:06.467408] ========================================================= [D 20:01:06.467432] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:06.467459] element 0: offset: 0x8dc3b000, size: 2097152 [D 20:01:06.467486] openib_check_cq: completed send wr_id 15e72d48. [D 20:01:06.467512] openib_check_cq: completed recv wr_id 15e73668. [D 20:01:06.467537] openib_post_rr: da13:3346 bh 16 wr_id 15e73668. [D 20:01:06.467568] post_sr: da13:3346 bh 20 len 48 wr_id 15e72d60. [D 20:01:06.467596] PINT_client_state_machine_test id 79 [D 20:01:06.467628] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:06.467654] ========================================================= [D 20:01:06.467679] PINT_process_request [D 20:01:06.467702] tiling 11 copies [D 20:01:06.467726] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.467753] lvl 0 el 0 blk 0 by 327155712 [D 20:01:06.467778] to 327155712 ta 301989888 fi 335544320 [D 20:01:06.467803] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:06.467827] basic type or contiguous data [D 20:01:06.467851] Request disp [D 20:01:06.467876] contig_offset = 327155712 contig_size = 41943040 lvl_flag = 1 [D 20:01:06.467901] calling distribute [D 20:01:06.467925] PINT_distribute [D 20:01:06.467948] of 327155712 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.467975] begin iteration loff: 327155712 [D 20:01:06.468000] segment exceeds byte limit [D 20:01:06.468025] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:06.468050] **********CALL***PROCESS********* [D 20:01:06.468073] segment of 25165824 sz 2097152 [D 20:01:06.468098] PINT_process_request [D 20:01:06.468122] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.468148] lvl 0 el 0 blk 0 by 25165824 [D 20:01:06.468173] to 25165824 ta 25165824 fi 27262976 [D 20:01:06.468197] basic type or contiguous data [D 20:01:06.468224] Request disp [D 20:01:06.468248] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:06.468273] process a segment [D 20:01:06.468297] of 25165824 sz 2097152 [D 20:01:06.468320] add a segment 0 [D 20:01:06.468344] retval = 2097152 [D 20:01:06.468368] exiting distribute returned less than expected [D 20:01:06.468392] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:06.468419] *****RETURN***FROM***PROCESS***** [D 20:01:06.468442] end iteration [D 20:01:06.468466] done with segments or bytes [D 20:01:06.468490] of 329252864 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.468516] next loff: 329252864 next poff: 329252864 [D 20:01:06.468541] return value 2097152 [D 20:01:06.468564] finished [D 20:01:06.468589] retval = 2097152 [D 20:01:06.468613] exiting distribute returned less than expected [D 20:01:06.468637] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 329252864 fo 335544320 eof 0 [D 20:01:06.468664] ========================================================= [D 20:01:06.468688] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:06.468713] element 0: offset: 0x91c3f000, size: 2097152 [D 20:01:06.468739] openib_check_cq: completed send wr_id 15e72d60. [D 20:01:06.468769] post_sr: da13:3346 bh 21 len 48 wr_id 15e72d78. [D 20:01:06.468838] openib_check_cq: completed send wr_id 15e72d78. [D 20:01:06.469333] openib_check_cq: completed recv wr_id 15e73680. [D 20:01:06.469360] openib_post_rr: da13:3346 bh 17 wr_id 15e73680. [D 20:01:06.469390] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:06.469416] ========================================================= [D 20:01:06.469441] PINT_process_request [D 20:01:06.469465] tiling 14 copies [D 20:01:06.469490] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.469515] lvl 0 el 0 blk 0 by 427819008 [D 20:01:06.469540] to 427819008 ta 402653184 fi 436207616 [D 20:01:06.469565] mto 25165824 mta 23068672 mfi 25165824 [D 20:01:06.469590] basic type or contiguous data [D 20:01:06.469614] Request disp [D 20:01:06.469637] contig_offset = 427819008 contig_size = 41943040 lvl_flag = 1 [D 20:01:06.469665] calling distribute [D 20:01:06.469690] PINT_distribute [D 20:01:06.469714] of 427819008 sz 8388608 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.469740] begin iteration loff: 427819008 [D 20:01:06.469765] segment exceeds byte limit [D 20:01:06.469789] client lstof 25165824 diff 0 sgof 25165824 [D 20:01:06.469814] **********CALL***PROCESS********* [D 20:01:06.469838] segment of 25165824 sz 2097152 [D 20:01:06.469862] PINT_process_request [D 20:01:06.469886] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.469912] lvl 0 el 0 blk 0 by 25165824 [D 20:01:06.469937] to 25165824 ta 25165824 fi 27262976 [D 20:01:06.469962] basic type or contiguous data [D 20:01:06.469986] Request disp [D 20:01:06.470010] contig_offset = 25165824 contig_size = 8388608 lvl_flag = 1 [D 20:01:06.470035] process a segment [D 20:01:06.470058] of 25165824 sz 2097152 [D 20:01:06.470083] add a segment 0 [D 20:01:06.470107] retval = 2097152 [D 20:01:06.470131] exiting distribute returned less than expected [D 20:01:06.470155] done sg 1 sm 64 by 2097152 bm 2097152 ta 25165824 to 27262976 fo 27262976 eof 0 [D 20:01:06.470181] *****RETURN***FROM***PROCESS***** [D 20:01:06.470205] end iteration [D 20:01:06.470233] done with segments or bytes [D 20:01:06.470261] of 429916160 sz 6291456 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.470291] next loff: 429916160 next poff: 429916160 [D 20:01:06.470319] return value 2097152 [D 20:01:06.470344] finished [D 20:01:06.470368] retval = 2097152 [D 20:01:06.470392] exiting distribute returned less than expected [D 20:01:06.470416] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 429916160 fo 436207616 eof 0 [D 20:01:06.470442] ========================================================= [D 20:01:06.470467] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:06.470492] element 0: offset: 0x8bc39000, size: 2097152 [D 20:01:06.470521] post_sr: da13:3346 bh 22 len 48 wr_id 15e72d90. [D 20:01:06.470591] openib_check_cq: completed send wr_id 15e72d90. [D 20:01:06.471557] openib_check_cq: completed recv wr_id 15e73698. [D 20:01:06.471584] openib_post_rr: da13:3346 bh 18 wr_id 15e73698. [D 20:01:06.471614] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:06.471640] ========================================================= [D 20:01:06.471664] PINT_process_request [D 20:01:06.471688] tiling 9 copies [D 20:01:06.471713] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.471738] lvl 0 el 0 blk 0 by 262144000 [D 20:01:06.471763] to 262144000 ta 234881024 fi 268435456 [D 20:01:06.471788] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:06.471813] basic type or contiguous data [D 20:01:06.471837] Request disp [D 20:01:06.471860] contig_offset = 262144000 contig_size = 39845888 lvl_flag = 1 [D 20:01:06.471885] calling distribute [D 20:01:06.471910] PINT_distribute [D 20:01:06.471934] of 262144000 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.471961] begin iteration loff: 262144000 [D 20:01:06.471985] segment exceeds byte limit [D 20:01:06.472009] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:06.472034] **********CALL***PROCESS********* [D 20:01:06.472058] segment of 27262976 sz 2097152 [D 20:01:06.472083] PINT_process_request [D 20:01:06.472106] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.472132] lvl 0 el 0 blk 0 by 27262976 [D 20:01:06.472157] to 27262976 ta 27262976 fi 29360128 [D 20:01:06.472182] basic type or contiguous data [D 20:01:06.472206] Request disp [D 20:01:06.472232] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:06.472257] process a segment [D 20:01:06.472281] of 27262976 sz 2097152 [D 20:01:06.472305] add a segment 0 [D 20:01:06.472330] retval = 2097152 [D 20:01:06.472353] exiting distribute returned less than expected [D 20:01:06.472380] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:06.472406] *****RETURN***FROM***PROCESS***** [D 20:01:06.472431] end iteration [D 20:01:06.472455] done with segments or bytes [D 20:01:06.472477] of 264241152 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.472504] next loff: 264241152 next poff: 264241152 [D 20:01:06.472528] return value 2097152 [D 20:01:06.472553] finished [D 20:01:06.472577] retval = 2097152 [D 20:01:06.472600] exiting distribute returned less than expected [D 20:01:06.472624] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 264241152 fo 268435456 eof 0 [D 20:01:06.472651] ========================================================= [D 20:01:06.472675] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:06.472701] element 0: offset: 0x95e43000, size: 2097152 [D 20:01:06.472729] post_sr: da13:3346 bh 23 len 48 wr_id 15e72da8. [D 20:01:06.472798] openib_check_cq: completed send wr_id 15e72da8. [D 20:01:06.476004] openib_check_cq: completed recv wr_id 15e736b0. [D 20:01:06.476032] openib_post_rr: da13:3346 bh 19 wr_id 15e736b0. [D 20:01:06.476062] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:06.476088] ========================================================= [D 20:01:06.476113] PINT_process_request [D 20:01:06.476137] tiling 10 copies [D 20:01:06.476161] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.476187] lvl 0 el 0 blk 0 by 295698432 [D 20:01:06.476215] to 295698432 ta 268435456 fi 301989888 [D 20:01:06.476240] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:06.476265] basic type or contiguous data [D 20:01:06.476289] Request disp [D 20:01:06.476313] contig_offset = 295698432 contig_size = 39845888 lvl_flag = 1 [D 20:01:06.476339] calling distribute [D 20:01:06.476363] PINT_distribute [D 20:01:06.476387] of 295698432 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.476413] begin iteration loff: 295698432 [D 20:01:06.476438] segment exceeds byte limit [D 20:01:06.476463] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:06.476488] **********CALL***PROCESS********* [D 20:01:06.476511] segment of 27262976 sz 2097152 [D 20:01:06.476536] PINT_process_request [D 20:01:06.476561] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.476587] lvl 0 el 0 blk 0 by 27262976 [D 20:01:06.476613] to 27262976 ta 27262976 fi 29360128 [D 20:01:06.476637] basic type or contiguous data [D 20:01:06.476661] Request disp [D 20:01:06.476685] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:06.476710] process a segment [D 20:01:06.476735] of 27262976 sz 2097152 [D 20:01:06.476758] add a segment 0 [D 20:01:06.476782] retval = 2097152 [D 20:01:06.476807] exiting distribute returned less than expected [D 20:01:06.476831] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:06.476858] *****RETURN***FROM***PROCESS***** [D 20:01:06.476881] end iteration [D 20:01:06.476905] done with segments or bytes [D 20:01:06.476929] of 297795584 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.476955] next loff: 297795584 next poff: 297795584 [D 20:01:06.476981] return value 2097152 [D 20:01:06.477005] finished [D 20:01:06.477028] retval = 2097152 [D 20:01:06.477053] exiting distribute returned less than expected [D 20:01:06.477077] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 297795584 fo 301989888 eof 0 [D 20:01:06.477104] ========================================================= [D 20:01:06.477128] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:06.477153] element 0: offset: 0x93e41000, size: 2097152 [D 20:01:06.477182] post_sr: da13:3346 bh 24 len 48 wr_id 15e72dc0. [D 20:01:06.477254] openib_check_cq: completed send wr_id 15e72dc0. [D 20:01:06.477624] PINT_client_state_machine_test id 79 [D 20:01:06.480451] openib_check_cq: completed recv wr_id 15e736c8. [D 20:01:06.480482] openib_post_rr: da13:3346 bh 20 wr_id 15e736c8. [D 20:01:06.480512] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:06.480538] ========================================================= [D 20:01:06.480563] PINT_process_request [D 20:01:06.480587] tiling 8 copies [D 20:01:06.480611] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.480638] lvl 0 el 0 blk 0 by 230686720 [D 20:01:06.480663] to 230686720 ta 201326592 fi 234881024 [D 20:01:06.480688] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:06.480712] basic type or contiguous data [D 20:01:06.480736] Request disp [D 20:01:06.480760] contig_offset = 230686720 contig_size = 37748736 lvl_flag = 1 [D 20:01:06.480786] calling distribute [D 20:01:06.480810] PINT_distribute [D 20:01:06.480834] of 230686720 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.480860] begin iteration loff: 230686720 [D 20:01:06.480885] segment exceeds byte limit [D 20:01:06.480909] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:06.480934] **********CALL***PROCESS********* [D 20:01:06.480957] segment of 29360128 sz 2097152 [D 20:01:06.480982] PINT_process_request [D 20:01:06.481006] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.481032] lvl 0 el 0 blk 0 by 29360128 [D 20:01:06.481058] to 29360128 ta 29360128 fi 31457280 [D 20:01:06.481081] basic type or contiguous data [D 20:01:06.481105] Request disp [D 20:01:06.481129] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:06.481154] process a segment [D 20:01:06.481178] of 29360128 sz 2097152 [D 20:01:06.481203] add a segment 0 [D 20:01:06.481226] retval = 2097152 [D 20:01:06.481250] exiting distribute returned less than expected [D 20:01:06.481274] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:06.481301] *****RETURN***FROM***PROCESS***** [D 20:01:06.481325] end iteration [D 20:01:06.481348] done with segments or bytes [D 20:01:06.481372] of 232783872 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.481398] next loff: 232783872 next poff: 232783872 [D 20:01:06.481423] return value 2097152 [D 20:01:06.481447] finished [D 20:01:06.481471] retval = 2097152 [D 20:01:06.481495] exiting distribute returned less than expected [D 20:01:06.481519] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 232783872 fo 234881024 eof 0 [D 20:01:06.481545] ========================================================= [D 20:01:06.481570] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:06.481595] element 0: offset: 0x98045000, size: 2097152 [D 20:01:06.481624] post_sr: da13:3346 bh 25 len 48 wr_id 15e72dd8. [D 20:01:06.481694] openib_check_cq: completed send wr_id 15e72dd8. [D 20:01:06.482680] openib_check_cq: completed recv wr_id 15e736e0. [D 20:01:06.482707] openib_post_rr: da13:3346 bh 21 wr_id 15e736e0. [D 20:01:06.482737] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:06.482763] ========================================================= [D 20:01:06.482788] PINT_process_request [D 20:01:06.482811] tiling 6 copies [D 20:01:06.482835] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.482862] lvl 0 el 0 blk 0 by 165675008 [D 20:01:06.482887] to 165675008 ta 134217728 fi 167772160 [D 20:01:06.482912] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:06.482936] basic type or contiguous data [D 20:01:06.482960] Request disp [D 20:01:06.482984] contig_offset = 165675008 contig_size = 35651584 lvl_flag = 1 [D 20:01:06.483009] calling distribute [D 20:01:06.483034] PINT_distribute [D 20:01:06.483057] of 165675008 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.483083] begin iteration loff: 165675008 [D 20:01:06.483108] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:06.483133] **********CALL***PROCESS********* [D 20:01:06.483157] segment of 31457280 sz 2097152 [D 20:01:06.483184] PINT_process_request [D 20:01:06.483208] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.483235] lvl 0 el 0 blk 0 by 31457280 [D 20:01:06.483260] to 31457280 ta 31457280 fi 33554432 [D 20:01:06.483284] basic type or contiguous data [D 20:01:06.483308] Request disp [D 20:01:06.483331] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:06.483356] process a segment [D 20:01:06.483380] of 31457280 sz 2097152 [D 20:01:06.483407] add a segment 0 [D 20:01:06.483430] retval = 2097152 [D 20:01:06.483454] return from level -1 [D 20:01:06.483478] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:06.483505] *****RETURN***FROM***PROCESS***** [D 20:01:06.483530] end iteration [D 20:01:06.483552] done with segments or bytes [D 20:01:06.483576] of 167772160 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.483603] next loff: 167772160 next poff: 167772160 [D 20:01:06.483627] (return value) 2097152 [D 20:01:06.483652] finished [D 20:01:06.483675] retval = 2097152 [D 20:01:06.483699] exiting distribute returned less than expected [D 20:01:06.483724] done sg 1 sm 64 by 2097152 bm 2097152 ta 134217728 to 167772160 fo 167772160 eof 0 [D 20:01:06.483750] ========================================================= [D 20:01:06.483774] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 27 [D 20:01:06.483799] element 0: offset: 0x9c249000, size: 2097152 [D 20:01:06.483828] post_sr: da13:3346 bh 26 len 48 wr_id 15e72df0. [D 20:01:06.483897] openib_check_cq: completed send wr_id 15e72df0. [D 20:01:06.484905] openib_check_cq: completed recv wr_id 15e736f8. [D 20:01:06.484932] openib_post_rr: da13:3346 bh 22 wr_id 15e736f8. [D 20:01:06.484959] openib_check_cq: completed recv wr_id 15e73710. [D 20:01:06.484985] openib_post_rr: da13:3346 bh 23 wr_id 15e73710. [D 20:01:06.485014] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:06.485041] ========================================================= [D 20:01:06.485065] PINT_process_request [D 20:01:06.485089] tiling 7 copies [D 20:01:06.485113] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.485139] lvl 0 el 0 blk 0 by 197132288 [D 20:01:06.485164] to 197132288 ta 167772160 fi 201326592 [D 20:01:06.485189] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:06.485214] basic type or contiguous data [D 20:01:06.485238] Request disp [D 20:01:06.485261] contig_offset = 197132288 contig_size = 37748736 lvl_flag = 1 [D 20:01:06.485286] calling distribute [D 20:01:06.485310] PINT_distribute [D 20:01:06.485334] of 197132288 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.485360] begin iteration loff: 197132288 [D 20:01:06.485384] segment exceeds byte limit [D 20:01:06.485408] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:06.485433] **********CALL***PROCESS********* [D 20:01:06.485457] segment of 29360128 sz 2097152 [D 20:01:06.485482] PINT_process_request [D 20:01:06.485505] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.485531] lvl 0 el 0 blk 0 by 29360128 [D 20:01:06.485556] to 29360128 ta 29360128 fi 31457280 [D 20:01:06.485581] basic type or contiguous data [D 20:01:06.485605] Request disp [D 20:01:06.485629] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:06.485653] process a segment [D 20:01:06.485677] of 29360128 sz 2097152 [D 20:01:06.485701] add a segment 0 [D 20:01:06.485725] retval = 2097152 [D 20:01:06.485749] exiting distribute returned less than expected [D 20:01:06.485774] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:06.485799] *****RETURN***FROM***PROCESS***** [D 20:01:06.485823] end iteration [D 20:01:06.485847] done with segments or bytes [D 20:01:06.485871] of 199229440 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.485897] next loff: 199229440 next poff: 199229440 [D 20:01:06.485924] return value 2097152 [D 20:01:06.485949] finished [D 20:01:06.485973] retval = 2097152 [D 20:01:06.485997] exiting distribute returned less than expected [D 20:01:06.486021] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 199229440 fo 201326592 eof 0 [D 20:01:06.486046] ========================================================= [D 20:01:06.486071] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:06.486097] element 0: offset: 0x9a047000, size: 2097152 [D 20:01:06.486125] post_sr: da13:3346 bh 27 len 48 wr_id 15e72e08. [D 20:01:06.486153] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:06.486178] ========================================================= [D 20:01:06.486203] PINT_process_request [D 20:01:06.486230] tiling 12 copies [D 20:01:06.486258] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.486288] lvl 0 el 0 blk 0 by 362807296 [D 20:01:06.486317] to 362807296 ta 335544320 fi 369098752 [D 20:01:06.486342] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:06.486368] basic type or contiguous data [D 20:01:06.486392] Request disp [D 20:01:06.486415] contig_offset = 362807296 contig_size = 39845888 lvl_flag = 1 [D 20:01:06.486440] calling distribute [D 20:01:06.486464] PINT_distribute [D 20:01:06.486488] of 362807296 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.486515] begin iteration loff: 362807296 [D 20:01:06.486539] segment exceeds byte limit [D 20:01:06.486564] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:06.486589] **********CALL***PROCESS********* [D 20:01:06.486613] segment of 27262976 sz 2097152 [D 20:01:06.486638] PINT_process_request [D 20:01:06.486661] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.486687] lvl 0 el 0 blk 0 by 27262976 [D 20:01:06.486713] to 27262976 ta 27262976 fi 29360128 [D 20:01:06.486738] basic type or contiguous data [D 20:01:06.486762] Request disp [D 20:01:06.486785] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:06.486810] process a segment [D 20:01:06.486839] of 27262976 sz 2097152 [D 20:01:06.486864] add a segment 0 [D 20:01:06.486887] retval = 2097152 [D 20:01:06.486911] exiting distribute returned less than expected [D 20:01:06.486935] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:06.486961] *****RETURN***FROM***PROCESS***** [D 20:01:06.486986] end iteration [D 20:01:06.487010] done with segments or bytes [D 20:01:06.487033] of 364904448 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.487058] next loff: 364904448 next poff: 364904448 [D 20:01:06.487083] return value 2097152 [D 20:01:06.487108] finished [D 20:01:06.487132] retval = 2097152 [D 20:01:06.487155] exiting distribute returned less than expected [D 20:01:06.487179] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 364904448 fo 369098752 eof 0 [D 20:01:06.487205] ========================================================= [D 20:01:06.487230] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:06.487255] element 0: offset: 0x8fe3d000, size: 2097152 [D 20:01:06.487281] openib_check_cq: completed send wr_id 15e72e08. [D 20:01:06.487307] openib_check_cq: completed recv wr_id 15e73728. [D 20:01:06.487334] openib_post_rr: da13:3346 bh 24 wr_id 15e73728. [D 20:01:06.487360] openib_check_cq: completed recv wr_id 15e73740. [D 20:01:06.487386] openib_post_rr: da13:3346 bh 25 wr_id 15e73740. [D 20:01:06.487417] post_sr: da13:3346 bh 28 len 48 wr_id 15e72e20. [D 20:01:06.487449] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:06.487475] ========================================================= [D 20:01:06.487500] PINT_process_request [D 20:01:06.487524] tiling 11 copies [D 20:01:06.487548] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.487574] lvl 0 el 0 blk 0 by 329252864 [D 20:01:06.487600] to 329252864 ta 301989888 fi 335544320 [D 20:01:06.487627] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:06.487652] basic type or contiguous data [D 20:01:06.487676] Request disp [D 20:01:06.487700] contig_offset = 329252864 contig_size = 39845888 lvl_flag = 1 [D 20:01:06.487726] calling distribute [D 20:01:06.487750] PINT_distribute [D 20:01:06.487773] of 329252864 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.487800] begin iteration loff: 329252864 [D 20:01:06.487825] segment exceeds byte limit [D 20:01:06.487850] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:06.487874] **********CALL***PROCESS********* [D 20:01:06.487898] segment of 27262976 sz 2097152 [D 20:01:06.487922] PINT_process_request [D 20:01:06.487946] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.487973] lvl 0 el 0 blk 0 by 27262976 [D 20:01:06.487998] to 27262976 ta 27262976 fi 29360128 [D 20:01:06.488022] basic type or contiguous data [D 20:01:06.488046] Request disp [D 20:01:06.488070] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:06.488095] process a segment [D 20:01:06.488119] of 27262976 sz 2097152 [D 20:01:06.488143] add a segment 0 [D 20:01:06.488167] retval = 2097152 [D 20:01:06.488191] exiting distribute returned less than expected [D 20:01:06.488218] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:06.488245] *****RETURN***FROM***PROCESS***** [D 20:01:06.488268] end iteration [D 20:01:06.488292] done with segments or bytes [D 20:01:06.488316] of 331350016 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.488342] next loff: 331350016 next poff: 331350016 [D 20:01:06.488367] return value 2097152 [D 20:01:06.488391] finished [D 20:01:06.488415] retval = 2097152 [D 20:01:06.488439] exiting distribute returned less than expected [D 20:01:06.488463] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 331350016 fo 335544320 eof 0 [D 20:01:06.488489] ========================================================= [D 20:01:06.488513] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:06.488539] element 0: offset: 0x91e3f000, size: 2097152 [D 20:01:06.488565] openib_check_cq: completed send wr_id 15e72e20. [D 20:01:06.488594] post_sr: da13:3346 bh 29 len 48 wr_id 15e72e38. [D 20:01:06.488621] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:06.488646] ========================================================= [D 20:01:06.488671] PINT_process_request [D 20:01:06.488696] tiling 13 copies [D 20:01:06.488719] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.488745] lvl 0 el 0 blk 0 by 396361728 [D 20:01:06.488771] to 396361728 ta 369098752 fi 402653184 [D 20:01:06.488796] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:06.488821] basic type or contiguous data [D 20:01:06.488844] Request disp [D 20:01:06.488868] contig_offset = 396361728 contig_size = 39845888 lvl_flag = 1 [D 20:01:06.488893] calling distribute [D 20:01:06.488917] PINT_distribute [D 20:01:06.488941] of 396361728 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.488967] begin iteration loff: 396361728 [D 20:01:06.488992] segment exceeds byte limit [D 20:01:06.489016] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:06.489041] **********CALL***PROCESS********* [D 20:01:06.489065] segment of 27262976 sz 2097152 [D 20:01:06.489089] PINT_process_request [D 20:01:06.489113] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.489140] lvl 0 el 0 blk 0 by 27262976 [D 20:01:06.489165] to 27262976 ta 27262976 fi 29360128 [D 20:01:06.489190] basic type or contiguous data [D 20:01:06.489213] Request disp [D 20:01:06.489237] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:06.489263] process a segment [D 20:01:06.489287] of 27262976 sz 2097152 [D 20:01:06.489311] add a segment 0 [D 20:01:06.489335] retval = 2097152 [D 20:01:06.489362] exiting distribute returned less than expected [D 20:01:06.489388] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:06.489414] *****RETURN***FROM***PROCESS***** [D 20:01:06.489439] end iteration [D 20:01:06.489462] done with segments or bytes [D 20:01:06.489486] of 398458880 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.489512] next loff: 398458880 next poff: 398458880 [D 20:01:06.489537] return value 2097152 [D 20:01:06.489562] finished [D 20:01:06.489584] retval = 2097152 [D 20:01:06.489608] exiting distribute returned less than expected [D 20:01:06.489633] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 398458880 fo 402653184 eof 0 [D 20:01:06.489659] ========================================================= [D 20:01:06.489684] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:06.489708] element 0: offset: 0x8de3b000, size: 2097152 [D 20:01:06.489735] openib_check_cq: completed send wr_id 15e72e38. [D 20:01:06.489761] openib_check_cq: completed recv wr_id 15e73758. [D 20:01:06.489788] openib_post_rr: da13:3346 bh 26 wr_id 15e73758. [D 20:01:06.489817] post_sr: da13:3346 bh 30 len 48 wr_id 15e72e50. [D 20:01:06.489845] PINT_client_state_machine_test id 79 [D 20:01:06.489876] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:06.489903] ========================================================= [D 20:01:06.489927] PINT_process_request [D 20:01:06.489952] tiling 14 copies [D 20:01:06.489975] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.490002] lvl 0 el 0 blk 0 by 429916160 [D 20:01:06.490027] to 429916160 ta 402653184 fi 436207616 [D 20:01:06.490052] mto 27262976 mta 25165824 mfi 27262976 [D 20:01:06.490077] basic type or contiguous data [D 20:01:06.490100] Request disp [D 20:01:06.490124] contig_offset = 429916160 contig_size = 39845888 lvl_flag = 1 [D 20:01:06.490149] calling distribute [D 20:01:06.490173] PINT_distribute [D 20:01:06.490197] of 429916160 sz 6291456 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.490228] begin iteration loff: 429916160 [D 20:01:06.490257] segment exceeds byte limit [D 20:01:06.490284] client lstof 27262976 diff 0 sgof 27262976 [D 20:01:06.490312] **********CALL***PROCESS********* [D 20:01:06.490338] segment of 27262976 sz 2097152 [D 20:01:06.490363] PINT_process_request [D 20:01:06.490386] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.490413] lvl 0 el 0 blk 0 by 27262976 [D 20:01:06.490438] to 27262976 ta 27262976 fi 29360128 [D 20:01:06.490463] basic type or contiguous data [D 20:01:06.490487] Request disp [D 20:01:06.490510] contig_offset = 27262976 contig_size = 6291456 lvl_flag = 1 [D 20:01:06.490535] process a segment [D 20:01:06.490559] of 27262976 sz 2097152 [D 20:01:06.490583] add a segment 0 [D 20:01:06.490607] retval = 2097152 [D 20:01:06.490632] exiting distribute returned less than expected [D 20:01:06.490655] done sg 1 sm 64 by 2097152 bm 2097152 ta 27262976 to 29360128 fo 29360128 eof 0 [D 20:01:06.490681] *****RETURN***FROM***PROCESS***** [D 20:01:06.490706] end iteration [D 20:01:06.490730] done with segments or bytes [D 20:01:06.490754] of 432013312 sz 4194304 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.490779] next loff: 432013312 next poff: 432013312 [D 20:01:06.490804] return value 2097152 [D 20:01:06.490828] finished [D 20:01:06.490852] retval = 2097152 [D 20:01:06.490876] exiting distribute returned less than expected [D 20:01:06.490900] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 432013312 fo 436207616 eof 0 [D 20:01:06.490926] ========================================================= [D 20:01:06.490950] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:06.490976] element 0: offset: 0x8be39000, size: 2097152 [D 20:01:06.491003] openib_check_cq: completed send wr_id 15e72e50. [D 20:01:06.491031] post_sr: da13:3346 bh 31 len 48 wr_id 15e72e68. [D 20:01:06.491103] openib_check_cq: completed send wr_id 15e72e68. [D 20:01:06.491593] openib_check_cq: completed recv wr_id 15e73770. [D 20:01:06.491620] openib_post_rr: da13:3346 bh 27 wr_id 15e73770. [D 20:01:06.491649] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:06.491675] ========================================================= [D 20:01:06.491700] PINT_process_request [D 20:01:06.491724] tiling 9 copies [D 20:01:06.491747] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.491774] lvl 0 el 0 blk 0 by 264241152 [D 20:01:06.491799] to 264241152 ta 234881024 fi 268435456 [D 20:01:06.491824] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:06.491848] basic type or contiguous data [D 20:01:06.491872] Request disp [D 20:01:06.491896] contig_offset = 264241152 contig_size = 37748736 lvl_flag = 1 [D 20:01:06.491921] calling distribute [D 20:01:06.491945] PINT_distribute [D 20:01:06.491969] of 264241152 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.491995] begin iteration loff: 264241152 [D 20:01:06.492020] segment exceeds byte limit [D 20:01:06.492045] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:06.492070] **********CALL***PROCESS********* [D 20:01:06.492094] segment of 29360128 sz 2097152 [D 20:01:06.492117] PINT_process_request [D 20:01:06.492141] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.492168] lvl 0 el 0 blk 0 by 29360128 [D 20:01:06.492193] to 29360128 ta 29360128 fi 31457280 [D 20:01:06.492219] basic type or contiguous data [D 20:01:06.492244] Request disp [D 20:01:06.492268] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:06.492293] process a segment [D 20:01:06.492317] of 29360128 sz 2097152 [D 20:01:06.492341] add a segment 0 [D 20:01:06.492364] retval = 2097152 [D 20:01:06.492389] exiting distribute returned less than expected [D 20:01:06.492413] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:06.492439] *****RETURN***FROM***PROCESS***** [D 20:01:06.492464] end iteration [D 20:01:06.492487] done with segments or bytes [D 20:01:06.492510] of 266338304 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.492536] next loff: 266338304 next poff: 266338304 [D 20:01:06.492561] return value 2097152 [D 20:01:06.492586] finished [D 20:01:06.492609] retval = 2097152 [D 20:01:06.492633] exiting distribute returned less than expected [D 20:01:06.492657] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 266338304 fo 268435456 eof 0 [D 20:01:06.492683] ========================================================= [D 20:01:06.492707] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:06.492733] element 0: offset: 0x96043000, size: 2097152 [D 20:01:06.492762] post_sr: da13:3346 bh 32 len 48 wr_id 15e72e80. [D 20:01:06.492831] openib_check_cq: completed send wr_id 15e72e80. [D 20:01:06.493818] openib_check_cq: completed recv wr_id 15e73788. [D 20:01:06.493845] openib_post_rr: da13:3346 bh 28 wr_id 15e73788. [D 20:01:06.493875] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:06.493901] ========================================================= [D 20:01:06.493926] PINT_process_request [D 20:01:06.493949] tiling 10 copies [D 20:01:06.493973] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.494000] lvl 0 el 0 blk 0 by 297795584 [D 20:01:06.494024] to 297795584 ta 268435456 fi 301989888 [D 20:01:06.494049] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:06.494073] basic type or contiguous data [D 20:01:06.494097] Request disp [D 20:01:06.494122] contig_offset = 297795584 contig_size = 37748736 lvl_flag = 1 [D 20:01:06.494147] calling distribute [D 20:01:06.494171] PINT_distribute [D 20:01:06.494194] of 297795584 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.494225] begin iteration loff: 297795584 [D 20:01:06.494256] segment exceeds byte limit [D 20:01:06.494284] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:06.494313] **********CALL***PROCESS********* [D 20:01:06.494338] segment of 29360128 sz 2097152 [D 20:01:06.494363] PINT_process_request [D 20:01:06.494387] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.494414] lvl 0 el 0 blk 0 by 29360128 [D 20:01:06.494439] to 29360128 ta 29360128 fi 31457280 [D 20:01:06.494463] basic type or contiguous data [D 20:01:06.494487] Request disp [D 20:01:06.494511] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:06.494536] process a segment [D 20:01:06.494560] of 29360128 sz 2097152 [D 20:01:06.494585] add a segment 0 [D 20:01:06.494608] retval = 2097152 [D 20:01:06.494632] exiting distribute returned less than expected [D 20:01:06.494657] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:06.494683] *****RETURN***FROM***PROCESS***** [D 20:01:06.494708] end iteration [D 20:01:06.494731] done with segments or bytes [D 20:01:06.494755] of 299892736 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.494781] next loff: 299892736 next poff: 299892736 [D 20:01:06.494805] return value 2097152 [D 20:01:06.494830] finished [D 20:01:06.494853] retval = 2097152 [D 20:01:06.494877] exiting distribute returned less than expected [D 20:01:06.494901] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 299892736 fo 301989888 eof 0 [D 20:01:06.494928] ========================================================= [D 20:01:06.494952] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:06.494976] element 0: offset: 0x94041000, size: 2097152 [D 20:01:06.495006] post_sr: da13:3346 bh 33 len 48 wr_id 15e72e98. [D 20:01:06.495078] openib_check_cq: completed send wr_id 15e72e98. [D 20:01:06.498266] openib_check_cq: completed recv wr_id 15e737a0. [D 20:01:06.498297] openib_post_rr: da13:3346 bh 29 wr_id 15e737a0. [D 20:01:06.498330] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:06.498355] ========================================================= [D 20:01:06.498380] PINT_process_request [D 20:01:06.498404] tiling 8 copies [D 20:01:06.498428] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.498455] lvl 0 el 0 blk 0 by 232783872 [D 20:01:06.498479] to 232783872 ta 201326592 fi 234881024 [D 20:01:06.498504] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:06.498529] basic type or contiguous data [D 20:01:06.498553] Request disp [D 20:01:06.498577] contig_offset = 232783872 contig_size = 35651584 lvl_flag = 1 [D 20:01:06.498601] calling distribute [D 20:01:06.498626] PINT_distribute [D 20:01:06.498650] of 232783872 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.498677] begin iteration loff: 232783872 [D 20:01:06.498702] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:06.498726] **********CALL***PROCESS********* [D 20:01:06.498750] segment of 31457280 sz 2097152 [D 20:01:06.498775] PINT_process_request [D 20:01:06.498799] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.498826] lvl 0 el 0 blk 0 by 31457280 [D 20:01:06.498850] to 31457280 ta 31457280 fi 33554432 [D 20:01:06.498874] basic type or contiguous data [D 20:01:06.498899] Request disp [D 20:01:06.498922] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:06.498948] process a segment [D 20:01:06.498970] of 31457280 sz 2097152 [D 20:01:06.498995] add a segment 0 [D 20:01:06.499019] retval = 2097152 [D 20:01:06.499043] return from level -1 [D 20:01:06.499067] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:06.499094] *****RETURN***FROM***PROCESS***** [D 20:01:06.499117] end iteration [D 20:01:06.499141] done with segments or bytes [D 20:01:06.499165] of 234881024 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.499191] next loff: 234881024 next poff: 234881024 [D 20:01:06.499218] (return value) 2097152 [D 20:01:06.499243] finished [D 20:01:06.499267] retval = 2097152 [D 20:01:06.499291] exiting distribute returned less than expected [D 20:01:06.499316] done sg 1 sm 64 by 2097152 bm 2097152 ta 201326592 to 234881024 fo 234881024 eof 0 [D 20:01:06.499341] ========================================================= [D 20:01:06.499365] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 29 [D 20:01:06.499391] element 0: offset: 0x98245000, size: 2097152 [D 20:01:06.499420] post_sr: da13:3346 bh 34 len 48 wr_id 15e72eb0. [D 20:01:06.499490] openib_check_cq: completed send wr_id 15e72eb0. [D 20:01:06.499873] PINT_client_state_machine_test id 79 [D 20:01:06.502713] openib_check_cq: completed recv wr_id 15e737b8. [D 20:01:06.502740] openib_post_rr: da13:3346 bh 30 wr_id 15e737b8. [D 20:01:06.502770] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ebf298. [D 20:01:06.502796] flowproto completing 0x15ebf298 [D 20:01:06.502822] Job flows in progress (callback time): 8 [D 20:01:06.502850] sm control op 6 [D 20:01:06.502875] [SM Entering]: (0x15ebe480) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:06.502902] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.502926] (0x15ebe518) io_datafile_complete_operations (tag 2) [D 20:01:06.502952] matched completed flow for context 0x15ebece8 [D 20:01:06.502977] *** all operations completed (msgpairs, flows, write acks) [D 20:01:06.503002] [SM Exiting]: (0x15ebe480) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:06.503028] [SM Entering]: (0x15ebe480) client_io_sm:io_analyze_results (status: 2) [D 20:01:06.503053] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.503078] (0x15ebe518) io state: io_analyze_results [D 20:01:06.503103] - io_check_context_status called [D 20:01:06.503128] 33554432 bytes read from context 0x15ebece8 [D 20:01:06.503152] [1/1] running size is 33554432 [D 20:01:06.503177] [2/1] running size is 33554432 [D 20:01:06.503206] no context errors found; using: Success [D 20:01:06.503231] total bytes transferred is 33554432 [D 20:01:06.503255] [SM Exiting]: (0x15ebe480) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:06.503281] [SM Entering]: (0x15ebe480) client_io_sm:io_cleanup (status: 2) [D 20:01:06.503307] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.503332] (0x15ebe518) io state: io_cleanup [D 20:01:06.503358] [SM Exiting]: (0x15ebe480) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:06.503386] client_state_machine_terminate smcb 0x15ebe480 [D 20:01:06.503411] add smcb 0x15ebe480 to completion list [D 20:01:06.503436] [SM frame get]: (0x15ebe480) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 21 get_next_full_readbuf: enter userFilePos 167772160, read_ahead_offset 503316480 -- buffer 0x15e32870 curbuf 22 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 22 aio_req 0x15e746a0 b->offset 167772160 b->size 33554432 [D 20:01:06.859033] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.859079] PINT_client_state_machine_test id 82 [D 20:01:06.859111] sm control op 300 [D 20:01:06.859137] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:01:06.859163] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:01:06.859194] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:01:06.859221] PINT_client_state_machine_test id 82 [D 20:01:06.859253] openib_check_cq: completed recv wr_id 15e737d0. [D 20:01:06.859281] openib_post_rr: da13:3346 bh 31 wr_id 15e737d0. [D 20:01:06.859309] openib_check_cq: completed recv wr_id 15e737e8. [D 20:01:06.859336] openib_post_rr: da13:3346 bh 32 wr_id 15e737e8. [D 20:01:06.859363] openib_check_cq: completed recv wr_id 15e73800. [D 20:01:06.859390] openib_post_rr: da13:3346 bh 33 wr_id 15e73800. [D 20:01:06.859422] openib_check_cq: completed recv wr_id 15e73818. [D 20:01:06.859450] openib_post_rr: da13:3346 bh 34 wr_id 15e73818. [D 20:01:06.859476] openib_check_cq: completed recv wr_id 15e73830. [D 20:01:06.859503] openib_post_rr: da13:3346 bh 35 wr_id 15e73830. [D 20:01:06.859529] openib_check_cq: completed recv wr_id 15e73848. [D 20:01:06.859556] openib_post_rr: da13:3346 bh 36 wr_id 15e73848. [D 20:01:06.859581] openib_check_cq: completed recv wr_id 15e73860. [D 20:01:06.859608] openib_post_rr: da13:3346 bh 37 wr_id 15e73860. [D 20:01:06.859634] openib_check_cq: completed recv wr_id 15e73878. [D 20:01:06.859661] openib_post_rr: da13:3346 bh 38 wr_id 15e73878. [D 20:01:06.859695] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:06.859722] ========================================================= [D 20:01:06.859746] PINT_process_request [D 20:01:06.859771] tiling 7 copies [D 20:01:06.859796] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.859823] lvl 0 el 0 blk 0 by 199229440 [D 20:01:06.859848] to 199229440 ta 167772160 fi 201326592 [D 20:01:06.859872] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:06.859897] basic type or contiguous data [D 20:01:06.859921] Request disp [D 20:01:06.859946] contig_offset = 199229440 contig_size = 35651584 lvl_flag = 1 [D 20:01:06.859971] calling distribute [D 20:01:06.859995] PINT_distribute [D 20:01:06.860018] of 199229440 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.860047] begin iteration loff: 199229440 [D 20:01:06.860072] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:06.860097] **********CALL***PROCESS********* [D 20:01:06.860121] segment of 31457280 sz 2097152 [D 20:01:06.860146] PINT_process_request [D 20:01:06.860171] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.860197] lvl 0 el 0 blk 0 by 31457280 [D 20:01:06.860224] to 31457280 ta 31457280 fi 33554432 [D 20:01:06.860250] basic type or contiguous data [D 20:01:06.860274] Request disp [D 20:01:06.860298] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:06.860322] process a segment [D 20:01:06.860346] of 31457280 sz 2097152 [D 20:01:06.860370] add a segment 0 [D 20:01:06.860394] retval = 2097152 [D 20:01:06.860418] return from level -1 [D 20:01:06.860443] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:06.860468] *****RETURN***FROM***PROCESS***** [D 20:01:06.860493] end iteration [D 20:01:06.860517] done with segments or bytes [D 20:01:06.860541] of 201326592 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.860567] next loff: 201326592 next poff: 201326592 [D 20:01:06.860592] (return value) 2097152 [D 20:01:06.860616] finished [D 20:01:06.860641] retval = 2097152 [D 20:01:06.860665] exiting distribute returned less than expected [D 20:01:06.860689] done sg 1 sm 64 by 2097152 bm 2097152 ta 167772160 to 201326592 fo 201326592 eof 0 [D 20:01:06.860715] ========================================================= [D 20:01:06.860739] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 28 [D 20:01:06.860765] element 0: offset: 0x9a247000, size: 2097152 [D 20:01:06.860796] post_sr: da13:3346 bh 35 len 48 wr_id 15e72ec8. [D 20:01:06.860825] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:06.860851] ========================================================= [D 20:01:06.860876] PINT_process_request [D 20:01:06.860900] tiling 11 copies [D 20:01:06.860924] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.860951] lvl 0 el 0 blk 0 by 331350016 [D 20:01:06.860976] to 331350016 ta 301989888 fi 335544320 [D 20:01:06.861001] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:06.861025] basic type or contiguous data [D 20:01:06.861049] Request disp [D 20:01:06.861073] contig_offset = 331350016 contig_size = 37748736 lvl_flag = 1 [D 20:01:06.861099] calling distribute [D 20:01:06.861125] PINT_distribute [D 20:01:06.861149] of 331350016 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.861176] begin iteration loff: 331350016 [D 20:01:06.861202] segment exceeds byte limit [D 20:01:06.861226] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:06.861250] **********CALL***PROCESS********* [D 20:01:06.861274] segment of 29360128 sz 2097152 [D 20:01:06.861299] PINT_process_request [D 20:01:06.861323] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.861349] lvl 0 el 0 blk 0 by 29360128 [D 20:01:06.861373] to 29360128 ta 29360128 fi 31457280 [D 20:01:06.861398] basic type or contiguous data [D 20:01:06.861422] Request disp [D 20:01:06.861446] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:06.861472] process a segment [D 20:01:06.861495] of 29360128 sz 2097152 [D 20:01:06.861519] add a segment 0 [D 20:01:06.861543] retval = 2097152 [D 20:01:06.861567] exiting distribute returned less than expected [D 20:01:06.861591] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:06.861618] *****RETURN***FROM***PROCESS***** [D 20:01:06.861641] end iteration [D 20:01:06.861665] done with segments or bytes [D 20:01:06.861689] of 333447168 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.861715] next loff: 333447168 next poff: 333447168 [D 20:01:06.861740] return value 2097152 [D 20:01:06.861764] finished [D 20:01:06.861788] retval = 2097152 [D 20:01:06.861812] exiting distribute returned less than expected [D 20:01:06.861836] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 333447168 fo 335544320 eof 0 [D 20:01:06.861862] ========================================================= [D 20:01:06.861886] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:06.861911] element 0: offset: 0x9203f000, size: 2097152 [D 20:01:06.861938] openib_check_cq: completed send wr_id 15e72ec8. [D 20:01:06.861968] post_sr: da13:3346 bh 36 len 48 wr_id 15e72ee0. [D 20:01:06.861995] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec5b50. [D 20:01:06.862021] flowproto completing 0x15ec5b50 [D 20:01:06.862049] Job flows in progress (callback time): 7 [D 20:01:06.862073] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:06.862099] ========================================================= [D 20:01:06.862124] PINT_process_request [D 20:01:06.862148] tiling 9 copies [D 20:01:06.862172] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.862198] lvl 0 el 0 blk 0 by 266338304 [D 20:01:06.862227] to 266338304 ta 234881024 fi 268435456 [D 20:01:06.862256] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:06.862285] basic type or contiguous data [D 20:01:06.862312] Request disp [D 20:01:06.862339] contig_offset = 266338304 contig_size = 35651584 lvl_flag = 1 [D 20:01:06.862364] calling distribute [D 20:01:06.862389] PINT_distribute [D 20:01:06.862413] of 266338304 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 471859200 exfl 0 [D 20:01:06.862440] begin iteration loff: 266338304 [D 20:01:06.862464] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:06.862490] **********CALL***PROCESS********* [D 20:01:06.862514] segment of 31457280 sz 2097152 [D 20:01:06.862539] PINT_process_request [D 20:01:06.862563] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.862588] lvl 0 el 0 blk 0 by 31457280 [D 20:01:06.862613] to 31457280 ta 31457280 fi 33554432 [D 20:01:06.862638] basic type or contiguous data [D 20:01:06.862662] Request disp [D 20:01:06.862686] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:06.862710] process a segment [D 20:01:06.862734] of 31457280 sz 2097152 [D 20:01:06.862758] add a segment 0 [D 20:01:06.862782] retval = 2097152 [D 20:01:06.862807] return from level -1 [D 20:01:06.862831] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:06.862859] *****RETURN***FROM***PROCESS***** [D 20:01:06.862884] end iteration [D 20:01:06.862908] done with segments or bytes [D 20:01:06.862932] of 268435456 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.862957] next loff: 268435456 next poff: 268435456 [D 20:01:06.862982] (return value) 2097152 [D 20:01:06.863006] finished [D 20:01:06.863030] retval = 2097152 [D 20:01:06.863054] exiting distribute returned less than expected [D 20:01:06.863079] done sg 1 sm 64 by 2097152 bm 2097152 ta 234881024 to 268435456 fo 268435456 eof 0 [D 20:01:06.863104] ========================================================= [D 20:01:06.863129] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 30 [D 20:01:06.863154] element 0: offset: 0x96243000, size: 2097152 [D 20:01:06.863181] openib_check_cq: completed send wr_id 15e72ee0. [D 20:01:06.863210] post_sr: da13:3346 bh 37 len 48 wr_id 15e72ef8. [D 20:01:06.863238] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:06.863264] ========================================================= [D 20:01:06.863287] PINT_process_request [D 20:01:06.863311] tiling 10 copies [D 20:01:06.863335] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.863364] lvl 0 el 0 blk 0 by 299892736 [D 20:01:06.863389] to 299892736 ta 268435456 fi 301989888 [D 20:01:06.863414] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:06.863439] basic type or contiguous data [D 20:01:06.863464] Request disp [D 20:01:06.863488] contig_offset = 299892736 contig_size = 35651584 lvl_flag = 1 [D 20:01:06.863512] calling distribute [D 20:01:06.863536] PINT_distribute [D 20:01:06.863560] of 299892736 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.863587] begin iteration loff: 299892736 [D 20:01:06.863612] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:06.863636] **********CALL***PROCESS********* [D 20:01:06.863660] segment of 31457280 sz 2097152 [D 20:01:06.863685] PINT_process_request [D 20:01:06.863710] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.863736] lvl 0 el 0 blk 0 by 31457280 [D 20:01:06.863760] to 31457280 ta 31457280 fi 33554432 [D 20:01:06.863786] basic type or contiguous data [D 20:01:06.863810] Request disp [D 20:01:06.863834] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:06.863859] process a segment [D 20:01:06.863883] of 31457280 sz 2097152 [D 20:01:06.863906] add a segment 0 [D 20:01:06.863931] retval = 2097152 [D 20:01:06.863955] return from level -1 [D 20:01:06.863979] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:06.864006] *****RETURN***FROM***PROCESS***** [D 20:01:06.864030] end iteration [D 20:01:06.864054] done with segments or bytes [D 20:01:06.864078] of 301989888 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.864104] next loff: 301989888 next poff: 301989888 [D 20:01:06.864129] (return value) 2097152 [D 20:01:06.864152] finished [D 20:01:06.864176] retval = 2097152 [D 20:01:06.864201] exiting distribute returned less than expected [D 20:01:06.864228] done sg 1 sm 64 by 2097152 bm 2097152 ta 268435456 to 301989888 fo 301989888 eof 0 [D 20:01:06.864253] ========================================================= [D 20:01:06.864278] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 31 [D 20:01:06.864304] element 0: offset: 0x94241000, size: 2097152 [D 20:01:06.864330] openib_check_cq: completed send wr_id 15e72ef8. [D 20:01:06.864359] post_sr: da13:3346 bh 38 len 48 wr_id 15e72f10. [D 20:01:06.864388] sm control op 6 [D 20:01:06.864414] [SM Entering]: (0x15ec4d38) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:06.864440] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.864465] (0x15ec4dd0) io_datafile_complete_operations (tag 2) [D 20:01:06.864490] matched completed flow for context 0x15ec55a0 [D 20:01:06.864516] *** all operations completed (msgpairs, flows, write acks) [D 20:01:06.864543] [SM Exiting]: (0x15ec4d38) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:06.864569] [SM Entering]: (0x15ec4d38) client_io_sm:io_analyze_results (status: 2) [D 20:01:06.864595] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.864620] (0x15ec4dd0) io state: io_analyze_results [D 20:01:06.864645] - io_check_context_status called [D 20:01:06.864669] 33554432 bytes read from context 0x15ec55a0 [D 20:01:06.864694] [1/1] running size is 33554432 [D 20:01:06.864719] [2/1] running size is 33554432 [D 20:01:06.864748] no context errors found; using: Success [D 20:01:06.864773] total bytes transferred is 33554432 [D 20:01:06.864798] [SM Exiting]: (0x15ec4d38) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:06.864824] [SM Entering]: (0x15ec4d38) client_io_sm:io_cleanup (status: 2) [D 20:01:06.864850] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.864874] (0x15ec4dd0) io state: io_cleanup [D 20:01:06.864901] [SM Exiting]: (0x15ec4d38) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:06.864928] client_state_machine_terminate smcb 0x15ec4d38 [D 20:01:06.864952] add smcb 0x15ec4d38 to completion list [D 20:01:06.864976] PINT_client_state_machine_test id 82 [D 20:01:06.865005] openib_check_cq: completed send wr_id 15e72f10. [D 20:01:06.865035] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:06.865060] ========================================================= [D 20:01:06.865084] PINT_process_request [D 20:01:06.865108] tiling 12 copies [D 20:01:06.865132] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.865159] lvl 0 el 0 blk 0 by 364904448 [D 20:01:06.865183] to 364904448 ta 335544320 fi 369098752 [D 20:01:06.865208] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:06.865233] basic type or contiguous data [D 20:01:06.865257] Request disp [D 20:01:06.865281] contig_offset = 364904448 contig_size = 37748736 lvl_flag = 1 [D 20:01:06.865306] calling distribute [D 20:01:06.865330] PINT_distribute [D 20:01:06.865354] of 364904448 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:06.865381] begin iteration loff: 364904448 [D 20:01:06.865406] segment exceeds byte limit [D 20:01:06.865429] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:06.865454] **********CALL***PROCESS********* [D 20:01:06.865478] segment of 29360128 sz 2097152 [D 20:01:06.865503] PINT_process_request [D 20:01:06.865527] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.865552] lvl 0 el 0 blk 0 by 29360128 [D 20:01:06.865577] to 29360128 ta 29360128 fi 31457280 [D 20:01:06.865602] basic type or contiguous data [D 20:01:06.865627] Request disp [D 20:01:06.865651] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:06.865675] process a segment [D 20:01:06.865699] of 29360128 sz 2097152 [D 20:01:06.865724] add a segment 0 [D 20:01:06.865748] retval = 2097152 [D 20:01:06.865772] exiting distribute returned less than expected [D 20:01:06.865796] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:06.865822] *****RETURN***FROM***PROCESS***** [D 20:01:06.865846] end iteration [D 20:01:06.865870] done with segments or bytes [D 20:01:06.865894] of 367001600 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.865920] next loff: 367001600 next poff: 367001600 [D 20:01:06.865944] return value 2097152 [D 20:01:06.865969] finished [D 20:01:06.865993] retval = 2097152 [D 20:01:06.866017] exiting distribute returned less than expected [D 20:01:06.866041] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 367001600 fo 369098752 eof 0 [D 20:01:06.866067] ========================================================= [D 20:01:06.866092] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:06.866117] element 0: offset: 0x9003d000, size: 2097152 [D 20:01:06.866146] openib_check_cq: completed recv wr_id 15e73890. [D 20:01:06.866172] openib_post_rr: da13:3346 bh 39 wr_id 15e73890. [D 20:01:06.866202] post_sr: da13:3346 bh 39 len 48 wr_id 15e72f28. [D 20:01:06.866233] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:06.866264] ========================================================= [D 20:01:06.866291] PINT_process_request [D 20:01:06.866319] tiling 13 copies [D 20:01:06.866344] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.866371] lvl 0 el 0 blk 0 by 398458880 [D 20:01:06.866395] to 398458880 ta 369098752 fi 402653184 [D 20:01:06.866420] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:06.866445] basic type or contiguous data [D 20:01:06.866470] Request disp [D 20:01:06.866494] contig_offset = 398458880 contig_size = 37748736 lvl_flag = 1 [D 20:01:06.866518] calling distribute [D 20:01:06.866542] PINT_distribute [D 20:01:06.866566] of 398458880 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.866593] begin iteration loff: 398458880 [D 20:01:06.866618] segment exceeds byte limit [D 20:01:06.866641] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:06.866666] **********CALL***PROCESS********* [D 20:01:06.866690] segment of 29360128 sz 2097152 [D 20:01:06.866715] PINT_process_request [D 20:01:06.866739] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.866765] lvl 0 el 0 blk 0 by 29360128 [D 20:01:06.866790] to 29360128 ta 29360128 fi 31457280 [D 20:01:06.866815] basic type or contiguous data [D 20:01:06.866839] Request disp [D 20:01:06.866863] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:06.866888] process a segment [D 20:01:06.866912] of 29360128 sz 2097152 [D 20:01:06.866936] add a segment 0 [D 20:01:06.866960] retval = 2097152 [D 20:01:06.866985] exiting distribute returned less than expected [D 20:01:06.867009] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:06.867034] *****RETURN***FROM***PROCESS***** [D 20:01:06.867059] end iteration [D 20:01:06.867083] done with segments or bytes [D 20:01:06.867107] of 400556032 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.867133] next loff: 400556032 next poff: 400556032 [D 20:01:06.867157] return value 2097152 [D 20:01:06.867182] finished [D 20:01:06.867206] retval = 2097152 [D 20:01:06.867230] exiting distribute returned less than expected [D 20:01:06.867254] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 400556032 fo 402653184 eof 0 [D 20:01:06.867280] ========================================================= [D 20:01:06.867304] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:06.867330] element 0: offset: 0x8e03b000, size: 2097152 [D 20:01:06.867356] openib_check_cq: completed send wr_id 15e72f28. [D 20:01:06.867385] post_sr: da13:3346 bh 40 len 48 wr_id 15e72f40. [D 20:01:06.867413] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:06.867439] ========================================================= [D 20:01:06.867464] PINT_process_request [D 20:01:06.867487] tiling 14 copies [D 20:01:06.867511] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.867538] lvl 0 el 0 blk 0 by 432013312 [D 20:01:06.867563] to 432013312 ta 402653184 fi 436207616 [D 20:01:06.867588] mto 29360128 mta 27262976 mfi 29360128 [D 20:01:06.867612] basic type or contiguous data [D 20:01:06.867636] Request disp [D 20:01:06.867660] contig_offset = 432013312 contig_size = 37748736 lvl_flag = 1 [D 20:01:06.867685] calling distribute [D 20:01:06.867709] PINT_distribute [D 20:01:06.867733] of 432013312 sz 4194304 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:06.867759] begin iteration loff: 432013312 [D 20:01:06.867784] segment exceeds byte limit [D 20:01:06.867809] client lstof 29360128 diff 0 sgof 29360128 [D 20:01:06.867834] **********CALL***PROCESS********* [D 20:01:06.867859] segment of 29360128 sz 2097152 [D 20:01:06.867885] PINT_process_request [D 20:01:06.867909] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:06.867935] lvl 0 el 0 blk 0 by 29360128 [D 20:01:06.867959] to 29360128 ta 29360128 fi 31457280 [D 20:01:06.867984] basic type or contiguous data [D 20:01:06.868008] Request disp [D 20:01:06.868032] contig_offset = 29360128 contig_size = 4194304 lvl_flag = 1 [D 20:01:06.868057] process a segment [D 20:01:06.868081] of 29360128 sz 2097152 [D 20:01:06.868105] add a segment 0 [D 20:01:06.868129] retval = 2097152 [D 20:01:06.868153] exiting distribute returned less than expected [D 20:01:06.868177] done sg 1 sm 64 by 2097152 bm 2097152 ta 29360128 to 31457280 fo 31457280 eof 0 [D 20:01:06.868204] *****RETURN***FROM***PROCESS***** [D 20:01:06.868230] end iteration [D 20:01:06.868254] done with segments or bytes [D 20:01:06.868278] of 434110464 sz 2097152 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:06.868304] next loff: 434110464 next poff: 434110464 [D 20:01:06.868329] return value 2097152 [D 20:01:06.868352] finished [D 20:01:06.868376] retval = 2097152 [D 20:01:06.868400] exiting distribute returned less than expected [D 20:01:06.868425] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 434110464 fo 436207616 eof 0 [D 20:01:06.868451] ========================================================= [D 20:01:06.868475] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:06.868500] element 0: offset: 0x8c039000, size: 2097152 [D 20:01:06.868526] openib_check_cq: completed send wr_id 15e72f40. [D 20:01:06.868556] post_sr: da13:3346 bh 41 len 48 wr_id 15e72f58. [D 20:01:06.868585] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec2678. [D 20:01:06.868612] flowproto completing 0x15ec2678 [D 20:01:06.868638] Job flows in progress (callback time): 6 [D 20:01:06.868665] sm control op 6 [D 20:01:06.868690] [SM Entering]: (0x15ec1880) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:06.868716] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.868741] (0x15ec1918) io_datafile_complete_operations (tag 2) [D 20:01:06.868766] matched completed flow for context 0x15ec20c8 [D 20:01:06.868791] *** all operations completed (msgpairs, flows, write acks) [D 20:01:06.868816] [SM Exiting]: (0x15ec1880) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:06.868842] [SM Entering]: (0x15ec1880) client_io_sm:io_analyze_results (status: 2) [D 20:01:06.868867] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.868893] (0x15ec1918) io state: io_analyze_results [D 20:01:06.868918] - io_check_context_status called [D 20:01:06.868942] 33554432 bytes read from context 0x15ec20c8 [D 20:01:06.868967] [1/1] running size is 33554432 [D 20:01:06.868991] [2/1] running size is 33554432 [D 20:01:06.869018] no context errors found; using: Success [D 20:01:06.869043] total bytes transferred is 33554432 [D 20:01:06.869067] [SM Exiting]: (0x15ec1880) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:06.869093] [SM Entering]: (0x15ec1880) client_io_sm:io_cleanup (status: 2) [D 20:01:06.869118] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 [D 20:01:06.869144] (0x15ec1918) io state: io_cleanup [D 20:01:06.869170] [SM Exiting]: (0x15ec1880) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:06.869195] client_state_machine_terminate smcb 0x15ec1880 [D 20:01:06.869220] add smcb 0x15ec1880 to completion list [D 20:01:06.869245] [SM frame get]: (0x15ec1880) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 22 get_next_full_readbuf: enter userFilePos 201326592, read_ahead_offset 503316480 -- buffer 0x15e32898 curbuf 23 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 23 aio_req 0x15e746c0 b->offset 201326592 b->size 33554432 [D 20:01:07.223564] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 [D 20:01:07.223608] PINT_client_state_machine_test id 85 [D 20:01:07.223634] [SM frame get]: (0x15ec4d38) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 23 get_next_full_readbuf: enter userFilePos 234881024, read_ahead_offset 503316480 -- buffer 0x15e328c0 curbuf 24 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 24 aio_req 0x15e746e0 b->offset 234881024 b->size 33554432 [D 20:01:07.578115] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:01:07.578164] PINT_client_state_machine_test id 88 [D 20:01:07.578202] openib_check_cq: completed send wr_id 15e72f58. [D 20:01:07.578233] openib_check_cq: completed recv wr_id 15e738a8. [D 20:01:07.578268] openib_post_rr: da13:3346 bh 40 wr_id 15e738a8. [D 20:01:07.578299] openib_check_cq: completed recv wr_id 15e738c0. [D 20:01:07.578330] openib_post_rr: da13:3346 bh 41 wr_id 15e738c0. [D 20:01:07.578356] openib_check_cq: completed recv wr_id 15e738d8. [D 20:01:07.578383] openib_post_rr: da13:3346 bh 42 wr_id 15e738d8. [D 20:01:07.578410] openib_check_cq: completed recv wr_id 15e738f0. [D 20:01:07.578436] openib_post_rr: da13:3346 bh 43 wr_id 15e738f0. [D 20:01:07.578462] openib_check_cq: completed recv wr_id 15e73908. [D 20:01:07.578489] openib_post_rr: da13:3346 bh 44 wr_id 15e73908. [D 20:01:07.578516] openib_check_cq: completed recv wr_id 15e73920. [D 20:01:07.578542] openib_post_rr: da13:3346 bh 45 wr_id 15e73920. [D 20:01:07.578578] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ec9028. [D 20:01:07.578605] flowproto completing 0x15ec9028 [D 20:01:07.578634] Job flows in progress (callback time): 5 [D 20:01:07.578660] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecc500. [D 20:01:07.578685] flowproto completing 0x15ecc500 [D 20:01:07.578712] Job flows in progress (callback time): 4 [D 20:01:07.578738] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:07.578763] ========================================================= [D 20:01:07.578788] PINT_process_request [D 20:01:07.578813] tiling 12 copies [D 20:01:07.578837] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:07.578864] lvl 0 el 0 blk 0 by 367001600 [D 20:01:07.578888] to 367001600 ta 335544320 fi 369098752 [D 20:01:07.578913] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:07.578938] basic type or contiguous data [D 20:01:07.578963] Request disp [D 20:01:07.578988] contig_offset = 367001600 contig_size = 35651584 lvl_flag = 1 [D 20:01:07.579012] calling distribute [D 20:01:07.579037] PINT_distribute [D 20:01:07.579061] of 367001600 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:07.579089] begin iteration loff: 367001600 [D 20:01:07.579114] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:07.579140] **********CALL***PROCESS********* [D 20:01:07.579164] segment of 31457280 sz 2097152 [D 20:01:07.579189] PINT_process_request [D 20:01:07.579213] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:07.579239] lvl 0 el 0 blk 0 by 31457280 [D 20:01:07.579264] to 31457280 ta 31457280 fi 33554432 [D 20:01:07.579288] basic type or contiguous data [D 20:01:07.579312] Request disp [D 20:01:07.579336] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:07.579360] process a segment [D 20:01:07.579384] of 31457280 sz 2097152 [D 20:01:07.579409] add a segment 0 [D 20:01:07.579433] retval = 2097152 [D 20:01:07.579457] return from level -1 [D 20:01:07.579482] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:07.579508] *****RETURN***FROM***PROCESS***** [D 20:01:07.579532] end iteration [D 20:01:07.579556] done with segments or bytes [D 20:01:07.579580] of 369098752 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:07.579606] next loff: 369098752 next poff: 369098752 [D 20:01:07.579631] (return value) 2097152 [D 20:01:07.579661] finished [D 20:01:07.579685] retval = 2097152 [D 20:01:07.579710] exiting distribute returned less than expected [D 20:01:07.579733] done sg 1 sm 64 by 2097152 bm 2097152 ta 335544320 to 369098752 fo 369098752 eof 0 [D 20:01:07.579760] ========================================================= [D 20:01:07.579785] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 33 [D 20:01:07.579811] element 0: offset: 0x9023d000, size: 2097152 [D 20:01:07.579847] post_sr: da13:3346 bh 42 len 48 wr_id 15e72f70. [D 20:01:07.579875] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:07.579902] ========================================================= [D 20:01:07.579926] PINT_process_request [D 20:01:07.579951] tiling 11 copies [D 20:01:07.579975] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:07.580001] lvl 0 el 0 blk 0 by 333447168 [D 20:01:07.580026] to 333447168 ta 301989888 fi 335544320 [D 20:01:07.580051] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:07.580076] basic type or contiguous data [D 20:01:07.580100] Request disp [D 20:01:07.580123] contig_offset = 333447168 contig_size = 35651584 lvl_flag = 1 [D 20:01:07.580148] calling distribute [D 20:01:07.580173] PINT_distribute [D 20:01:07.580196] of 333447168 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 476053504 exfl 0 [D 20:01:07.580225] begin iteration loff: 333447168 [D 20:01:07.580250] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:07.580275] **********CALL***PROCESS********* [D 20:01:07.580299] segment of 31457280 sz 2097152 [D 20:01:07.580324] PINT_process_request [D 20:01:07.580348] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:07.580374] lvl 0 el 0 blk 0 by 31457280 [D 20:01:07.580399] to 31457280 ta 31457280 fi 33554432 [D 20:01:07.580424] basic type or contiguous data [D 20:01:07.580448] Request disp [D 20:01:07.580471] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:07.580496] process a segment [D 20:01:07.580520] of 31457280 sz 2097152 [D 20:01:07.580544] add a segment 0 [D 20:01:07.580569] retval = 2097152 [D 20:01:07.580593] return from level -1 [D 20:01:07.580616] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:07.580642] *****RETURN***FROM***PROCESS***** [D 20:01:07.580667] end iteration [D 20:01:07.580691] done with segments or bytes [D 20:01:07.580715] of 335544320 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:07.580740] next loff: 335544320 next poff: 335544320 [D 20:01:07.580765] (return value) 2097152 [D 20:01:07.580789] finished [D 20:01:07.580813] retval = 2097152 [D 20:01:07.580837] exiting distribute returned less than expected [D 20:01:07.580862] done sg 1 sm 64 by 2097152 bm 2097152 ta 301989888 to 335544320 fo 335544320 eof 0 [D 20:01:07.580887] ========================================================= [D 20:01:07.580912] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 32 [D 20:01:07.580937] element 0: offset: 0x9223f000, size: 2097152 [D 20:01:07.580964] openib_check_cq: completed send wr_id 15e72f70. [D 20:01:07.580993] post_sr: da13:3346 bh 43 len 48 wr_id 15e72f88. [D 20:01:07.581021] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:07.581047] ========================================================= [D 20:01:07.581070] PINT_process_request [D 20:01:07.581095] tiling 13 copies [D 20:01:07.581119] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:07.581145] lvl 0 el 0 blk 0 by 400556032 [D 20:01:07.581170] to 400556032 ta 369098752 fi 402653184 [D 20:01:07.581194] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:07.581220] basic type or contiguous data [D 20:01:07.581244] Request disp [D 20:01:07.581268] contig_offset = 400556032 contig_size = 35651584 lvl_flag = 1 [D 20:01:07.581293] calling distribute [D 20:01:07.581316] PINT_distribute [D 20:01:07.581340] of 400556032 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:07.581370] begin iteration loff: 400556032 [D 20:01:07.581396] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:07.581420] **********CALL***PROCESS********* [D 20:01:07.581444] segment of 31457280 sz 2097152 [D 20:01:07.581469] PINT_process_request [D 20:01:07.581493] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:07.581519] lvl 0 el 0 blk 0 by 31457280 [D 20:01:07.581543] to 31457280 ta 31457280 fi 33554432 [D 20:01:07.581568] basic type or contiguous data [D 20:01:07.581592] Request disp [D 20:01:07.581616] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:07.581642] process a segment [D 20:01:07.581665] of 31457280 sz 2097152 [D 20:01:07.581689] add a segment 0 [D 20:01:07.581713] retval = 2097152 [D 20:01:07.581737] return from level -1 [D 20:01:07.581762] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:07.581788] *****RETURN***FROM***PROCESS***** [D 20:01:07.581812] end iteration [D 20:01:07.581836] done with segments or bytes [D 20:01:07.581860] of 402653184 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:07.581886] next loff: 402653184 next poff: 402653184 [D 20:01:07.581910] (return value) 2097152 [D 20:01:07.581934] finished [D 20:01:07.581958] retval = 2097152 [D 20:01:07.581983] exiting distribute returned less than expected [D 20:01:07.582007] done sg 1 sm 64 by 2097152 bm 2097152 ta 369098752 to 402653184 fo 402653184 eof 0 [D 20:01:07.582033] ========================================================= [D 20:01:07.582057] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 34 [D 20:01:07.582083] element 0: offset: 0x8e23b000, size: 2097152 [D 20:01:07.582109] openib_check_cq: completed send wr_id 15e72f88. [D 20:01:07.582138] post_sr: da13:3346 bh 44 len 48 wr_id 15e72fa0. [D 20:01:07.582170] sm control op 6 [D 20:01:07.582196] [SM Entering]: (0x15ec8210) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:07.582226] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:01:07.582256] (0x15ec82a8) io_datafile_complete_operations (tag 2) [D 20:01:07.582285] matched completed flow for context 0x15ec8a78 [D 20:01:07.582315] *** all operations completed (msgpairs, flows, write acks) [D 20:01:07.582343] [SM Exiting]: (0x15ec8210) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:07.582369] [SM Entering]: (0x15ec8210) client_io_sm:io_analyze_results (status: 2) [D 20:01:07.582394] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:01:07.582419] (0x15ec82a8) io state: io_analyze_results [D 20:01:07.582445] - io_check_context_status called [D 20:01:07.582469] 33554432 bytes read from context 0x15ec8a78 [D 20:01:07.582494] [1/1] running size is 33554432 [D 20:01:07.582519] [2/1] running size is 33554432 [D 20:01:07.582551] no context errors found; using: Success [D 20:01:07.582575] total bytes transferred is 33554432 [D 20:01:07.582600] [SM Exiting]: (0x15ec8210) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:07.582626] [SM Entering]: (0x15ec8210) client_io_sm:io_cleanup (status: 2) [D 20:01:07.582652] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 [D 20:01:07.582676] (0x15ec82a8) io state: io_cleanup [D 20:01:07.582704] [SM Exiting]: (0x15ec8210) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:07.582731] client_state_machine_terminate smcb 0x15ec8210 [D 20:01:07.582756] add smcb 0x15ec8210 to completion list [D 20:01:07.582781] sm control op 6 [D 20:01:07.582805] [SM Entering]: (0x15ecb6e8) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:07.582831] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:01:07.582856] (0x15ecb780) io_datafile_complete_operations (tag 2) [D 20:01:07.582881] matched completed flow for context 0x15ecbf50 [D 20:01:07.582906] *** all operations completed (msgpairs, flows, write acks) [D 20:01:07.582934] [SM Exiting]: (0x15ecb6e8) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:07.582960] [SM Entering]: (0x15ecb6e8) client_io_sm:io_analyze_results (status: 2) [D 20:01:07.582985] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:01:07.583010] (0x15ecb780) io state: io_analyze_results [D 20:01:07.583035] - io_check_context_status called [D 20:01:07.583059] 33554432 bytes read from context 0x15ecbf50 [D 20:01:07.583084] [1/1] running size is 33554432 [D 20:01:07.583108] [2/1] running size is 33554432 [D 20:01:07.583135] no context errors found; using: Success [D 20:01:07.583159] total bytes transferred is 33554432 [D 20:01:07.583184] [SM Exiting]: (0x15ecb6e8) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:07.583209] [SM Entering]: (0x15ecb6e8) client_io_sm:io_cleanup (status: 2) [D 20:01:07.583235] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:01:07.583260] (0x15ecb780) io state: io_cleanup [D 20:01:07.583288] [SM Exiting]: (0x15ecb6e8) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:07.583314] client_state_machine_terminate smcb 0x15ecb6e8 [D 20:01:07.583339] add smcb 0x15ecb6e8 to completion list [D 20:01:07.583364] [SM frame get]: (0x15ec8210) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 24 get_next_full_readbuf: enter userFilePos 268435456, read_ahead_offset 503316480 -- buffer 0x15e328e8 curbuf 25 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 25 aio_req 0x15e74700 b->offset 268435456 b->size 33554432 [D 20:01:07.938679] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 [D 20:01:07.938727] PINT_client_state_machine_test id 91 [D 20:01:07.938753] [SM frame get]: (0x15ecb6e8) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 25 get_next_full_readbuf: enter userFilePos 301989888, read_ahead_offset 503316480 -- buffer 0x15e32910 curbuf 26 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 26 aio_req 0x15e74720 b->offset 301989888 b->size 33554432 [D 20:01:08.292572] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.292618] PINT_client_state_machine_test id 94 [D 20:01:08.292653] sm control op 300 [D 20:01:08.292678] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:01:08.292705] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:01:08.292739] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:01:08.292766] PINT_client_state_machine_test id 94 [D 20:01:08.292800] openib_check_cq: completed send wr_id 15e72fa0. [D 20:01:08.292828] openib_check_cq: completed recv wr_id 15e73938. [D 20:01:08.292858] openib_post_rr: da13:3346 bh 46 wr_id 15e73938. [D 20:01:08.292886] openib_check_cq: completed recv wr_id 15e73950. [D 20:01:08.292913] openib_post_rr: da13:3346 bh 47 wr_id 15e73950. [D 20:01:08.292939] openib_check_cq: completed recv wr_id 15e73968. [D 20:01:08.292965] openib_post_rr: da13:3346 bh 48 wr_id 15e73968. [D 20:01:08.292999] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:08.293026] ========================================================= [D 20:01:08.293051] PINT_process_request [D 20:01:08.293076] tiling 14 copies [D 20:01:08.293101] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:08.293128] lvl 0 el 0 blk 0 by 434110464 [D 20:01:08.293152] to 434110464 ta 402653184 fi 436207616 [D 20:01:08.293178] mto 31457280 mta 29360128 mfi 31457280 [D 20:01:08.293203] basic type or contiguous data [D 20:01:08.293230] Request disp [D 20:01:08.293254] contig_offset = 434110464 contig_size = 35651584 lvl_flag = 1 [D 20:01:08.293280] calling distribute [D 20:01:08.293304] PINT_distribute [D 20:01:08.293329] of 434110464 sz 2097152 ix 0 sm 64 by 0 bm 2097152 fsz 478150656 exfl 0 [D 20:01:08.293367] begin iteration loff: 434110464 [D 20:01:08.293394] client lstof 31457280 diff 0 sgof 31457280 [D 20:01:08.293419] **********CALL***PROCESS********* [D 20:01:08.293443] segment of 31457280 sz 2097152 [D 20:01:08.293468] PINT_process_request [D 20:01:08.293492] Do seq of 0 ne 33554432 st 0 nb 1 ub 33554432 lb 0 as 33554432 co 0 [D 20:01:08.293518] lvl 0 el 0 blk 0 by 31457280 [D 20:01:08.293543] to 31457280 ta 31457280 fi 33554432 [D 20:01:08.293568] basic type or contiguous data [D 20:01:08.293592] Request disp [D 20:01:08.293616] contig_offset = 31457280 contig_size = 2097152 lvl_flag = 1 [D 20:01:08.293641] process a segment [D 20:01:08.293664] of 31457280 sz 2097152 [D 20:01:08.293689] add a segment 0 [D 20:01:08.293713] retval = 2097152 [D 20:01:08.293737] return from level -1 [D 20:01:08.293761] done sg 1 sm 64 by 2097152 bm 2097152 ta 31457280 to 33554432 fo 33554432 eof 0 [D 20:01:08.293787] *****RETURN***FROM***PROCESS***** [D 20:01:08.293812] end iteration [D 20:01:08.293835] done with segments or bytes [D 20:01:08.293859] of 436207616 sz 0 sg 1 sm 64 by 2097152 bm 2097152 [D 20:01:08.293885] next loff: 436207616 next poff: 436207616 [D 20:01:08.293910] (return value) 2097152 [D 20:01:08.293934] finished [D 20:01:08.293958] retval = 2097152 [D 20:01:08.293983] exiting distribute returned less than expected [D 20:01:08.294007] done sg 1 sm 64 by 2097152 bm 2097152 ta 402653184 to 436207616 fo 436207616 eof 0 [D 20:01:08.294034] ========================================================= [D 20:01:08.294058] BMI_post_recv_list: addr: 367403208, count: 1, total_size: 2097152, tag: 35 [D 20:01:08.294084] element 0: offset: 0x8c239000, size: 2097152 [D 20:01:08.294119] post_sr: da13:3346 bh 45 len 48 wr_id 15e72fb8. [D 20:01:08.294149] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed2eb0. [D 20:01:08.294175] flowproto completing 0x15ed2eb0 [D 20:01:08.294203] Job flows in progress (callback time): 3 [D 20:01:08.294233] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ecf9d8. [D 20:01:08.294263] flowproto completing 0x15ecf9d8 [D 20:01:08.294293] Job flows in progress (callback time): 2 [D 20:01:08.294322] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed6388. [D 20:01:08.294350] flowproto completing 0x15ed6388 [D 20:01:08.294376] Job flows in progress (callback time): 1 [D 20:01:08.294415] sm control op 6 [D 20:01:08.294440] [SM Entering]: (0x15ed2098) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:08.294466] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.294492] (0x15ed2130) io_datafile_complete_operations (tag 2) [D 20:01:08.294518] matched completed flow for context 0x15ed2900 [D 20:01:08.294544] *** all operations completed (msgpairs, flows, write acks) [D 20:01:08.294569] [SM Exiting]: (0x15ed2098) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:08.294594] [SM Entering]: (0x15ed2098) client_io_sm:io_analyze_results (status: 2) [D 20:01:08.294620] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.294645] (0x15ed2130) io state: io_analyze_results [D 20:01:08.294671] - io_check_context_status called [D 20:01:08.294695] 33554432 bytes read from context 0x15ed2900 [D 20:01:08.294720] [1/1] running size is 33554432 [D 20:01:08.294745] [2/1] running size is 33554432 [D 20:01:08.294776] no context errors found; using: Success [D 20:01:08.294801] total bytes transferred is 33554432 [D 20:01:08.294826] [SM Exiting]: (0x15ed2098) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:08.294852] [SM Entering]: (0x15ed2098) client_io_sm:io_cleanup (status: 2) [D 20:01:08.294878] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.294902] (0x15ed2130) io state: io_cleanup [D 20:01:08.294930] [SM Exiting]: (0x15ed2098) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:08.294957] client_state_machine_terminate smcb 0x15ed2098 [D 20:01:08.294984] add smcb 0x15ed2098 to completion list [D 20:01:08.295009] sm control op 6 [D 20:01:08.295034] [SM Entering]: (0x15ecebc0) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:08.295060] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.295085] (0x15ecec58) io_datafile_complete_operations (tag 2) [D 20:01:08.295110] matched completed flow for context 0x15ecf428 [D 20:01:08.295135] *** all operations completed (msgpairs, flows, write acks) [D 20:01:08.295160] [SM Exiting]: (0x15ecebc0) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:08.295186] [SM Entering]: (0x15ecebc0) client_io_sm:io_analyze_results (status: 2) [D 20:01:08.295211] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.295236] (0x15ecec58) io state: io_analyze_results [D 20:01:08.295261] - io_check_context_status called [D 20:01:08.295285] 33554432 bytes read from context 0x15ecf428 [D 20:01:08.295310] [1/1] running size is 33554432 [D 20:01:08.295334] [2/1] running size is 33554432 [D 20:01:08.295360] no context errors found; using: Success [D 20:01:08.295385] total bytes transferred is 33554432 [D 20:01:08.295410] [SM Exiting]: (0x15ecebc0) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:08.295435] [SM Entering]: (0x15ecebc0) client_io_sm:io_cleanup (status: 2) [D 20:01:08.295460] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.295486] (0x15ecec58) io state: io_cleanup [D 20:01:08.295512] [SM Exiting]: (0x15ecebc0) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:08.295537] client_state_machine_terminate smcb 0x15ecebc0 [D 20:01:08.295562] add smcb 0x15ecebc0 to completion list [D 20:01:08.295587] sm control op 6 [D 20:01:08.295611] [SM Entering]: (0x15ed5570) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:08.295637] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.295661] (0x15ed5608) io_datafile_complete_operations (tag 2) [D 20:01:08.295687] matched completed flow for context 0x15ed5dd8 [D 20:01:08.295712] *** all operations completed (msgpairs, flows, write acks) [D 20:01:08.295736] [SM Exiting]: (0x15ed5570) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:08.295763] [SM Entering]: (0x15ed5570) client_io_sm:io_analyze_results (status: 2) [D 20:01:08.295787] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.295813] (0x15ed5608) io state: io_analyze_results [D 20:01:08.295837] - io_check_context_status called [D 20:01:08.295862] 33554432 bytes read from context 0x15ed5dd8 [D 20:01:08.295886] [1/1] running size is 33554432 [D 20:01:08.295910] [2/1] running size is 33554432 [D 20:01:08.295937] no context errors found; using: Success [D 20:01:08.295961] total bytes transferred is 33554432 [D 20:01:08.295986] [SM Exiting]: (0x15ed5570) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:08.296011] [SM Entering]: (0x15ed5570) client_io_sm:io_cleanup (status: 2) [D 20:01:08.296037] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.296062] (0x15ed5608) io state: io_cleanup [D 20:01:08.296088] [SM Exiting]: (0x15ed5570) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:08.296113] client_state_machine_terminate smcb 0x15ed5570 [D 20:01:08.296138] add smcb 0x15ed5570 to completion list [D 20:01:08.296163] [SM frame get]: (0x15ecebc0) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 26 get_next_full_readbuf: enter userFilePos 335544320, read_ahead_offset 503316480 -- buffer 0x15e32938 curbuf 27 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 27 aio_req 0x15e74740 b->offset 335544320 b->size 33554432 [D 20:01:08.651032] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 [D 20:01:08.651076] PINT_client_state_machine_test id 97 [D 20:01:08.651103] [SM frame get]: (0x15ed2098) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 27 get_next_full_readbuf: enter userFilePos 369098752, read_ahead_offset 503316480 -- buffer 0x15e32960 curbuf 28 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 28 aio_req 0x15e74760 b->offset 369098752 b->size 33554432 [D 20:01:09.006567] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 [D 20:01:09.006614] PINT_client_state_machine_test id 100 [D 20:01:09.006640] [SM frame get]: (0x15ed5570) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 28 get_next_full_readbuf: enter userFilePos 402653184, read_ahead_offset 503316480 -- buffer 0x15e32988 curbuf 29 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 29 aio_req 0x15e74780 b->offset 402653184 b->size 33554432 [D 20:01:09.362170] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:01:09.362215] PINT_client_state_machine_test id 103 [D 20:01:09.362255] sm control op 300 [D 20:01:09.362284] [SM Entering]: (0x15e4ea10) client_job_timer_sm:do_work (status: 0) [D 20:01:09.362315] [SM frame get]: (0x15e4ea10) op-id: 300 index: 0 base-frm: 0 [D 20:01:09.362360] [SM Exiting]: (0x15e4ea10) client_job_timer_sm:do_work (error code: 0), (action: DEFERRED) [D 20:01:09.362388] PINT_client_state_machine_test id 103 [D 20:01:09.362421] openib_check_cq: completed send wr_id 15e72fb8. [D 20:01:09.362449] openib_check_cq: completed recv wr_id 15e73980. [D 20:01:09.362478] openib_post_rr: da13:3346 bh 49 wr_id 15e73980. [D 20:01:09.362511] flowproto-multiqueue bmi_to_mem_callback_fn, error_code: 0, flow: 0x15ed9860. [D 20:01:09.362538] flowproto completing 0x15ed9860 [D 20:01:09.362567] Job flows in progress (callback time): 0 [D 20:01:09.362627] sm control op 6 [D 20:01:09.362654] [SM Entering]: (0x15ed8a48) client_io_sm:io_datafile_complete_operations (status: 2) [D 20:01:09.362682] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:01:09.362707] (0x15ed8ae0) io_datafile_complete_operations (tag 2) [D 20:01:09.362733] matched completed flow for context 0x15ed92b0 [D 20:01:09.362759] *** all operations completed (msgpairs, flows, write acks) [D 20:01:09.362784] [SM Exiting]: (0x15ed8a48) client_io_sm:io_datafile_complete_operations (error code: 133), (action: COMPLETE) [D 20:01:09.362811] [SM Entering]: (0x15ed8a48) client_io_sm:io_analyze_results (status: 2) [D 20:01:09.362836] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:01:09.362861] (0x15ed8ae0) io state: io_analyze_results [D 20:01:09.362887] - io_check_context_status called [D 20:01:09.362912] 33554432 bytes read from context 0x15ed92b0 [D 20:01:09.362936] [1/1] running size is 33554432 [D 20:01:09.362961] [2/1] running size is 33554432 [D 20:01:09.362992] no context errors found; using: Success [D 20:01:09.363017] total bytes transferred is 33554432 [D 20:01:09.363041] [SM Exiting]: (0x15ed8a48) client_io_sm:io_analyze_results (error code: 0), (action: COMPLETE) [D 20:01:09.363068] [SM Entering]: (0x15ed8a48) client_io_sm:io_cleanup (status: 2) [D 20:01:09.363093] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 [D 20:01:09.363119] (0x15ed8ae0) io state: io_cleanup [D 20:01:09.363148] [SM Exiting]: (0x15ed8a48) client_io_sm:io_cleanup (error code: 0), (action: TERMINATE) [D 20:01:09.363175] client_state_machine_terminate smcb 0x15ed8a48 [D 20:01:09.363200] add smcb 0x15ed8a48 to completion list [D 20:01:09.363224] [SM frame get]: (0x15ed8a48) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 29 get_next_full_readbuf: enter userFilePos 436207616, read_ahead_offset 503316480 -- buffer 0x15e329b0 curbuf 30 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 30 aio_req 0x15e747a0 b->offset 436207616 b->size 33554432 [D 20:01:09.719120] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 [D 20:01:09.719177] PINT_client_state_machine_test id 106 [D 20:01:09.719203] [SM frame get]: (0x15edbf20) op-id: 6 index: 0 base-frm: 0 get_next_full_readbuf: exit curbuf 30 get_next_full_readbuf: enter userFilePos 469762048, read_ahead_offset 503316480 -- buffer 0x15e329d8 curbuf 31 buffer_ahead_offset 31 size 0 read_ahead_offset 503316480 get_next_full_readbuf at end of file, no more bufs to fill pvfs2_aio_check id 0 buf 31 aio_req 0x15e747c0 b->offset 469762048 b->size 33554432 [D 20:01:10.075294] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 [D 20:01:10.075340] PINT_client_state_machine_test id 109 [D 20:01:10.075366] [SM frame get]: (0x15edf3f8) op-id: 6 index: 0 base-frm: 0 ERROR pvfs2_aio_check called to PVFS_sys_wait for op_id 109 got error 0 ERROR pvfs2_aio_check: buf 31 offset 469762048 b->size (33554432) != total_completed (10485760) run 'gdb -p 19921' to debug ddikick.x: Fatal error detected. The error is most likely to be in the application, so check for input errors, disk space, memory needs, application bugs, etc. ddikick.x will now clean up all processes, and exit...