Race in gop_mq

[PerilousApricot]

Triggered in gridftp:

WARNING: ThreadSanitizer: data race (pid=6468)
  Write of size 8 at 0x7da000008210 by thread T35:
    #0 close /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1565 (globus-gridftp-server+0x00000042fe68)
    #1 gop_mq_conn_teardown /home/meloam/lstore/src/gop/mq_portal.c:1542:9 (libgop.so.0+0x00000003ae70)
    #2 gop_mq_conn_thread /home/meloam/lstore/src/gop/mq_portal.c:1439:5 (libgop.so.0+0x00000003aa12)

  Previous read of size 8 at 0x7da000008210 by thread T2 (mutexes: write M124, write M2504):
    #0 read /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:759 (globus-gridftp-server+0x000000440fb9)
    #1 mq_conn_create_actual /home/meloam/lstore/src/gop/mq_portal.c:1491:9 (libgop.so.0+0x00000003b65a)
    #2 mq_conn_create /home/meloam/lstore/src/gop/mq_portal.c:1517:15 (libgop.so.0+0x000000030b85)
    #3 gop_mq_submit /home/meloam/lstore/src/gop/mq_portal.c:274:15 (libgop.so.0+0x000000030762)
    #4 mq_task_send /home/meloam/lstore/src/gop/mq_portal.c:319:12 (libgop.so.0+0x000000031016)
    #5 _gop_mq_submit_op /home/meloam/lstore/src/gop/mq_portal.c:1801:5 (libgop.so.0+0x00000003ccd0)
    #6 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:221:9 (libgop.so.0+0x00000000ee74)
    #7 gop_waitall /home/meloam/lstore/src/gop/gop.c:441:13 (libgop.so.0+0x000000011eb7)
    #8 gop_sync_exec /home/meloam/lstore/src/gop/gop.c:616:11 (libgop.so.0+0x00000001382e)
    #9 lio_get_multiple_attrs /home/meloam/lstore/src/lio/lio_core_os.c:895:11 (liblio.so.0+0x000000055661)
    #10 lio_stat /home/meloam/lstore/src/lio/lio_core_os.c:1308:11 (liblio.so.0+0x00000005c60a)
    #11 plugin_stat /home/meloam/lstore/binding/gridftp/src/stat.c:55:14 (libglobus_gridftp_server_lstore.so+0x0000000054b1)
    #12 user_stat /home/meloam/lstore/binding/gridftp/src/thunk.c:132:19 (libglobus_gridftp_server_lstore.so+0x00000000709b)
    #13 globus_l_gfs_lstore_stat /home/meloam/lstore/binding/gridftp/src/lstore_dsi.c:201:18 (libglobus_gridftp_server_lstore.so+0x000000003588)
    #14 globus_l_gfs_blocking_dispatch_kickout <null> (libglobus_gridftp_server.so.6+0x000000046b54)
    #15 globus_l_gfs_authorize_cb <null> (libglobus_gridftp_server.so.6+0x0000000340c8)
    #16 globus_l_gfs_acl_kickout <null> (libglobus_gridftp_server.so.6+0x00000002a911)
    #17 globus_l_callback_thread_poll <null> (libglobus_common.so.0+0x00000002517e)
    #18 globus_l_thread_pool_thread_start <null> (libglobus_common.so.0+0x000000062cf4)
    #19 thread_starter <null> (libglobus_thread_pthread.so+0x000000003b30)

  As if synchronized via sleep:
    #0 usleep /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:357 (globus-gridftp-server+0x000000429a77)
    #1 zero_native_send /home/meloam/lstore/src/gop/mq_zmq.c:152:21 (libgop.so.0+0x00000002cfd6)
    #2 mq_conn_make /home/meloam/lstore/src/gop/mq_portal.c:1268:12 (libgop.so.0+0x00000003940b)
    #3 gop_mq_conn_thread /home/meloam/lstore/src/gop/mq_portal.c:1345:18 (libgop.so.0+0x000000039c01)

  Location is file descriptor 33 created by thread T2 at:
    #0 pipe /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1601 (globus-gridftp-server+0x00000042b70e)
    #1 mq_conn_create_actual /home/meloam/lstore/src/gop/mq_portal.c:1484:5 (libgop.so.0+0x00000003b485)
    #2 mq_conn_create /home/meloam/lstore/src/gop/mq_portal.c:1517:15 (libgop.so.0+0x000000030b85)
    #3 gop_mq_submit /home/meloam/lstore/src/gop/mq_portal.c:274:15 (libgop.so.0+0x000000030762)
    #4 mq_task_send /home/meloam/lstore/src/gop/mq_portal.c:319:12 (libgop.so.0+0x000000031016)
    #5 _gop_mq_submit_op /home/meloam/lstore/src/gop/mq_portal.c:1801:5 (libgop.so.0+0x00000003ccd0)
    #6 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:221:9 (libgop.so.0+0x00000000ee74)
    #7 gop_waitall /home/meloam/lstore/src/gop/gop.c:441:13 (libgop.so.0+0x000000011eb7)
    #8 gop_sync_exec /home/meloam/lstore/src/gop/gop.c:616:11 (libgop.so.0+0x00000001382e)
    #9 lio_get_multiple_attrs /home/meloam/lstore/src/lio/lio_core_os.c:895:11 (liblio.so.0+0x000000055661)
    #10 lio_stat /home/meloam/lstore/src/lio/lio_core_os.c:1308:11 (liblio.so.0+0x00000005c60a)
    #11 plugin_stat /home/meloam/lstore/binding/gridftp/src/stat.c:55:14 (libglobus_gridftp_server_lstore.so+0x0000000054b1)
    #12 user_stat /home/meloam/lstore/binding/gridftp/src/thunk.c:132:19 (libglobus_gridftp_server_lstore.so+0x00000000709b)
    #13 globus_l_gfs_lstore_stat /home/meloam/lstore/binding/gridftp/src/lstore_dsi.c:201:18 (libglobus_gridftp_server_lstore.so+0x000000003588)
    #14 globus_l_gfs_blocking_dispatch_kickout <null> (libglobus_gridftp_server.so.6+0x000000046b54)
    #15 globus_l_gfs_authorize_cb <null> (libglobus_gridftp_server.so.6+0x0000000340c8)
    #16 globus_l_gfs_acl_kickout <null> (libglobus_gridftp_server.so.6+0x00000002a911)
    #17 globus_l_callback_thread_poll <null> (libglobus_common.so.0+0x00000002517e)
    #18 globus_l_thread_pool_thread_start <null> (libglobus_common.so.0+0x000000062cf4)
    #19 thread_starter <null> (libglobus_thread_pthread.so+0x000000003b30)

  Mutex M124 (0x7d900000a2a0) created at:
    #0 pthread_mutex_init /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1119 (globus-gridftp-server+0x000000433f55)
    #1 apr_thread_mutex_create /home/meloam/lstore/vendor/apr-accre/locks/unix/thread_mutex.c:71:14 (liblio.so.0+0x0000001a8b26)
    #2 tbx_pc_new /home/meloam/lstore/src/toolbox/pigeon_coop.c:326:29 (libtoolbox.so.0+0x00000001e9d4)
    #3 gop_init_opque_system /home/meloam/lstore/src/gop/opque.c:114:24 (libgop.so.0+0x0000000231ba)
    #4 gop_construct_fn /home/meloam/lstore/src/gop/constructor.c:27:5 (libgop.so.0+0x00000000d2f5)
    #5 _dl_init_internal <null> (ld-linux-x86-64.so.2+0x00000000f3a2)
    #6 <null> <null> (libltdl.so.7+0x0000000066a1)
    #7 globus_extension_activate <null> (libglobus_common.so.0+0x000000073628)
    #8 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #9 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #10 main <null> (globus-gridftp-server+0x0000004acbc2)

  Mutex M2504 (0x7d90011fa0a8) created at:
    #0 pthread_mutex_init /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1119 (globus-gridftp-server+0x000000433f55)
    #1 apr_thread_mutex_create /home/meloam/lstore/vendor/apr-accre/locks/unix/thread_mutex.c:71:14 (liblio.so.0+0x0000001a8b26)
    #2 mq_task_send /home/meloam/lstore/src/gop/mq_portal.c:314:13 (libgop.so.0+0x000000030f72)
    #3 _gop_mq_submit_op /home/meloam/lstore/src/gop/mq_portal.c:1801:5 (libgop.so.0+0x00000003ccd0)
    #4 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:221:9 (libgop.so.0+0x00000000ee74)
    #5 gop_waitall /home/meloam/lstore/src/gop/gop.c:441:13 (libgop.so.0+0x000000011eb7)
    #6 gop_sync_exec /home/meloam/lstore/src/gop/gop.c:616:11 (libgop.so.0+0x00000001382e)
    #7 lio_get_multiple_attrs /home/meloam/lstore/src/lio/lio_core_os.c:895:11 (liblio.so.0+0x000000055661)
    #8 lio_stat /home/meloam/lstore/src/lio/lio_core_os.c:1308:11 (liblio.so.0+0x00000005c60a)
    #9 plugin_stat /home/meloam/lstore/binding/gridftp/src/stat.c:55:14 (libglobus_gridftp_server_lstore.so+0x0000000054b1)
    #10 user_stat /home/meloam/lstore/binding/gridftp/src/thunk.c:132:19 (libglobus_gridftp_server_lstore.so+0x00000000709b)
    #11 globus_l_gfs_lstore_stat /home/meloam/lstore/binding/gridftp/src/lstore_dsi.c:201:18 (libglobus_gridftp_server_lstore.so+0x000000003588)
    #12 globus_l_gfs_blocking_dispatch_kickout <null> (libglobus_gridftp_server.so.6+0x000000046b54)
    #13 globus_l_gfs_authorize_cb <null> (libglobus_gridftp_server.so.6+0x0000000340c8)
    #14 globus_l_gfs_acl_kickout <null> (libglobus_gridftp_server.so.6+0x00000002a911)
    #15 globus_l_callback_thread_poll <null> (libglobus_common.so.0+0x00000002517e)
    #16 globus_l_thread_pool_thread_start <null> (libglobus_common.so.0+0x000000062cf4)
    #17 thread_starter <null> (libglobus_thread_pthread.so+0x000000003b30)

  Thread T35 (tid=6704, running) created by thread T2 at:
    #0 pthread_create /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (globus-gridftp-server+0x000000425266)
    #1 mq_conn_create_actual /home/meloam/lstore/src/gop/mq_portal.c:1488:5 (libgop.so.0+0x00000003b583)
    #2 mq_conn_create /home/meloam/lstore/src/gop/mq_portal.c:1517:15 (libgop.so.0+0x000000030b85)
    #3 gop_mq_submit /home/meloam/lstore/src/gop/mq_portal.c:274:15 (libgop.so.0+0x000000030762)
    #4 mq_task_send /home/meloam/lstore/src/gop/mq_portal.c:319:12 (libgop.so.0+0x000000031016)
    #5 _gop_mq_submit_op /home/meloam/lstore/src/gop/mq_portal.c:1801:5 (libgop.so.0+0x00000003ccd0)
    #6 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:221:9 (libgop.so.0+0x00000000ee74)
    #7 gop_waitall /home/meloam/lstore/src/gop/gop.c:441:13 (libgop.so.0+0x000000011eb7)
    #8 gop_sync_exec /home/meloam/lstore/src/gop/gop.c:616:11 (libgop.so.0+0x00000001382e)
    #9 lio_get_multiple_attrs /home/meloam/lstore/src/lio/lio_core_os.c:895:11 (liblio.so.0+0x000000055661)
    #10 lio_stat /home/meloam/lstore/src/lio/lio_core_os.c:1308:11 (liblio.so.0+0x00000005c60a)
    #11 plugin_stat /home/meloam/lstore/binding/gridftp/src/stat.c:55:14 (libglobus_gridftp_server_lstore.so+0x0000000054b1)
    #12 user_stat /home/meloam/lstore/binding/gridftp/src/thunk.c:132:19 (libglobus_gridftp_server_lstore.so+0x00000000709b)
    #13 globus_l_gfs_lstore_stat /home/meloam/lstore/binding/gridftp/src/lstore_dsi.c:201:18 (libglobus_gridftp_server_lstore.so+0x000000003588)
    #14 globus_l_gfs_blocking_dispatch_kickout <null> (libglobus_gridftp_server.so.6+0x000000046b54)
    #15 globus_l_gfs_authorize_cb <null> (libglobus_gridftp_server.so.6+0x0000000340c8)
    #16 globus_l_gfs_acl_kickout <null> (libglobus_gridftp_server.so.6+0x00000002a911)
    #17 globus_l_callback_thread_poll <null> (libglobus_common.so.0+0x00000002517e)
    #18 globus_l_thread_pool_thread_start <null> (libglobus_common.so.0+0x000000062cf4)
    #19 thread_starter <null> (libglobus_thread_pthread.so+0x000000003b30)

  Thread T2 (tid=6540, running) created by main thread at:
    #0 pthread_create /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (globus-gridftp-server+0x000000425266)
    #1 globus_l_pthread_thread_create <null> (libglobus_thread_pthread.so+0x0000000032df)
    #2 globus_thread_create <null> (libglobus_common.so.0+0x000000060f71)
    #3 globus_i_thread_start <null> (libglobus_common.so.0+0x0000000633c9)
    #4 globus_l_callback_activate <null> (libglobus_common.so.0+0x0000000206cc)
    #5 globus_module_activate_proxy <null> (libglobus_common.so.0+0x000000047d96)
    #6 globus_module_activate <null> (libglobus_common.so.0+0x000000048782)
    #7 globus_l_callback_activate <null> (libglobus_common.so.0+0x00000001816d)
    #8 globus_module_activate_proxy <null> (libglobus_common.so.0+0x000000047d96)
    #9 globus_module_activate <null> (libglobus_common.so.0+0x000000048782)
    #10 globus_l_common_activate <null> (libglobus_common.so.0+0x00000002daa6)
    #11 globus_module_activate_proxy <null> (libglobus_common.so.0+0x000000047d96)
    #12 globus_module_activate <null> (libglobus_common.so.0+0x000000048782)
    #13 main <null> (globus-gridftp-server+0x0000004ac8f1)

SUMMARY: ThreadSanitizer: data race /home/meloam/lstore/src/gop/mq_portal.c:1542:9 in gop_mq_conn_teardown

[tacketar]

Should be fixed with 8f08b07523a304d1d4714b85fa60139044cdb512. But it's hard to trigger that bug so it may need another iteration.