And a backtrace of another run in gdb. It is somewhat strange that these crashes happen from XrdPfc::Prefetch() while it seems to be setting up the response-handler ... while the valgrind report before is for a normal read triggered by a user read request.

malloc(): mismatching next->prev_size (unsorted)

Thread 20 "xrootd" received signal SIGABRT, Aborted.
[Switching to Thread 0x7fffe6a27700 (LWP 2398935)]
0x00007ffff64dfa4f in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: yum debuginfo-install expat-2.2.5-4.el8.x86_64 globus-common-18.11-1.el8.x86_64 globus-gsi-callback-6.2-1.el8.x86_64 globus-gsi-cert-utils-10.8-1.el8.x86_64 globus-gsi-credential-8.3-1.el8.x86_64 globus-gsi-openssl-error-4.3-1.el8.x86_64 globus-gsi-proxy-core-9.6-1.el8.x86_64 globus-gsi-proxy-ssl-6.5-1.el8.x86_64 globus-gsi-sysconfig-9.4-1.el8.x86_64 globus-gssapi-gsi-14.17-1.el8.x86_64 globus-openssl-module-5.2-1.el8.x86_64 lcmaps-1.6.6-1.14.osg35.el8.x86_64 lcmaps-plugins-basic-1.7.0-2.osg35.el8.x86_64 lcmaps-plugins-verify-proxy-1.5.11-1.1.osg35.el8.x86_64 lcmaps-plugins-voms-1.7.1-1.6.osg35.el8.x86_64 libblkid-2.32.1-32.el8.x86_64 libcap-2.48-2.el8.x86_64 libgcc-8.5.0-10.el8.x86_64 libgcrypt-1.8.5-6.el8.x86_64 libgpg-error-1.31-1.el8.x86_64 libmount-2.32.1-32.el8.x86_64 libselinux-2.9-5.el8.x86_64 libstdc++-8.5.0-10.el8.x86_64 libtool-ltdl-2.4.6-25.el8.x86_64 libuuid-2.32.1-32.el8.x86_64 libxml2-2.9.7-11.el8.x86_64 lz4-libs-1.8.3-3.el8_4.x86_64 openssl-libs-1.1.1k-5.el8_5.x86_64 pcre2-10.32-2.el8.x86_64 systemd-libs-239-56.el8.x86_64 voms-2.1.0-0.14.rc2.2.osg35.el8.x86_64 xrootd-lcmaps-1.7.8-3.osgup.el8.x86_64 xz-libs-5.2.4-3.el8.x86_64 zlib-1.2.11-17.el8.x86_64
(gdb) bt
#0  0x00007ffff64dfa4f in raise () from /lib64/libc.so.6
#1  0x00007ffff64b2db5 in abort () from /lib64/libc.so.6
#2  0x00007ffff6522057 in __libc_message () from /lib64/libc.so.6
#3  0x00007ffff65291bc in malloc_printerr () from /lib64/libc.so.6
#4  0x00007ffff652c3c4 in _int_malloc () from /lib64/libc.so.6
#5  0x00007ffff652d842 in malloc () from /lib64/libc.so.6
#6  0x00007ffff70a6d6c in operator new(unsigned long) () from /lib64/libstdc++.so.6
#7  0x00007fffec4f2606 in __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::allocate (this=<optimized out>, __n=1) at /usr/include/c++/8/ext/new_allocator.h:99
#8  std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > >::allocate (__a=..., __n=1) at /usr/include/c++/8/bits/alloc_traits.h:436
#9  std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_get_node (this=<optimized out>) at /usr/include/c++/8/bits/stl_tree.h:588
#10 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_create_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&> (this=<optimized out>) at /usr/include/c++/8/bits/stl_tree.h:642
#11 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node::operator()<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&> (this=<optimized out>, __arg={...}) at /usr/include/c++/8/bits/stl_tree.h:556
#12 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_clone_node<std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node> (this=0x7fff9a160468, __node_gen=...,
    __x=0x7fff80010e10) at /usr/include/c++/8/bits/stl_tree.h:666
#13 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_copy<std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node> (this=this@entry=0x7fff9a160468, __x=0x7fff80010e10,
    __p=__p@entry=0x7fff9a160470, __node_gen=...) at /usr/include/c++/8/bits/stl_tree.h:1841
#14 0x00007fffec52eccd in std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_copy<std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node> (__gen=..., __x=...,
    this=0x7fff9a160468) at /usr/include/c++/8/bits/stl_tree.h:896
#15 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_copy (__x=..., this=0x7fff9a160468) at /usr/include/c++/8/bits/stl_tree.h:909
#16 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Rb_tree (__x=..., this=0x7fff9a160468) at /usr/include/c++/8/bits/stl_tree.h:947
#17 std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::map (this=<optimized out>) at /usr/include/c++/8/bits/stl_map.h:205
#18 XrdCl::URL::URL (this=<optimized out>) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/./XrdCl/XrdClURL.hh:30
#19 XrdCl::XRootDMsgHandler::XRootDMsgHandler (this=0x7fff9a160360, msg=0x7fff9a19d480, respHandler=<optimized out>, url=0x7ffee0000de0, sidMgr=std::shared_ptr<XrdCl::SIDManager> (use count 2, weak count 0) = {...},
    lFileHandler=0x7fff8406ab20) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/./XrdCl/XrdClXRootDMsgHandler.hh:190
#20 0x00007fffec5297c5 in XrdCl::MessageUtils::SendMessage (url=..., msg=msg@entry=0x7fff9a19d480, handler=handler@entry=0x7fff9a165670, sendParams=..., lFileHandler=0x7fff8406ab20) at /usr/include/c++/8/ext/atomicity.h:96
#21 0x00007fffec53c18d in XrdCl::FileStateHandler::IssueRequest (this=0x7fff84059900, url=..., msg=0x7fff9a19d480, handler=0x7fff9a165670, sendParams=...)
    at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFileStateHandler.cc:3302
#22 0x00007fffec53e4ca in XrdCl::FileStateHandler::SendOrQueue (this=0x7fff84059900, url=..., msg=<optimized out>, handler=0x7fff9a165670, sendParams=...)
    at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFileStateHandler.cc:2921
#23 0x00007fffec53edc4 in XrdCl::FileStateHandler::Read (this=0x7fff84059900, offset=2322071552, size=524288, buffer=0x7fff989b8000, handler=0x7fff99887340, timeout=0) at /usr/include/c++/8/bits/basic_string.h:252
#24 0x00007fffec53f67d in XrdCl::FileStateHandler::PgRead (this=0x7fff84059900, offset=2322071552, size=524288, buffer=0x7fff989b8000, handler=0x7fff7405b4d0, timeout=0)
    at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFileStateHandler.cc:1170
#25 0x00007fffec533691 in XrdCl::File::PgRead (this=this@entry=0x7fff84044ed8, offset=offset@entry=2322071552, size=size@entry=524288, buffer=buffer@entry=0x7fff989b8000, handler=handler@entry=0x7fff7405b4d0, timeout=0)
    at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFile.cc:236
#26 0x00007fffec8125ad in XrdPosixFile::pgRead (this=0x7fff84044e40, iocb=...,
    buff=0x7fff989b8000 "\247\277\261\342XUez\235\266\357\375\214\255\254\230U\372\341\362\207-\f\376\255S\364s@\n&\311$(P\\ͣ\311I3H\017H~씼\253\350n\247", <incomplete sequence \325>, offs=2322071552, rlen=524288, csvec=...,
    opts=0, csfix=0x7fff9a19d530) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPosix/XrdPosixFile.cc:490
#27 0x00007fffe7877ce8 in XrdPfc::File::ProcessBlockRequest (this=this@entry=0x7fff840e7880, b=0x7fff9a19d4e0) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/./XrdOuc/XrdOucCache.hh:334
#28 0x00007fffe7877d34 in XrdPfc::File::ProcessBlockRequests (this=this@entry=0x7fff840e7880, blks=std::__cxx11::list = {...}) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfcFile.cc:600
#29 0x00007fffe787cb1f in XrdPfc::File::Prefetch (this=0x7fff840e7880) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfcFile.cc:1343
#30 0x00007fffe7867b45 in XrdPfc::Cache::Prefetch (this=0x6fa820) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfc.cc:752
#31 0x00007fffe7867b71 in PrefetchThread () at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfc.cc:71
#32 0x00007ffff7805de7 in XrdSysThread_Xeq (myargs=0x718030) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdSys/XrdSysPthread.cc:86
#33 0x00007ffff685e17f in start_thread () from /lib64/libpthread.so.0
#34 0x00007ffff64cad83 in clone () from /lib64/libc.so.6

—
Reply to this email directly, view it on GitHub, or unsubscribe.
Triage notifications on the go with GitHub Mobile for iOS or Android.
You are receiving this because you commented.Message ID: <xrootd/xrootd/issues/1616/1035673138@github.com>

Use REPLY-ALL to reply to list

To unsubscribe from the XROOTD-DEV list, click the following link:
https://listserv.slac.stanford.edu/cgi-bin/wa?SUBED1=XROOTD-DEV&A=1