And a backtrace of another run in gdb. It is somewhat strange that these crashes happen from XrdPfc::Prefetch() while it seems to be setting up the response-handler ... while the valgrind report before is for a normal read triggered by a user read request.
```
malloc(): mismatching next->prev_size (unsorted)
Thread 20 "xrootd" received signal SIGABRT, Aborted.
[Switching to Thread 0x7fffe6a27700 (LWP 2398935)]
0x00007ffff64dfa4f in raise () from /lib64/libc.so.6
Missing separate debuginfos, use: yum debuginfo-install expat-2.2.5-4.el8.x86_64 globus-common-18.11-1.el8.x86_64 globus-gsi-callback-6.2-1.el8.x86_64 globus-gsi-cert-utils-10.8-1.el8.x86_64 globus-gsi-credential-8.3-1.el8.x86_64 globus-gsi-openssl-error-4.3-1.el8.x86_64 globus-gsi-proxy-core-9.6-1.el8.x86_64 globus-gsi-proxy-ssl-6.5-1.el8.x86_64 globus-gsi-sysconfig-9.4-1.el8.x86_64 globus-gssapi-gsi-14.17-1.el8.x86_64 globus-openssl-module-5.2-1.el8.x86_64 lcmaps-1.6.6-1.14.osg35.el8.x86_64 lcmaps-plugins-basic-1.7.0-2.osg35.el8.x86_64 lcmaps-plugins-verify-proxy-1.5.11-1.1.osg35.el8.x86_64 lcmaps-plugins-voms-1.7.1-1.6.osg35.el8.x86_64 libblkid-2.32.1-32.el8.x86_64 libcap-2.48-2.el8.x86_64 libgcc-8.5.0-10.el8.x86_64 libgcrypt-1.8.5-6.el8.x86_64 libgpg-error-1.31-1.el8.x86_64 libmount-2.32.1-32.el8.x86_64 libselinux-2.9-5.el8.x86_64 libstdc++-8.5.0-10.el8.x86_64 libtool-ltdl-2.4.6-25.el8.x86_64 libuuid-2.32.1-32.el8.x86_64 libxml2-2.9.7-11.el8.x86_64 lz4-libs-1.8.3-3.el8_4.x86_64 openssl-libs-1.1.1k-5.el8_5.x86_64 pcre2-10.32-2.el8.x86_64 systemd-libs-239-56.el8.x86_64 voms-2.1.0-0.14.rc2.2.osg35.el8.x86_64 xrootd-lcmaps-1.7.8-3.osgup.el8.x86_64 xz-libs-5.2.4-3.el8.x86_64 zlib-1.2.11-17.el8.x86_64
(gdb) bt
#0 0x00007ffff64dfa4f in raise () from /lib64/libc.so.6
#1 0x00007ffff64b2db5 in abort () from /lib64/libc.so.6
#2 0x00007ffff6522057 in __libc_message () from /lib64/libc.so.6
#3 0x00007ffff65291bc in malloc_printerr () from /lib64/libc.so.6
#4 0x00007ffff652c3c4 in _int_malloc () from /lib64/libc.so.6
#5 0x00007ffff652d842 in malloc () from /lib64/libc.so.6
#6 0x00007ffff70a6d6c in operator new(unsigned long) () from /lib64/libstdc++.so.6
#7 0x00007fffec4f2606 in __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::allocate (this=<optimized out>, __n=1) at /usr/include/c++/8/ext/new_allocator.h:99
#8 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > >::allocate (__a=..., __n=1) at /usr/include/c++/8/bits/alloc_traits.h:436
#9 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_get_node (this=<optimized out>) at /usr/include/c++/8/bits/stl_tree.h:588
#10 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_create_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&> (this=<optimized out>) at /usr/include/c++/8/bits/stl_tree.h:642
#11 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node::operator()<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&> (this=<optimized out>, __arg={...}) at /usr/include/c++/8/bits/stl_tree.h:556
#12 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_clone_node<std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node> (this=0x7fff9a160468, __node_gen=...,
__x=0x7fff80010e10) at /usr/include/c++/8/bits/stl_tree.h:666
#13 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_copy<std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node> (this=this@entry=0x7fff9a160468, __x=0x7fff80010e10,
__p=__p@entry=0x7fff9a160470, __node_gen=...) at /usr/include/c++/8/bits/stl_tree.h:1841
#14 0x00007fffec52eccd in std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_copy<std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node> (__gen=..., __x=...,
this=0x7fff9a160468) at /usr/include/c++/8/bits/stl_tree.h:896
#15 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_copy (__x=..., this=0x7fff9a160468) at /usr/include/c++/8/bits/stl_tree.h:909
#16 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Rb_tree (__x=..., this=0x7fff9a160468) at /usr/include/c++/8/bits/stl_tree.h:947
#17 std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::map (this=<optimized out>) at /usr/include/c++/8/bits/stl_map.h:205
#18 XrdCl::URL::URL (this=<optimized out>) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/./XrdCl/XrdClURL.hh:30
#19 XrdCl::XRootDMsgHandler::XRootDMsgHandler (this=0x7fff9a160360, msg=0x7fff9a19d480, respHandler=<optimized out>, url=0x7ffee0000de0, sidMgr=std::shared_ptr<XrdCl::SIDManager> (use count 2, weak count 0) = {...},
lFileHandler=0x7fff8406ab20) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/./XrdCl/XrdClXRootDMsgHandler.hh:190
#20 0x00007fffec5297c5 in XrdCl::MessageUtils::SendMessage (url=..., msg=msg@entry=0x7fff9a19d480, handler=handler@entry=0x7fff9a165670, sendParams=..., lFileHandler=0x7fff8406ab20) at /usr/include/c++/8/ext/atomicity.h:96
#21 0x00007fffec53c18d in XrdCl::FileStateHandler::IssueRequest (this=0x7fff84059900, url=..., msg=0x7fff9a19d480, handler=0x7fff9a165670, sendParams=...)
at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFileStateHandler.cc:3302
#22 0x00007fffec53e4ca in XrdCl::FileStateHandler::SendOrQueue (this=0x7fff84059900, url=..., msg=<optimized out>, handler=0x7fff9a165670, sendParams=...)
at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFileStateHandler.cc:2921
#23 0x00007fffec53edc4 in XrdCl::FileStateHandler::Read (this=0x7fff84059900, offset=2322071552, size=524288, buffer=0x7fff989b8000, handler=0x7fff99887340, timeout=0) at /usr/include/c++/8/bits/basic_string.h:252
#24 0x00007fffec53f67d in XrdCl::FileStateHandler::PgRead (this=0x7fff84059900, offset=2322071552, size=524288, buffer=0x7fff989b8000, handler=0x7fff7405b4d0, timeout=0)
at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFileStateHandler.cc:1170
#25 0x00007fffec533691 in XrdCl::File::PgRead (this=this@entry=0x7fff84044ed8, offset=offset@entry=2322071552, size=size@entry=524288, buffer=buffer@entry=0x7fff989b8000, handler=handler@entry=0x7fff7405b4d0, timeout=0)
at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFile.cc:236
#26 0x00007fffec8125ad in XrdPosixFile::pgRead (this=0x7fff84044e40, iocb=...,
buff=0x7fff989b8000 "\247\277\261\342XUez\235\266\357\375\214\255\254\230U\372\341\362\207-\f\376\255S\364s@\n&\311$(P\\ͣ\311I3H\017H~씼\253\350n\247", <incomplete sequence \325>, offs=2322071552, rlen=524288, csvec=...,
opts=0, csfix=0x7fff9a19d530) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPosix/XrdPosixFile.cc:490
#27 0x00007fffe7877ce8 in XrdPfc::File::ProcessBlockRequest (this=this@entry=0x7fff840e7880, b=0x7fff9a19d4e0) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/./XrdOuc/XrdOucCache.hh:334
#28 0x00007fffe7877d34 in XrdPfc::File::ProcessBlockRequests (this=this@entry=0x7fff840e7880, blks=std::__cxx11::list = {...}) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfcFile.cc:600
#29 0x00007fffe787cb1f in XrdPfc::File::Prefetch (this=0x7fff840e7880) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfcFile.cc:1343
#30 0x00007fffe7867b45 in XrdPfc::Cache::Prefetch (this=0x6fa820) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfc.cc:752
#31 0x00007fffe7867b71 in PrefetchThread () at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfc.cc:71
#32 0x00007ffff7805de7 in XrdSysThread_Xeq (myargs=0x718030) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdSys/XrdSysPthread.cc:86
#33 0x00007ffff685e17f in start_thread () from /lib64/libpthread.so.0
#34 0x00007ffff64cad83 in clone () from /lib64/libc.so.6
```
--
Reply to this email directly or view it on GitHub:
https://github.com/xrootd/xrootd/issues/1616#issuecomment-1035673138
You are receiving this because you commented.
Message ID: <[log in to unmask]>
########################################################################
Use REPLY-ALL to reply to list
To unsubscribe from the XROOTD-DEV list, click the following link:
https://listserv.slac.stanford.edu/cgi-bin/wa?SUBED1=XROOTD-DEV&A=1
