LISTSERV 16.5 - XROOTD-DEV Archives

Well, that simply sounds like a memory corruption issue manifesting itlef 
in any number of ways.

Andy


On Thu, 10 Feb 2022, Matev? Tadel wrote:

> And a backtrace of another run in gdb. It is somewhat strange that these crashes happen from XrdPfc::Prefetch() while it seems to be setting up the response-handler ... while the valgrind report before is for a normal read triggered by a user read request.
> ```
> malloc(): mismatching next->prev_size (unsorted)
>
> Thread 20 "xrootd" received signal SIGABRT, Aborted.
> [Switching to Thread 0x7fffe6a27700 (LWP 2398935)]
> 0x00007ffff64dfa4f in raise () from /lib64/libc.so.6
> Missing separate debuginfos, use: yum debuginfo-install expat-2.2.5-4.el8.x86_64 globus-common-18.11-1.el8.x86_64 globus-gsi-callback-6.2-1.el8.x86_64 globus-gsi-cert-utils-10.8-1.el8.x86_64 globus-gsi-credential-8.3-1.el8.x86_64 globus-gsi-openssl-error-4.3-1.el8.x86_64 globus-gsi-proxy-core-9.6-1.el8.x86_64 globus-gsi-proxy-ssl-6.5-1.el8.x86_64 globus-gsi-sysconfig-9.4-1.el8.x86_64 globus-gssapi-gsi-14.17-1.el8.x86_64 globus-openssl-module-5.2-1.el8.x86_64 lcmaps-1.6.6-1.14.osg35.el8.x86_64 lcmaps-plugins-basic-1.7.0-2.osg35.el8.x86_64 lcmaps-plugins-verify-proxy-1.5.11-1.1.osg35.el8.x86_64 lcmaps-plugins-voms-1.7.1-1.6.osg35.el8.x86_64 libblkid-2.32.1-32.el8.x86_64 libcap-2.48-2.el8.x86_64 libgcc-8.5.0-10.el8.x86_64 libgcrypt-1.8.5-6.el8.x86_64 libgpg-error-1.31-1.el8.x86_64 libmount-2.32.1-32.el8.x86_64 libselinux-2.9-5.el8.x86_64 libstdc++-8.5.0-10.el8.x86_64 libtool-ltdl-2.4.6-25.el8.x86_64 libuuid-2.32.1-32.el8.x86_64 libxml2-2.9.7-11.el8.x86_64 lz4-libs-1.8.3-3.el8_
 4.x86_64 openssl-libs-1.1.1k-5.el8_5.x86_64 pcre2-10.32-2.el8.x86_64 systemd-libs-239-56.el8.x86_64 voms-2.1.0-0.14.rc2.2.osg35.el8.x86_64 xrootd-lcmaps-1.7.8-3.osgup.el8.x86_64 xz-libs-5.2.4-3.el8.x86_64 zlib-1.2.11-17.el8.x86_64
> (gdb) bt
> #0  0x00007ffff64dfa4f in raise () from /lib64/libc.so.6
> #1  0x00007ffff64b2db5 in abort () from /lib64/libc.so.6
> #2  0x00007ffff6522057 in __libc_message () from /lib64/libc.so.6
> #3  0x00007ffff65291bc in malloc_printerr () from /lib64/libc.so.6
> #4  0x00007ffff652c3c4 in _int_malloc () from /lib64/libc.so.6
> #5  0x00007ffff652d842 in malloc () from /lib64/libc.so.6
> #6  0x00007ffff70a6d6c in operator new(unsigned long) () from /lib64/libstdc++.so.6
> #7  0x00007fffec4f2606 in __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::allocate (this=<optimized out>, __n=1) at /usr/include/c++/8/ext/new_allocator.h:99
> #8  std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > >::allocate (__a=..., __n=1) at /usr/include/c++/8/bits/alloc_traits.h:436
> #9  std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_get_node (this=<optimized out>) at /usr/include/c++/8/bits/stl_tree.h:588
> #10 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_create_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&> (this=<optimized 
 out>) at /usr/include/c++/8/bits/stl_tree.h:642
> #11 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node::operator()<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&> (this=<o
 ptimized out>, __arg={...}) at /usr/include/c++/8/bits/stl_tree.h:556
> #12 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_clone_node<std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11:
 :basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node> (this=0x7fff9a160468, __node_gen=...,
>    __x=0x7fff80010e10) at /usr/include/c++/8/bits/stl_tree.h:666
> #13 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_copy<std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic
 _string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node> ***@***.***=0x7fff9a160468, __x=0x7fff80010e10,
>    ***@***.***=0x7fff9a160470, __node_gen=...) at /usr/include/c++/8/bits/stl_tree.h:1841
> #14 0x00007fffec52eccd in std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_copy<std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > cons
 t, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Alloc_node> (__gen=..., __x=...,
>    this=0x7fff9a160468) at /usr/include/c++/8/bits/stl_tree.h:896
> #15 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_copy (__x=..., this=0x7fff9a160468) at /usr/include/c++/8/bits/stl_tree.h:909
> #16 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Rb_tree (__x=..., this=0x7fff9a160468) at /usr/include/c++/8/bits/stl_tree.h:947
> #17 std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::map (this=<optimized out>) at /usr/include/c++/8/bits/stl_map.h:205
> #18 XrdCl::URL::URL (this=<optimized out>) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/./XrdCl/XrdClURL.hh:30
> #19 XrdCl::XRootDMsgHandler::XRootDMsgHandler (this=0x7fff9a160360, msg=0x7fff9a19d480, respHandler=<optimized out>, url=0x7ffee0000de0, sidMgr=std::shared_ptr<XrdCl::SIDManager> (use count 2, weak count 0) = {...},
>    lFileHandler=0x7fff8406ab20) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/./XrdCl/XrdClXRootDMsgHandler.hh:190
> #20 0x00007fffec5297c5 in XrdCl::MessageUtils::SendMessage (url=..., ***@***.***=0x7fff9a19d480, ***@***.***=0x7fff9a165670, sendParams=..., lFileHandler=0x7fff8406ab20) at /usr/include/c++/8/ext/atomicity.h:96
> #21 0x00007fffec53c18d in XrdCl::FileStateHandler::IssueRequest (this=0x7fff84059900, url=..., msg=0x7fff9a19d480, handler=0x7fff9a165670, sendParams=...)
>    at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFileStateHandler.cc:3302
> #22 0x00007fffec53e4ca in XrdCl::FileStateHandler::SendOrQueue (this=0x7fff84059900, url=..., msg=<optimized out>, handler=0x7fff9a165670, sendParams=...)
>    at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFileStateHandler.cc:2921
> #23 0x00007fffec53edc4 in XrdCl::FileStateHandler::Read (this=0x7fff84059900, offset=2322071552, size=524288, buffer=0x7fff989b8000, handler=0x7fff99887340, timeout=0) at /usr/include/c++/8/bits/basic_string.h:252
> #24 0x00007fffec53f67d in XrdCl::FileStateHandler::PgRead (this=0x7fff84059900, offset=2322071552, size=524288, buffer=0x7fff989b8000, handler=0x7fff7405b4d0, timeout=0)
>    at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFileStateHandler.cc:1170
> #25 0x00007fffec533691 in XrdCl::File::PgRead ***@***.***=0x7fff84044ed8, ***@***.***=2322071552, ***@***.***=524288, ***@***.***=0x7fff989b8000, ***@***.***=0x7fff7405b4d0, timeout=0)
>    at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdCl/XrdClFile.cc:236
> #26 0x00007fffec8125ad in XrdPosixFile::pgRead (this=0x7fff84044e40, iocb=...,
>    buff=0x7fff989b8000 "\247\277\261\342XUez\235\266\357\375\214\255\254\230U\372\341\362\207-\f\376\255S\364s@\n&\311$(P\\?\311I3H\017H~?\253\350n\247", <incomplete sequence \325>, offs=2322071552, rlen=524288, csvec=...,
>    opts=0, csfix=0x7fff9a19d530) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPosix/XrdPosixFile.cc:490
> #27 0x00007fffe7877ce8 in XrdPfc::File::ProcessBlockRequest ***@***.***=0x7fff840e7880, b=0x7fff9a19d4e0) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/./XrdOuc/XrdOucCache.hh:334
> #28 0x00007fffe7877d34 in XrdPfc::File::ProcessBlockRequests ***@***.***=0x7fff840e7880, blks=std::__cxx11::list = {...}) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfcFile.cc:600
> #29 0x00007fffe787cb1f in XrdPfc::File::Prefetch (this=0x7fff840e7880) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfcFile.cc:1343
> #30 0x00007fffe7867b45 in XrdPfc::Cache::Prefetch (this=0x6fa820) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfc.cc:752
> #31 0x00007fffe7867b71 in PrefetchThread () at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdPfc/XrdPfc.cc:71
> #32 0x00007ffff7805de7 in XrdSysThread_Xeq (myargs=0x718030) at /usr/src/debug/xrootd-5.4.1-0.rc1.osg35up.el8.x86_64/xrootd/src/XrdSys/XrdSysPthread.cc:86
> #33 0x00007ffff685e17f in start_thread () from /lib64/libpthread.so.0
> #34 0x00007ffff64cad83 in clone () from /lib64/libc.so.6
> ```
>
> --
> Reply to this email directly or view it on GitHub:
> https://github.com/xrootd/xrootd/issues/1616#issuecomment-1035673138
> You are receiving this because you are subscribed to this thread.
>
> Message ID: ***@***.***>

-- 
Reply to this email directly or view it on GitHub:
https://github.com/xrootd/xrootd/issues/1616#issuecomment-1035680675
You are receiving this because you commented.

Message ID: <[log in to unmask]>

########################################################################
Use REPLY-ALL to reply to list

To unsubscribe from the XROOTD-DEV list, click the following link:
https://listserv.slac.stanford.edu/cgi-bin/wa?SUBED1=XROOTD-DEV&A=1