 |
 |
Today we spotted another case that seems to share similarities with this one. This time we could take full stacktraces and a core dump: https://cernbox.cern.ch/s/uSQHjqHLfWnEHkp
(gdb) thread
[Current thread is 39 (Thread 0x7f510affc700 (LWP 11623))]
(gdb) bt
#0 0x00007f512f66a184 in pthread_rwlock_rdlock () from /lib64/libpthread.so.0
#1 0x00007f5133635e45 in XrdSysRWLock::ReadLock (this=0x7f510cc692c8) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysPthread.hh:419
#2 XrdSysRWLockHelper::XrdSysRWLockHelper (rd=true, l=..., this=<synthetic pointer>) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysPthread.hh:419
#3 XrdCl::PostMaster::Send (this=0x7f512ac0e2a0, url=..., msg=0x7f50ee8be240, handler=handler@entry=0x7f50ee8f7b00, stateful=stateful@entry=true, expires=1676649588) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClPostMaster.cc:220
#4 0x00007f5133666567 in XrdCl::XRootDMsgHandler::RetryAtServer (this=0x7f50ee8f7b00, url=..., entryType=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClXRootDMsgHandler.cc:2169
#5 0x00007f5133666b05 in XrdCl::XRootDMsgHandler::WaitDone (this=0x7f50ee8f7b00) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClXRootDMsgHandler.cc:1120
#6 0x00007f5133666bed in (anonymous namespace)::WaitTask::Run (this=<optimized out>, now=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClXRootDMsgHandler.cc:69
#7 0x00007f513364d3ac in XrdCl::TaskManager::RunTasks (this=0x7f510cc98180) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClTaskManager.cc:222
#8 0x00007f513364d509 in RunRunnerThread (arg=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClTaskManager.cc:38
#9 0x00007f512f666ea5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007f512f38fb0d in clone () from /lib64/libc.so.6
(gdb) p trIt
$36 = {task = 0x7f50ee4ac140, execTime = 0, own = true}
(gdb) p (XrdCl::Task)* 0x7f50ee4ac140
$37 = {_vptr.Task = 0x7f5133946ed8 <vtable for (anonymous namespace)::WaitTask+16>, pName = "WaitTask for: 0x0x7f50ee8be240"}
(gdb) f 1
#1 0x00007f5133635e45 in XrdSysRWLock::ReadLock (this=0x7f510cc692c8) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysPthread.hh:419
419 { if (rd) l.ReadLock();
(gdb) p *this
$38 = {lock = {__data = {__lock = 0, __nr_readers = 0, __readers_wakeup = 802, __writer_wakeup = 1987, __nr_readers_queued = 119, __nr_writers_queued = 2, __writer = 11617, __shared = 0, __pad1 = 0, __pad2 = 0, __flags = 0},
__size = "\000\000\000\000\000\000\000\000\"\003\000\000\303\a\000\000w\000\000\000\002\000\000\000a-", '\000' <repeats 29 times>, __align = 0}}
(gdb) t 36
[Switching to thread 36 (Thread 0x7f510c7ff700 (LWP 11617))]
#0 0x00007f512f66df40 in __pause_nocancel () from /lib64/libpthread.so.0
(gdb) bt
#0 0x00007f512f66df40 in __pause_nocancel () from /lib64/libpthread.so.0
#1 0x00007f512f664bcc in __pthread_mutex_lock_full () from /lib64/libpthread.so.0
#2 0x00007f5133637a7f in XrdSysMutex::Lock (this=0x7f50ee40f118) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysPthread.hh:281
#3 XrdSysMutexHelper::XrdSysMutexHelper (mutex=..., this=<synthetic pointer>) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysPthread.hh:281
#4 XrdCl::TickGeneratorTask::Invalidate (this=0x7f50ee40f100) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClChannel.cc:72
#5 XrdCl::Channel::~Channel (this=0x7f50fdc403c0, __in_chrg=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClChannel.cc:134
#6 0x00007f51336348bd in XrdCl::PostMaster::ForceDisconnect (this=0x7f512ac0e2a0, url=...) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClPostMaster.cc:317
#7 0x00007f513363dc6b in XrdCl::Stream::OnReadTimeout (this=0x7f50fdc5e540, substream=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClStream.cc:1064
#8 0x00007f51336ba87d in XrdCl::AsyncSocketHandler::OnReadTimeout (this=this@entry=0x7f50fdc58900) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClAsyncSocketHandler.cc:690
#9 0x00007f51336bcff7 in XrdCl::AsyncSocketHandler::Event (this=0x7f50fdc58900, type=2 '\002') at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClAsyncSocketHandler.cc:239
#10 0x00007f5133632368 in (anonymous namespace)::SocketCallBack::Event (this=0x7f50f93ce360, chP=<optimized out>, cbArg=<optimized out>, evFlags=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClPollerBuiltIn.cc:83
#11 0x00007f51332457db in XrdSys::IOEvents::Poller::CbkXeq (this=0x7f510ccc41e0, cP=0x7f51003be060, events=2, eNum=<optimized out>, eTxt=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysIOEvents.cc:721
#12 0x00007f5133245d9e in XrdSys::IOEvents::Poller::CbkTMO (this=0x7f510ccc41e0) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysIOEvents.cc:626
#13 0x00007f5133246ca1 in XrdSys::IOEvents::PollE::Begin (syncsem=<optimized out>, retcode=<optimized out>, eTxt=<optimized out>, this=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysIOEventsPollE.icc:216
#14 XrdSys::IOEvents::PollE::Begin (this=0x7f510ccc41e0, syncsem=<optimized out>, retcode=<optimized out>, eTxt=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysIOEventsPollE.icc:196
#15 0x00007f51332433bd in XrdSys::IOEvents::BootStrap::Start (parg=0x7f510dbfd980) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysIOEvents.cc:149
#16 0x00007f513324be27 in XrdSysThread_Xeq (myargs=0x7f510cc13940) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysPthread.cc:86
#17 0x00007f512f666ea5 in start_thread () from /lib64/libpthread.so.0
#18 0x00007f512f38fb0d in clone () from /lib64/libc.so.6
(gdb) f 2
#2 0x00007f5133637a7f in XrdSysMutex::Lock (this=0x7f50ee40f118) at /usr/src/debug/xrootd-5.5.4/src/XrdSys/XrdSysPthread.hh:281
281 {mutex.Lock();
(gdb) p *this
$39 = {cs = {__data = {__lock = -1052688063, __count = 1296384321, __owner = 909144128, __nusers = 926298934, __kind = 895692849, __spins = 14646, __elision = 12850, __list = {__prev = 0x68632e6e7265632e, __next = 0x302e353231313a}},
__size = "AAA\[log in to unmask]:1125.0", __align = 5567928264984445249}}
(gdb) p (XrdCl::Task)*0x7f50ee40f100
$40 = {_vptr.Task = 0x27, pName = <error reading variable: Cannot access memory at address 0xf>}
(gdb) f 5
#5 XrdCl::Channel::~Channel (this=0x7f50fdc403c0, __in_chrg=<optimized out>) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClChannel.cc:134
134 pTickGenerator->Invalidate();
(gdb) p *this
$41 = {pUrl = {pHostId = "[log in to unmask]:1094", pProtocol = "root", pUserName = "AAAAAADk", pPassword = "", pHostName = "eoshome-f.cern.ch", pPort = 1094, pPath = "", pParams = std::map with 16 elements = {["eos.app"] = "fuse::bi",
["eos.bookingsize"] = "0", ["eos.checksum"] = "ignore", ["eos.lfn"] = "ino:4b2fc24a0000000", ["fuse.exe"] = "/usr/bin/python3.6", ["fuse.gid"] = "2821", ["fuse.pid"] = "7676", ["fuse.uid"] = "115670", ["fuse.ver"] = "5.1.5",
["mgm.fusex"] = "1", ["mgm.mtime"] = "0", ["xrd.k5ccname"] = "/pool/condor/dir_7580/ffedship.cc", ["xrd.wantprot"] = "krb5,unix", ["xrdcl.requuid"] = "3f28c8b7-f52b-4806-b772-e9d453a7197c", ["xrdcl.secgid"] = "2821",
["xrdcl.secuid"] = "115670"},
pURL = "root:[log in to unmask]:1094/?eos.app=fuse::bi&eos.bookingsize=0&eos.checksum=ignore&eos.lfn=ino:4b2fc24a0000000&fuse.exe=/usr/bin/python3.6&fuse.gid=2821&fuse.pid=7676&fuse.uid=115670&fuse."...}, pPoller = 0x7f510cc98240,
pTransport = 0x7f510ccd1040, pTaskManager = 0x7f510cc98180, pStream = 0x7f50fdc5e540, pMutex = {cs = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}},
__size = '\000' <repeats 39 times>, __align = 0}}, pChannelData = {pHolder = 0x7f50fdc0e0a0, pTypeInfo = 0x7f5133948298 <typeinfo for XrdCl::XRootDChannelInfo*>, pOwn = true}, pIncoming = {pHandlers = std::map with 0 elements,
pMutex = {<XrdSysMutex> = {cs = {__data = {__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 1, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 16 times>, "\001", '\000' <repeats 22 times>,
__align = 0}}, <No data fields>}}, pTickGenerator = 0x7f50ee40f100, pJobManager = 0x7f510ccc4100}
(gdb) f 4
#4 XrdCl::TickGeneratorTask::Invalidate (this=0x7f50ee40f100) at /usr/src/debug/xrootd-5.5.4/src/XrdCl/XrdClChannel.cc:72
72 XrdSysMutexHelper lck( pMtx );
(gdb) p pMtx
$42 = {cs = {__data = {__lock = -1052688063, __count = 1296384321, __owner = 909144128, __nusers = 926298934, __kind = 895692849, __spins = 14646, __elision = 12850, __list = {__prev = 0x68632e6e7265632e, __next = 0x302e353231313a}},
__size = "AAA\[log in to unmask]:1125.0", __align = 5567928264984445249}}
(gdb) p this
$43 = (XrdCl::TickGeneratorTask * const) 0x7f50ee40f100
(gdb) p *this
$44 = {<XrdCl::Task> = {_vptr.Task = 0x27, pName = <error reading variable: Cannot access memory at address 0xf>}, pChannel = 0x1, pMtx = {cs = {__data = {__lock = -1052688063, __count = 1296384321, __owner = 909144128, __nusers = 926298934,
__kind = 895692849, __spins = 14646, __elision = 12850, __list = {__prev = 0x68632e6e7265632e, __next = 0x302e353231313a}}, __size = "AAA\[log in to unmask]:1125.0", __align = 5567928264984445249}}}
(gdb) p *pTaskManager
$45 = {pResolution = 1, pTasks = std::multiset with 1 element = {[0] = {task = 0x7f50ec501b20, execTime = 1676649532, own = true}}, pToBeUnregistered = empty std::list, pRunnerThread = 139986053613312, pRunning = true, pMutex = {cs = {__data = {
__lock = 0, __count = 0, __owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}}, pOpMutex = {cs = {__data = {__lock = 0, __count = 0,
__owner = 0, __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}}}
(gdb) p (XrdCl::Task) *0x7f50ec501b20
$46 = {_vptr.Task = 0x7f5133946ed8 <vtable for (anonymous namespace)::WaitTask+16>, pName = "WaitTask for: 0x0x7f50ffd78e80"}
(gdb) p (XrdCl::Message)*0x7f50ffd78e80
$62 = {<XrdCl::Buffer> = {_vptr.Buffer = 0x7f51339481a0 <vtable for XrdCl::Message+16>, pBuffer = 0x7f5100333800 "\002", pSize = 480, pCursor = 480}, pIsMarshalled = true, pSessionId = 0, pDescription = "", pVirtReqID = 0}
—
Reply to this email directly, view it on GitHub, or unsubscribe.
You are receiving this because you are subscribed to this thread.
Use REPLY-ALL to reply to list
To unsubscribe from the XROOTD-DEV list, click the following link:
https://listserv.slac.stanford.edu/cgi-bin/wa?SUBED1=XROOTD-DEV&A=1