Hi,
In the same stress test I'm running, I've got about 5 servers that stopped accepting clients. Stack trace below. I accidentally screwed up the process, so no core is available.
However, I was able to get line numbers:
(gdb) bt
#0 0x000000355020d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003550208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003550208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x000000000043bb26 in Lock (Peer=..., opts=0) at ../../src/XrdSys/XrdSysPthread.hh:132
#4 XrdLink::Alloc (Peer=..., opts=0) at XrdLink.cc:247
#5 0x0000000000443a28 in XrdInet::Accept (this=0x14204b10, opts=<value optimized out>, timeout=<value optimized out>) at XrdInet.cc:58
#6 0x000000000043c4d2 in mainAccept (parg=<value optimized out>) at XrdMain.cc:101
#7 0x000000000043c673 in main (argc=9, argv=0x7ffff40c04f8) at XrdMain.cc:186
The relevant lines from XrdLink.cc are:
// Return the link
//
statsMutex.Lock(); <-- line 247
LinkCountTot++;
if (LinkCountMax == LinkCount++) LinkCountMax = LinkCount;
statsMutex.UnLock();
So - somewhere, the statsMutex got locked and never unlocked.
Ideas?
Brian
Thread 43 (Thread 0x4091b940 (LWP 4286)):
#0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x0000000000481eb2 in XrdSysCondVar::Wait(int) ()
#2 0x0000000000438881 in XrdBuffManager::Reshape() ()
#3 0x0000000000438b99 in XrdReshaper(void*) ()
#4 0x0000000000481d17 in XrdSysThread_Xeq ()
#5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 42 (Thread 0x41923940 (LWP 4287)):
#0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x0000000000481eb2 in XrdSysCondVar::Wait(int) ()
#2 0x000000000043fab2 in XrdScheduler::TimeSched() ()
#3 0x000000000043fb19 in XrdStartTSched(void*) ()
#4 0x0000000000481d17 in XrdSysThread_Xeq ()
#5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 41 (Thread 0x41a24940 (LWP 4288)):
#0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6
#1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) ()
#2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) ()
#3 0x0000000000434925 in XrdXrootdProtocol::do_Read() ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 40 (Thread 0x41b25940 (LWP 4289)):
#0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6
#1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) ()
#2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) ()
#3 0x0000000000434925 in XrdXrootdProtocol::do_Read() ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 39 (Thread 0x40168940 (LWP 4290)):
#0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6
#1 0x000000000043cd0d in XrdPollE::Start(XrdSysSemaphore*, int&) ()
#2 0x000000000043c6e8 in XrdStartPolling(void*) ()
#3 0x0000000000481d17 in XrdSysThread_Xeq ()
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 38 (Thread 0x41785940 (LWP 4291)):
#0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6
#1 0x000000000043cd0d in XrdPollE::Start(XrdSysSemaphore*, int&) ()
#2 0x000000000043c6e8 in XrdStartPolling(void*) ()
#3 0x0000000000481d17 in XrdSysThread_Xeq ()
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 37 (Thread 0x40ed8940 (LWP 4293)):
#0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6
#1 0x000000000043cd0d in XrdPollE::Start(XrdSysSemaphore*, int&) ()
#2 0x000000000043c6e8 in XrdStartPolling(void*) ()
#3 0x0000000000481d17 in XrdSysThread_Xeq ()
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 36 (Thread 0x405cc940 (LWP 4295)):
#0 0x000000355020e1c1 in nanosleep () from /lib64/libpthread.so.0
#1 0x000000000045592a in XrdAccConfig_Refresh(void*) ()
#2 0x0000000000481d17 in XrdSysThread_Xeq ()
#3 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#4 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 35 (Thread 0x40a1c940 (LWP 4296)):
#0 0x000000355020d91b in read () from /lib64/libpthread.so.0
#1 0x000000000047cd60 in XrdOucStream::GetLine() ()
#2 0x00000000004479a8 in XrdOfsEvr::recvEvents() ()
#3 0x0000000000447a69 in XrdOfsEvRecv(void*) ()
#4 0x0000000000481d17 in XrdSysThread_Xeq ()
#5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 34 (Thread 0x41611940 (LWP 4297)):
#0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0
#1 0x0000000000447b2f in XrdOfsEvr::flushEvents() ()
#2 0x0000000000447bd9 in XrdOfsEvFlush(void*) ()
#3 0x0000000000481d17 in XrdSysThread_Xeq ()
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 33 (Thread 0x40fff940 (LWP 4298)):
#0 0x000000355020dc81 in recv () from /lib64/libpthread.so.0
#1 0x000000000046da37 in XrdCmsFinderTRG::Start() ()
#2 0x0000000000481d17 in XrdSysThread_Xeq ()
#3 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#4 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 32 (Thread 0x41e8c940 (LWP 4300)):
#0 0x000000355020db3b in accept () from /lib64/libpthread.so.0
#1 0x0000000000477e24 in XrdNetSocket::Accept(int) ()
#2 0x0000000000422e0d in XrdXrootdAdmin::Start(XrdNetSocket*) ()
#3 0x0000000000423819 in XrdXrootdInitAdmin(void*) ()
#4 0x0000000000481d17 in XrdSysThread_Xeq ()
#5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 31 (Thread 0x41f8d940 (LWP 4317)):
#0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaacd24835 in GCTaskManager::get_task(unsigned int) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaaacd25ae3 in GCTaskThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 30 (Thread 0x4208e940 (LWP 4318)):
#0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaacd24835 in GCTaskManager::get_task(unsigned int) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaaacd25ae3 in GCTaskThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 29 (Thread 0x4218f940 (LWP 4319)):
#0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaacd24835 in GCTaskManager::get_task(unsigned int) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaaacd25ae3 in GCTaskThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 28 (Thread 0x42290940 (LWP 4320)):
#0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaacd24835 in GCTaskManager::get_task(unsigned int) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaaacd25ae3 in GCTaskThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 27 (Thread 0x41d3a940 (LWP 4321)):
#0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7956 in os::PlatformEvent::park(long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacf7f09b in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaad0ee493 in VMThread::loop() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaaad0ee08e in VMThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 26 (Thread 0x42391940 (LWP 4322)):
#0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaad06bf2a in ObjectMonitor::wait(long, bool, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaad069293 in ObjectSynchronizer::wait(Handle, long, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaace01b7e in JVM_MonitorWait () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaab4010c88 in ?? ()
#6 0x00000000151e5800 in ?? ()
#7 0x00002aaaad068eb5 in ObjectSynchronizer::fast_exit(oopDesc*, BasicLock*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#8 0x00002aaab41a81a8 in ?? ()
#9 0x0000000000000000 in ?? ()
Thread 25 (Thread 0x42492940 (LWP 4323)):
#0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaad06bf2a in ObjectMonitor::wait(long, bool, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaad069293 in ObjectSynchronizer::wait(Handle, long, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaace01b7e in JVM_MonitorWait () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaab4010c88 in ?? ()
#6 0x00000000151ed000 in ?? ()
#7 0x00000000151ed878 in ?? ()
#8 0x00000000151ed000 in ?? ()
#9 0x0000000042491a08 in ?? ()
#10 0x0000000000000000 in ?? ()
Thread 24 (Thread 0x4045c940 (LWP 4324)):
#0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0
#1 0x00002aaaacfa84a5 in check_pending_signals(bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacfa11c7 in signal_thread_entry(JavaThread*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaad09e452 in JavaThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 23 (Thread 0x40c50940 (LWP 4325)):
#0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaacf7f5cd in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaacc68bf3 in CompileQueue::get() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaaacc6a781 in CompileBroker::compiler_thread_loop() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#6 0x00002aaaad0a4969 in compiler_thread_entry(JavaThread*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#7 0x00002aaaad09e452 in JavaThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#8 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#9 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#10 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 22 (Thread 0x41c26940 (LWP 4326)):
#0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaacf7f5cd in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaacc68bf3 in CompileQueue::get() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaaacc6a781 in CompileBroker::compiler_thread_loop() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#6 0x00002aaaad0a4969 in compiler_thread_entry(JavaThread*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#7 0x00002aaaad09e452 in JavaThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#8 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#9 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#10 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 21 (Thread 0x42593940 (LWP 4327)):
#0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaacf28ec2 in LowMemoryDetector::low_memory_detector_thread_entry(JavaThread*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x00002aaaad09e452 in JavaThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 20 (Thread 0x42694940 (LWP 4328)):
#0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaacfa7956 in os::PlatformEvent::park(long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#2 0x00002aaaacfa5b5b in os::sleep(Thread*, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#3 0x00002aaaad09d54b in WatcherThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#4 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so
#5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 19 (Thread 0x42795940 (LWP 4329)):
#0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aaaaf0529cd in iepoll () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so
#2 0x00002aaaaf0528a4 in Java_sun_nio_ch_EPollArrayWrapper_epollWait () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so
#3 0x00002aaab4157072 in ?? ()
#4 0x00002aaabfdc7750 in ?? ()
#5 0x00002aaab41358e6 in ?? ()
#6 0x00002aab6519ab40 in ?? ()
#7 0x00002aaab7953550 in ?? ()
#8 0x00002aaabfdc7420 in ?? ()
#9 0x00002aaab4136618 in ?? ()
#10 0x0000000042794470 in ?? ()
#11 0x0000000014cd0800 in ?? ()
#12 0x0000000000000001 in ?? ()
#13 0x00002aaab4150634 in ?? ()
#14 0x00002aaabfdc7750 in ?? ()
#15 0x00002aaa00000001 in ?? ()
#16 0x000000000000ea60 in ?? ()
#17 0x00002aaabfdc7888 in ?? ()
#18 0x0000000000000007 in ?? ()
#19 0x00002aaabfdc7750 in ?? ()
#20 0x0000000000000009 in ?? ()
#21 0x00000000427943a8 in ?? ()
#22 0x00002aaabc4c0ea8 in ?? ()
#23 0x00002aaab4161740 in ?? ()
#24 0x00002aab6519ab40 in ?? ()
#25 0x00002aaabfdc7390 in ?? ()
#26 0x00002aaabfdc7720 in ?? ()
#27 0x000000000000ea60 in ?? ()
#28 0x00002aaabfdc7738 in ?? ()
#29 0x00002aaabfdc7390 in ?? ()
#30 0x00002aaabfdc7720 in ?? ()
#31 0x00002aaabfdc7390 in ?? ()
#32 0x00002aaabfdc7738 in ?? ()
#33 0x00002aab651970f8 in ?? ()
#34 0x00002aab6519ab40 in ?? ()
#35 0x00002aab65197028 in ?? ()
#36 0x0000000000000009 in ?? ()
#37 0x0000000000000009 in ?? ()
#38 0x0000000000000009 in ?? ()
#39 0x0000000000000001 in ?? ()
#40 0x0000000000000009 in ?? ()
#41 0x0000000000099473 in ?? ()
#42 0x0000000000000000 in ?? ()
Thread 18 (Thread 0x406cd940 (LWP 802)):
#0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6
#1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) ()
#2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) ()
#3 0x0000000000434925 in XrdXrootdProtocol::do_Read() ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 17 (Thread 0x42896940 (LWP 872)):
#0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6
#1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) ()
#2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) ()
#3 0x0000000000434925 in XrdXrootdProtocol::do_Read() ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 16 (Thread 0x42997940 (LWP 1695)):
#0 0x000000354f6cb186 in poll () from /lib64/libc.so.6
#1 0x000000000043afe5 in XrdLink::Recv(char*, int, int) ()
#2 0x000000000042df5e in XrdXrootdProtocol::getData(char const*, char*, int) ()
#3 0x000000000042e945 in XrdXrootdProtocol::Process(XrdLink*) ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x41302940 (LWP 1789)):
#0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6
#1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) ()
#2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) ()
#3 0x0000000000434925 in XrdXrootdProtocol::do_Read() ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x407ce940 (LWP 1935)):
#0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6
#1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) ()
#2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) ()
#3 0x0000000000434925 in XrdXrootdProtocol::do_Read() ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x40b1d940 (LWP 2272)):
#0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aaaaf0529cd in iepoll () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so
#2 0x00002aaaaf0528a4 in Java_sun_nio_ch_EPollArrayWrapper_epollWait () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so
#3 0x00002aaab4157072 in ?? ()
#4 0x00002aaabfb2eba0 in ?? ()
#5 0x00002aaab41358e6 in ?? ()
#6 0x00002aab649340f0 in ?? ()
#7 0x00002aaab4155e0c in ?? ()
#8 0x00002aaabfb2e870 in ?? ()
#9 0x00002aaab4136618 in ?? ()
#10 0x00002aaac0087b08 in ?? ()
#11 0x00002aaab4136618 in ?? ()
#12 0x0000000000000001 in ?? ()
#13 0x00002aaab4150634 in ?? ()
#14 0x00002aaabfb2eba0 in ?? ()
#15 0x00002aaa00000001 in ?? ()
#16 0x000000000000ea60 in ?? ()
#17 0x00002aaabfb2ecd8 in ?? ()
#18 0x0000000000000007 in ?? ()
#19 0x00002aaabfb2eba0 in ?? ()
#20 0x0000000000000009 in ?? ()
#21 0x00002aaab70c8050 in ?? ()
#22 0x00002aaabc4c0ea8 in ?? ()
#23 0x00002aaab4161740 in ?? ()
#24 0x00002aab649340f0 in ?? ()
#25 0x00002aaabfb2e7e0 in ?? ()
#26 0x00002aaabfb2eb70 in ?? ()
#27 0x000000000000ea60 in ?? ()
#28 0x00002aaabfb2eb88 in ?? ()
#29 0x00002aaabfb2e7e0 in ?? ()
#30 0x00002aaabfb2eb70 in ?? ()
#31 0x00002aaabfb2e7e0 in ?? ()
#32 0x00002aaabfb2eb88 in ?? ()
#33 0x00002aab64928620 in ?? ()
#34 0x00002aab649340f0 in ?? ()
#35 0x00002aab64928550 in ?? ()
#36 0x0000000000000009 in ?? ()
#37 0x0000000000000009 in ?? ()
#38 0x0000000000000009 in ?? ()
#39 0x0000000000000001 in ?? ()
#40 0x0000000000000009 in ?? ()
#41 0x0000000000045367 in ?? ()
#42 0x0000000000000000 in ?? ()
Thread 12 (Thread 0x41100940 (LWP 3448)):
#0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aaaaf0529cd in iepoll () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so
#2 0x00002aaaaf0528a4 in Java_sun_nio_ch_EPollArrayWrapper_epollWait () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so
#3 0x00002aaab4157072 in ?? ()
#4 0x00002aaabf9e8448 in ?? ()
#5 0x00002aaab41358e6 in ?? ()
#6 0x00002aab6474b9b8 in ?? ()
#7 0x00002aaab4155e0c in ?? ()
#8 0x00002aaabf9e8118 in ?? ()
#9 0x00002aaab4136618 in ?? ()
#10 0x00002aaabfb2e870 in ?? ()
#11 0x00002aaab4136618 in ?? ()
#12 0x0000000000000001 in ?? ()
#13 0x00002aaab4150634 in ?? ()
#14 0x00002aaabf9e8448 in ?? ()
#15 0x00002aaa00000001 in ?? ()
#16 0x000000000000ea60 in ?? ()
#17 0x00002aaabf9e8580 in ?? ()
#18 0x0000000000000007 in ?? ()
#19 0x00002aaabf9e8448 in ?? ()
#20 0x0000000000000009 in ?? ()
#21 0x00002aaab70c8050 in ?? ()
#22 0x00002aaabc4c0ea8 in ?? ()
#23 0x00002aaab4161740 in ?? ()
#24 0x00002aab6474b9b8 in ?? ()
#25 0x00002aaabf9e8088 in ?? ()
#26 0x00002aaabf9e8418 in ?? ()
#27 0x000000000000ea60 in ?? ()
#28 0x00002aaabf9e8430 in ?? ()
#29 0x00002aaabf9e8088 in ?? ()
#30 0x00002aaabf9e8418 in ?? ()
#31 0x00002aaabf9e8088 in ?? ()
#32 0x00002aaabf9e8430 in ?? ()
#33 0x00002aab646041d8 in ?? ()
#34 0x00002aab6474b9b8 in ?? ()
#35 0x00002aab64604108 in ?? ()
#36 0x0000000000000009 in ?? ()
#37 0x0000000000000009 in ?? ()
#38 0x0000000000000009 in ?? ()
#39 0x0000000000000001 in ?? ()
#40 0x0000000000000009 in ?? ()
#41 0x0000000000043570 in ?? ()
#42 0x0000000000000000 in ?? ()
Thread 11 (Thread 0x41201940 (LWP 3552)):
#0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0
#1 0x000000000043e92e in XrdScheduler::Run() ()
#2 0x000000000043eb19 in XrdStartWorking(void*) ()
#3 0x0000000000481d17 in XrdSysThread_Xeq ()
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x41504940 (LWP 3862)):
#0 0x000000354f6cb186 in poll () from /lib64/libc.so.6
#1 0x000000000043afe5 in XrdLink::Recv(char*, int, int) ()
#2 0x000000000042df5e in XrdXrootdProtocol::getData(char const*, char*, int) ()
#3 0x000000000042e945 in XrdXrootdProtocol::Process(XrdLink*) ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x42a98940 (LWP 3863)):
#0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0
#1 0x000000000043e92e in XrdScheduler::Run() ()
#2 0x000000000043eb19 in XrdStartWorking(void*) ()
#3 0x0000000000481d17 in XrdSysThread_Xeq ()
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x42b99940 (LWP 3864)):
#0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0
#1 0x000000000043e92e in XrdScheduler::Run() ()
#2 0x000000000043eb19 in XrdStartWorking(void*) ()
#3 0x0000000000481d17 in XrdSysThread_Xeq ()
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x42c9a940 (LWP 3865)):
#0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0
#1 0x000000000043e92e in XrdScheduler::Run() ()
#2 0x000000000043eb19 in XrdStartWorking(void*) ()
#3 0x0000000000481d17 in XrdSysThread_Xeq ()
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x42d9b940 (LWP 3866)):
#0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6
#1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) ()
#2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) ()
#3 0x0000000000434925 in XrdXrootdProtocol::do_Read() ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x42e9c940 (LWP 3867)):
#0 0x000000355020d89b in write () from /lib64/libpthread.so.0
#1 0x000000000043aac6 in XrdLink::Send(iovec const*, int, int) ()
#2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) ()
#3 0x0000000000434925 in XrdXrootdProtocol::do_Read() ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x42f9d940 (LWP 3868)):
#0 0x000000355020d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003550208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003550208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000439230 in XrdLink::syncStats(int*) ()
#4 0x000000000043a5c1 in XrdLink::Stats(char*, int, int) ()
#5 0x00000000004400f5 in XrdStats::Stats(int) ()
#6 0x00000000004403d4 in XrdStats::Report(char**, int, int) ()
#7 0x0000000000440496 in XrdStatsJob::DoIt() ()
#8 0x000000000043e9a8 in XrdScheduler::Run() ()
#9 0x000000000043eb19 in XrdStartWorking(void*) ()
#10 0x0000000000481d17 in XrdSysThread_Xeq ()
#11 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#12 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x4309e940 (LWP 3869)):
#0 0x000000354f6cb186 in poll () from /lib64/libc.so.6
#1 0x000000000043afe5 in XrdLink::Recv(char*, int, int) ()
#2 0x000000000042df5e in XrdXrootdProtocol::getData(char const*, char*, int) ()
#3 0x000000000042e945 in XrdXrootdProtocol::Process(XrdLink*) ()
#4 0x0000000000439f80 in XrdLink::DoIt() ()
#5 0x000000000043e9a8 in XrdScheduler::Run() ()
#6 0x000000000043eb19 in XrdStartWorking(void*) ()
#7 0x0000000000481d17 in XrdSysThread_Xeq ()
#8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x41403940 (LWP 5571)):
#0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0
#1 0x000000000043e92e in XrdScheduler::Run() ()
#2 0x000000000043eb19 in XrdStartWorking(void*) ()
#3 0x0000000000481d17 in XrdSysThread_Xeq ()
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x2b151fb5de20 (LWP 4285)):
#0 0x000000355020d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003550208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003550208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x000000000043bb26 in XrdLink::Alloc(XrdNetPeer&, int) ()
#4 0x0000000000443a28 in XrdInet::Accept(int, int) ()
#5 0x000000000043c4d2 in mainAccept(void*) ()
#6 0x000000000043c673 in main ()
[root@red-gridftp4 ~]# pstack 4285 | head
Thread 43 (Thread 0x4091b940 (LWP 4286)):
#0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x0000000000481eb2 in XrdSysCondVar::Wait(int) ()
#2 0x0000000000438881 in XrdBuffManager::Reshape() ()
#3 0x0000000000438b99 in XrdReshaper(void*) ()
#4 0x0000000000481d17 in XrdSysThread_Xeq ()
#5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 42 (Thread 0x41923940 (LWP 4287)):
#0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
[root@red-gridftp4 ~]# pstack 4285 | tail
#4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0
#5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x2b151fb5de20 (LWP 4285)):
#0 0x000000355020d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003550208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003550208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x000000000043bb26 in XrdLink::Alloc(XrdNetPeer&, int) ()
#4 0x0000000000443a28 in XrdInet::Accept(int, int) ()
#5 0x000000000043c4d2 in mainAccept(void*) ()
#6 0x000000000043c673 in main ()
|
