 |
 |
Hi, In the same stress test I'm running, I've got about 5 servers that stopped accepting clients. Stack trace below. I accidentally screwed up the process, so no core is available. However, I was able to get line numbers: (gdb) bt #0 0x000000355020d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x0000003550208e1a in _L_lock_1034 () from /lib64/libpthread.so.0 #2 0x0000003550208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x000000000043bb26 in Lock (Peer=..., opts=0) at ../../src/XrdSys/XrdSysPthread.hh:132 #4 XrdLink::Alloc (Peer=..., opts=0) at XrdLink.cc:247 #5 0x0000000000443a28 in XrdInet::Accept (this=0x14204b10, opts=<value optimized out>, timeout=<value optimized out>) at XrdInet.cc:58 #6 0x000000000043c4d2 in mainAccept (parg=<value optimized out>) at XrdMain.cc:101 #7 0x000000000043c673 in main (argc=9, argv=0x7ffff40c04f8) at XrdMain.cc:186 The relevant lines from XrdLink.cc are: // Return the link // statsMutex.Lock(); <-- line 247 LinkCountTot++; if (LinkCountMax == LinkCount++) LinkCountMax = LinkCount; statsMutex.UnLock(); So - somewhere, the statsMutex got locked and never unlocked. Ideas? Brian Thread 43 (Thread 0x4091b940 (LWP 4286)): #0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000481eb2 in XrdSysCondVar::Wait(int) () #2 0x0000000000438881 in XrdBuffManager::Reshape() () #3 0x0000000000438b99 in XrdReshaper(void*) () #4 0x0000000000481d17 in XrdSysThread_Xeq () #5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 42 (Thread 0x41923940 (LWP 4287)): #0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000481eb2 in XrdSysCondVar::Wait(int) () #2 0x000000000043fab2 in XrdScheduler::TimeSched() () #3 0x000000000043fb19 in XrdStartTSched(void*) () #4 0x0000000000481d17 in XrdSysThread_Xeq () #5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 41 (Thread 0x41a24940 (LWP 4288)): #0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6 #1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) () #2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) () #3 0x0000000000434925 in XrdXrootdProtocol::do_Read() () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 40 (Thread 0x41b25940 (LWP 4289)): #0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6 #1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) () #2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) () #3 0x0000000000434925 in XrdXrootdProtocol::do_Read() () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 39 (Thread 0x40168940 (LWP 4290)): #0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6 #1 0x000000000043cd0d in XrdPollE::Start(XrdSysSemaphore*, int&) () #2 0x000000000043c6e8 in XrdStartPolling(void*) () #3 0x0000000000481d17 in XrdSysThread_Xeq () #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 38 (Thread 0x41785940 (LWP 4291)): #0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6 #1 0x000000000043cd0d in XrdPollE::Start(XrdSysSemaphore*, int&) () #2 0x000000000043c6e8 in XrdStartPolling(void*) () #3 0x0000000000481d17 in XrdSysThread_Xeq () #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 37 (Thread 0x40ed8940 (LWP 4293)): #0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6 #1 0x000000000043cd0d in XrdPollE::Start(XrdSysSemaphore*, int&) () #2 0x000000000043c6e8 in XrdStartPolling(void*) () #3 0x0000000000481d17 in XrdSysThread_Xeq () #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 36 (Thread 0x405cc940 (LWP 4295)): #0 0x000000355020e1c1 in nanosleep () from /lib64/libpthread.so.0 #1 0x000000000045592a in XrdAccConfig_Refresh(void*) () #2 0x0000000000481d17 in XrdSysThread_Xeq () #3 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #4 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 35 (Thread 0x40a1c940 (LWP 4296)): #0 0x000000355020d91b in read () from /lib64/libpthread.so.0 #1 0x000000000047cd60 in XrdOucStream::GetLine() () #2 0x00000000004479a8 in XrdOfsEvr::recvEvents() () #3 0x0000000000447a69 in XrdOfsEvRecv(void*) () #4 0x0000000000481d17 in XrdSysThread_Xeq () #5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 34 (Thread 0x41611940 (LWP 4297)): #0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0 #1 0x0000000000447b2f in XrdOfsEvr::flushEvents() () #2 0x0000000000447bd9 in XrdOfsEvFlush(void*) () #3 0x0000000000481d17 in XrdSysThread_Xeq () #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 33 (Thread 0x40fff940 (LWP 4298)): #0 0x000000355020dc81 in recv () from /lib64/libpthread.so.0 #1 0x000000000046da37 in XrdCmsFinderTRG::Start() () #2 0x0000000000481d17 in XrdSysThread_Xeq () #3 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #4 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 32 (Thread 0x41e8c940 (LWP 4300)): #0 0x000000355020db3b in accept () from /lib64/libpthread.so.0 #1 0x0000000000477e24 in XrdNetSocket::Accept(int) () #2 0x0000000000422e0d in XrdXrootdAdmin::Start(XrdNetSocket*) () #3 0x0000000000423819 in XrdXrootdInitAdmin(void*) () #4 0x0000000000481d17 in XrdSysThread_Xeq () #5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 31 (Thread 0x41f8d940 (LWP 4317)): #0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaacd24835 in GCTaskManager::get_task(unsigned int) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaaacd25ae3 in GCTaskThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 30 (Thread 0x4208e940 (LWP 4318)): #0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaacd24835 in GCTaskManager::get_task(unsigned int) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaaacd25ae3 in GCTaskThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 29 (Thread 0x4218f940 (LWP 4319)): #0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaacd24835 in GCTaskManager::get_task(unsigned int) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaaacd25ae3 in GCTaskThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 28 (Thread 0x42290940 (LWP 4320)): #0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaacd24835 in GCTaskManager::get_task(unsigned int) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaaacd25ae3 in GCTaskThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 27 (Thread 0x41d3a940 (LWP 4321)): #0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7956 in os::PlatformEvent::park(long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacf7f09b in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaad0ee493 in VMThread::loop() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaaad0ee08e in VMThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 26 (Thread 0x42391940 (LWP 4322)): #0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaad06bf2a in ObjectMonitor::wait(long, bool, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaad069293 in ObjectSynchronizer::wait(Handle, long, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaace01b7e in JVM_MonitorWait () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaab4010c88 in ?? () #6 0x00000000151e5800 in ?? () #7 0x00002aaaad068eb5 in ObjectSynchronizer::fast_exit(oopDesc*, BasicLock*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #8 0x00002aaab41a81a8 in ?? () #9 0x0000000000000000 in ?? () Thread 25 (Thread 0x42492940 (LWP 4323)): #0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaad06bf2a in ObjectMonitor::wait(long, bool, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaad069293 in ObjectSynchronizer::wait(Handle, long, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaace01b7e in JVM_MonitorWait () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaab4010c88 in ?? () #6 0x00000000151ed000 in ?? () #7 0x00000000151ed878 in ?? () #8 0x00000000151ed000 in ?? () #9 0x0000000042491a08 in ?? () #10 0x0000000000000000 in ?? () Thread 24 (Thread 0x4045c940 (LWP 4324)): #0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0 #1 0x00002aaaacfa84a5 in check_pending_signals(bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacfa11c7 in signal_thread_entry(JavaThread*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaad09e452 in JavaThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 23 (Thread 0x40c50940 (LWP 4325)): #0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaacf7f5cd in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaacc68bf3 in CompileQueue::get() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaaacc6a781 in CompileBroker::compiler_thread_loop() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #6 0x00002aaaad0a4969 in compiler_thread_entry(JavaThread*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #7 0x00002aaaad09e452 in JavaThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #8 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #9 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #10 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 22 (Thread 0x41c26940 (LWP 4326)): #0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaacf7f5cd in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaacc68bf3 in CompileQueue::get() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaaacc6a781 in CompileBroker::compiler_thread_loop() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #6 0x00002aaaad0a4969 in compiler_thread_entry(JavaThread*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #7 0x00002aaaad09e452 in JavaThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #8 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #9 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #10 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 21 (Thread 0x42593940 (LWP 4327)): #0 0x000000355020aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7807 in os::PlatformEvent::park() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacf7ef65 in Monitor::IWait(Thread*, long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaacf7f72e in Monitor::wait(bool, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaacf28ec2 in LowMemoryDetector::low_memory_detector_thread_entry(JavaThread*, Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x00002aaaad09e452 in JavaThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #6 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #7 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #8 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 20 (Thread 0x42694940 (LWP 4328)): #0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002aaaacfa7956 in os::PlatformEvent::park(long) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #2 0x00002aaaacfa5b5b in os::sleep(Thread*, long, bool) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #3 0x00002aaaad09d54b in WatcherThread::run() () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #4 0x00002aaaacfa80df in java_start(Thread*) () from /usr/java/latest/jre/lib/amd64/server/libjvm.so #5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 19 (Thread 0x42795940 (LWP 4329)): #0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6 #1 0x00002aaaaf0529cd in iepoll () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so #2 0x00002aaaaf0528a4 in Java_sun_nio_ch_EPollArrayWrapper_epollWait () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so #3 0x00002aaab4157072 in ?? () #4 0x00002aaabfdc7750 in ?? () #5 0x00002aaab41358e6 in ?? () #6 0x00002aab6519ab40 in ?? () #7 0x00002aaab7953550 in ?? () #8 0x00002aaabfdc7420 in ?? () #9 0x00002aaab4136618 in ?? () #10 0x0000000042794470 in ?? () #11 0x0000000014cd0800 in ?? () #12 0x0000000000000001 in ?? () #13 0x00002aaab4150634 in ?? () #14 0x00002aaabfdc7750 in ?? () #15 0x00002aaa00000001 in ?? () #16 0x000000000000ea60 in ?? () #17 0x00002aaabfdc7888 in ?? () #18 0x0000000000000007 in ?? () #19 0x00002aaabfdc7750 in ?? () #20 0x0000000000000009 in ?? () #21 0x00000000427943a8 in ?? () #22 0x00002aaabc4c0ea8 in ?? () #23 0x00002aaab4161740 in ?? () #24 0x00002aab6519ab40 in ?? () #25 0x00002aaabfdc7390 in ?? () #26 0x00002aaabfdc7720 in ?? () #27 0x000000000000ea60 in ?? () #28 0x00002aaabfdc7738 in ?? () #29 0x00002aaabfdc7390 in ?? () #30 0x00002aaabfdc7720 in ?? () #31 0x00002aaabfdc7390 in ?? () #32 0x00002aaabfdc7738 in ?? () #33 0x00002aab651970f8 in ?? () #34 0x00002aab6519ab40 in ?? () #35 0x00002aab65197028 in ?? () #36 0x0000000000000009 in ?? () #37 0x0000000000000009 in ?? () #38 0x0000000000000009 in ?? () #39 0x0000000000000001 in ?? () #40 0x0000000000000009 in ?? () #41 0x0000000000099473 in ?? () #42 0x0000000000000000 in ?? () Thread 18 (Thread 0x406cd940 (LWP 802)): #0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6 #1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) () #2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) () #3 0x0000000000434925 in XrdXrootdProtocol::do_Read() () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 17 (Thread 0x42896940 (LWP 872)): #0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6 #1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) () #2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) () #3 0x0000000000434925 in XrdXrootdProtocol::do_Read() () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 16 (Thread 0x42997940 (LWP 1695)): #0 0x000000354f6cb186 in poll () from /lib64/libc.so.6 #1 0x000000000043afe5 in XrdLink::Recv(char*, int, int) () #2 0x000000000042df5e in XrdXrootdProtocol::getData(char const*, char*, int) () #3 0x000000000042e945 in XrdXrootdProtocol::Process(XrdLink*) () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 15 (Thread 0x41302940 (LWP 1789)): #0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6 #1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) () #2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) () #3 0x0000000000434925 in XrdXrootdProtocol::do_Read() () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x407ce940 (LWP 1935)): #0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6 #1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) () #2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) () #3 0x0000000000434925 in XrdXrootdProtocol::do_Read() () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 13 (Thread 0x40b1d940 (LWP 2272)): #0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6 #1 0x00002aaaaf0529cd in iepoll () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so #2 0x00002aaaaf0528a4 in Java_sun_nio_ch_EPollArrayWrapper_epollWait () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so #3 0x00002aaab4157072 in ?? () #4 0x00002aaabfb2eba0 in ?? () #5 0x00002aaab41358e6 in ?? () #6 0x00002aab649340f0 in ?? () #7 0x00002aaab4155e0c in ?? () #8 0x00002aaabfb2e870 in ?? () #9 0x00002aaab4136618 in ?? () #10 0x00002aaac0087b08 in ?? () #11 0x00002aaab4136618 in ?? () #12 0x0000000000000001 in ?? () #13 0x00002aaab4150634 in ?? () #14 0x00002aaabfb2eba0 in ?? () #15 0x00002aaa00000001 in ?? () #16 0x000000000000ea60 in ?? () #17 0x00002aaabfb2ecd8 in ?? () #18 0x0000000000000007 in ?? () #19 0x00002aaabfb2eba0 in ?? () #20 0x0000000000000009 in ?? () #21 0x00002aaab70c8050 in ?? () #22 0x00002aaabc4c0ea8 in ?? () #23 0x00002aaab4161740 in ?? () #24 0x00002aab649340f0 in ?? () #25 0x00002aaabfb2e7e0 in ?? () #26 0x00002aaabfb2eb70 in ?? () #27 0x000000000000ea60 in ?? () #28 0x00002aaabfb2eb88 in ?? () #29 0x00002aaabfb2e7e0 in ?? () #30 0x00002aaabfb2eb70 in ?? () #31 0x00002aaabfb2e7e0 in ?? () #32 0x00002aaabfb2eb88 in ?? () #33 0x00002aab64928620 in ?? () #34 0x00002aab649340f0 in ?? () #35 0x00002aab64928550 in ?? () #36 0x0000000000000009 in ?? () #37 0x0000000000000009 in ?? () #38 0x0000000000000009 in ?? () #39 0x0000000000000001 in ?? () #40 0x0000000000000009 in ?? () #41 0x0000000000045367 in ?? () #42 0x0000000000000000 in ?? () Thread 12 (Thread 0x41100940 (LWP 3448)): #0 0x000000354f6d4358 in epoll_wait () from /lib64/libc.so.6 #1 0x00002aaaaf0529cd in iepoll () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so #2 0x00002aaaaf0528a4 in Java_sun_nio_ch_EPollArrayWrapper_epollWait () from /usr/java/jdk1.6.0_21/jre/lib/amd64/libnio.so #3 0x00002aaab4157072 in ?? () #4 0x00002aaabf9e8448 in ?? () #5 0x00002aaab41358e6 in ?? () #6 0x00002aab6474b9b8 in ?? () #7 0x00002aaab4155e0c in ?? () #8 0x00002aaabf9e8118 in ?? () #9 0x00002aaab4136618 in ?? () #10 0x00002aaabfb2e870 in ?? () #11 0x00002aaab4136618 in ?? () #12 0x0000000000000001 in ?? () #13 0x00002aaab4150634 in ?? () #14 0x00002aaabf9e8448 in ?? () #15 0x00002aaa00000001 in ?? () #16 0x000000000000ea60 in ?? () #17 0x00002aaabf9e8580 in ?? () #18 0x0000000000000007 in ?? () #19 0x00002aaabf9e8448 in ?? () #20 0x0000000000000009 in ?? () #21 0x00002aaab70c8050 in ?? () #22 0x00002aaabc4c0ea8 in ?? () #23 0x00002aaab4161740 in ?? () #24 0x00002aab6474b9b8 in ?? () #25 0x00002aaabf9e8088 in ?? () #26 0x00002aaabf9e8418 in ?? () #27 0x000000000000ea60 in ?? () #28 0x00002aaabf9e8430 in ?? () #29 0x00002aaabf9e8088 in ?? () #30 0x00002aaabf9e8418 in ?? () #31 0x00002aaabf9e8088 in ?? () #32 0x00002aaabf9e8430 in ?? () #33 0x00002aab646041d8 in ?? () #34 0x00002aab6474b9b8 in ?? () #35 0x00002aab64604108 in ?? () #36 0x0000000000000009 in ?? () #37 0x0000000000000009 in ?? () #38 0x0000000000000009 in ?? () #39 0x0000000000000001 in ?? () #40 0x0000000000000009 in ?? () #41 0x0000000000043570 in ?? () #42 0x0000000000000000 in ?? () Thread 11 (Thread 0x41201940 (LWP 3552)): #0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0 #1 0x000000000043e92e in XrdScheduler::Run() () #2 0x000000000043eb19 in XrdStartWorking(void*) () #3 0x0000000000481d17 in XrdSysThread_Xeq () #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x41504940 (LWP 3862)): #0 0x000000354f6cb186 in poll () from /lib64/libc.so.6 #1 0x000000000043afe5 in XrdLink::Recv(char*, int, int) () #2 0x000000000042df5e in XrdXrootdProtocol::getData(char const*, char*, int) () #3 0x000000000042e945 in XrdXrootdProtocol::Process(XrdLink*) () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x42a98940 (LWP 3863)): #0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0 #1 0x000000000043e92e in XrdScheduler::Run() () #2 0x000000000043eb19 in XrdStartWorking(void*) () #3 0x0000000000481d17 in XrdSysThread_Xeq () #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x42b99940 (LWP 3864)): #0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0 #1 0x000000000043e92e in XrdScheduler::Run() () #2 0x000000000043eb19 in XrdStartWorking(void*) () #3 0x0000000000481d17 in XrdSysThread_Xeq () #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x42c9a940 (LWP 3865)): #0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0 #1 0x000000000043e92e in XrdScheduler::Run() () #2 0x000000000043eb19 in XrdStartWorking(void*) () #3 0x0000000000481d17 in XrdSysThread_Xeq () #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x42d9b940 (LWP 3866)): #0 0x000000354f6ccd83 in writev () from /lib64/libc.so.6 #1 0x000000000043aa06 in XrdLink::Send(iovec const*, int, int) () #2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) () #3 0x0000000000434925 in XrdXrootdProtocol::do_Read() () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x42e9c940 (LWP 3867)): #0 0x000000355020d89b in write () from /lib64/libpthread.so.0 #1 0x000000000043aac6 in XrdLink::Send(iovec const*, int, int) () #2 0x0000000000430004 in XrdXrootdResponse::Send(void*, int) () #3 0x0000000000434925 in XrdXrootdProtocol::do_Read() () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x42f9d940 (LWP 3868)): #0 0x000000355020d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x0000003550208e1a in _L_lock_1034 () from /lib64/libpthread.so.0 #2 0x0000003550208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000000439230 in XrdLink::syncStats(int*) () #4 0x000000000043a5c1 in XrdLink::Stats(char*, int, int) () #5 0x00000000004400f5 in XrdStats::Stats(int) () #6 0x00000000004403d4 in XrdStats::Report(char**, int, int) () #7 0x0000000000440496 in XrdStatsJob::DoIt() () #8 0x000000000043e9a8 in XrdScheduler::Run() () #9 0x000000000043eb19 in XrdStartWorking(void*) () #10 0x0000000000481d17 in XrdSysThread_Xeq () #11 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #12 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x4309e940 (LWP 3869)): #0 0x000000354f6cb186 in poll () from /lib64/libc.so.6 #1 0x000000000043afe5 in XrdLink::Recv(char*, int, int) () #2 0x000000000042df5e in XrdXrootdProtocol::getData(char const*, char*, int) () #3 0x000000000042e945 in XrdXrootdProtocol::Process(XrdLink*) () #4 0x0000000000439f80 in XrdLink::DoIt() () #5 0x000000000043e9a8 in XrdScheduler::Run() () #6 0x000000000043eb19 in XrdStartWorking(void*) () #7 0x0000000000481d17 in XrdSysThread_Xeq () #8 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #9 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x41403940 (LWP 5571)): #0 0x000000355020cd01 in sem_wait () from /lib64/libpthread.so.0 #1 0x000000000043e92e in XrdScheduler::Run() () #2 0x000000000043eb19 in XrdStartWorking(void*) () #3 0x0000000000481d17 in XrdSysThread_Xeq () #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x2b151fb5de20 (LWP 4285)): #0 0x000000355020d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x0000003550208e1a in _L_lock_1034 () from /lib64/libpthread.so.0 #2 0x0000003550208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x000000000043bb26 in XrdLink::Alloc(XrdNetPeer&, int) () #4 0x0000000000443a28 in XrdInet::Accept(int, int) () #5 0x000000000043c4d2 in mainAccept(void*) () #6 0x000000000043c673 in main () [root@red-gridftp4 ~]# pstack 4285 | head Thread 43 (Thread 0x4091b940 (LWP 4286)): #0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000481eb2 in XrdSysCondVar::Wait(int) () #2 0x0000000000438881 in XrdBuffManager::Reshape() () #3 0x0000000000438b99 in XrdReshaper(void*) () #4 0x0000000000481d17 in XrdSysThread_Xeq () #5 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #6 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 42 (Thread 0x41923940 (LWP 4287)): #0 0x000000355020b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 [root@red-gridftp4 ~]# pstack 4285 | tail #4 0x000000355020673d in start_thread () from /lib64/libpthread.so.0 #5 0x000000354f6d3f6d in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x2b151fb5de20 (LWP 4285)): #0 0x000000355020d4c4 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x0000003550208e1a in _L_lock_1034 () from /lib64/libpthread.so.0 #2 0x0000003550208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x000000000043bb26 in XrdLink::Alloc(XrdNetPeer&, int) () #4 0x0000000000443a28 in XrdInet::Accept(int, int) () #5 0x000000000043c4d2 in mainAccept(void*) () #6 0x000000000043c673 in main ()