WARNING: Container does not have an exec helper script, calling '/srv/.osgvo-user-job-wrapper.sh' directly JANA ERROR>> =========================================================== There was a crash. This is the entire stack trace of all threads: =========================================================== Thread 9 (Thread 0x2b6f75986700 (LWP 239)): #0 0x00002b6f5e683965 in pthread_cond_wait GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000010862e0 in jana::JApplication::EventBufferThread (this=this entry=0x7ffea47a4ba0) at src/JANA/JApplication.cc:742 #2 0x00000000010865a9 in LaunchEventBufferThread (arg=0x7ffea47a4ba0) at src/JANA/JApplication.cc:666 #3 0x00002b6f5e67fdd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00002b6f5f3b4ead in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x2b6f75b87700 (LWP 240)): #0 0x00002b6f5f37be2d in nanosleep () from /lib64/libc.so.6 #1 0x00002b6f5f37bcc4 in sleep () from /lib64/libc.so.6 #2 0x00002b6f61c0af8c in GarbageCollectorThread (arg=0x2b6f78006730, thr=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientConnMgr.cc:93 #3 0x00002b6f61e6508f in XrdSysThread_Xeq (myargs=0x2b6f780089e0) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdSys/XrdSysPthread.cc:86 #4 0x00002b6f5e67fdd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00002b6f5f3b4ead in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x2b6f76189700 (LWP 241)): #0 0x00002b6f5f3aa20d in poll () from /lib64/libc.so.6 #1 0x00002b6f61bf4591 in XrdClientSock::RecvRaw (this=0x2b6f7810f220, buffer=0x2b6f840009c0, length=8, substreamid=, usedsubstreamid=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientSock.cc:156 #2 0x00002b6f61c140af in XrdClientPhyConnection::ReadRaw (this=0x2b6f7810dd40, buf=0x2b6f840009c0, len=8, substreamid=-1, usedsubstreamid=0x2b6f76182068) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:381 #3 0x00002b6f61c16b73 in XrdClientMessage::ReadRaw (this=0x2b6f84000980, phy=0x2b6f7810dd40) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientMessage.cc:172 #4 0x00002b6f61c12a58 in XrdClientPhyConnection::BuildMessage (this=0x2b6f7810dd40, IgnoreTimeouts=true, Enqueue=true) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:462 #5 0x00002b6f61c1579a in SocketReaderThread (arg=0x2b6f7810dd40, thr=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:79 #6 0x00002b6f61e6508f in XrdSysThread_Xeq (myargs=0x2b6f7810f170) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdSys/XrdSysPthread.cc:86 #7 0x00002b6f5e67fdd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00002b6f5f3b4ead in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x2b6f7638a700 (LWP 242)): #0 0x00002b6f5f37ba89 in waitpid () from /lib64/libc.so.6 #1 0x00002b6f5f2f9de2 in do_system () from /lib64/libc.so.6 #2 0x00002b6f5f2fa191 in system () from /lib64/libc.so.6 #3 0x00002b6f58c45439 in TUnixSystem::StackTrace (this=0x39c06b0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/root/root-6.08.06/core/unix/src/TUnixSystem.cxx:2405 #4 0x00002b6f58c47d0c in TUnixSystem::DispatchSignals (this=0x39c06b0, sig=kSigSegmentationViolation) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/root/root-6.08.06/core/unix/src/TUnixSystem.cxx:3625 #5 #6 clear (this=) at /usr/include/c++/4.8.2/bits/stl_vector.h:1126 #7 CopyTo (data=std::vector of length 0, capacity 0, this=0x3a0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JFactory.h:435 #8 DParsedEvent::CopyToFactories (this=this entry=0x2b6f982bf390, loop=loop entry=0x2b6f880008c0) at libraries/DAQ/DParsedEvent.h:260 #9 0x0000000000efeb13 in JEventSource_EVIOpp::GetObjects (this=0x2b6f7810a6b0, event=..., factory=) at libraries/DAQ/JEventSource_EVIOpp.cc:599 #10 0x0000000000aa57c7 in jana::JEvent::GetObjects (this=this entry=0x2b6f880008c8, t=std::vector of length 0, capacity 0, factory=factory entry=0x2b6f881a7040) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEvent.h:97 #11 0x0000000000aa5ae0 in GetFromSource (factory=0x2b6f881a7040, t=std::vector of length 0, capacity 0, this=0x2b6f880008c0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:477 #12 jana::JEventLoop::GetFromFactory (this=this entry=0x2b6f880008c0, t=std::vector of length 0, capacity 0, tag=tag entry=0x2b6f7557ad8f "", data_source= 0x2b6f76382e20: jana::JEventLoop::DATA_FROM_FACTORY, allow_deftag=allow_deftag entry=true) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:427 #13 0x0000000000aa5dc3 in jana::JEventLoop::Get (this=0x2b6f880008c0, t=std::vector of length 0, capacity 0, tag=0x2b6f7557ad8f "", allow_deftag=) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:309 #14 0x00002b6f7555a8ac in GetSingle (exception_if_not_one=true, tag=0x2b6f7557ad8f "", t=, this=0x2b6f880008c0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:239 #15 JEventProcessor_TPOL_tree::evnt (this=0x4efdbc0, loop=0x2b6f880008c0, eventnumber=389400255) at plugins/Analysis/TPOL_tree/JEventProcessor_TPOL_tree.cc:169 #16 0x00000000010a0b5a in jana::JEventLoop::OneEvent (this=this entry=0x2b6f880008c0) at src/JANA/JEventLoop.cc:693 #17 0x00000000010a1af4 in jana::JEventLoop::Loop (this=this entry=0x2b6f880008c0) at src/JANA/JEventLoop.cc:496 #18 0x0000000001078daa in LaunchThread (arg=0x7ffea47a4ba0) at src/JANA/JApplication.cc:1382 #19 0x00002b6f5e67fdd5 in start_thread () from /lib64/libpthread.so.0 #20 0x00002b6f5f3b4ead in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x2b6f7658b700 (LWP 243)): #0 0x00002b6f5f3aa20d in poll () from /lib64/libc.so.6 #1 0x00002b6f61bf4591 in XrdClientSock::RecvRaw (this=0x2b6f7810fcb0, buffer=0x2b6f8c000950, length=8, substreamid=, usedsubstreamid=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientSock.cc:156 #2 0x00002b6f61c140af in XrdClientPhyConnection::ReadRaw (this=0x2b6f7810afc0, buf=0x2b6f8c000950, len=8, substreamid=-1, usedsubstreamid=0x2b6f76584068) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:381 #3 0x00002b6f61c16b73 in XrdClientMessage::ReadRaw (this=0x2b6f8c000910, phy=0x2b6f7810afc0) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientMessage.cc:172 #4 0x00002b6f61c12a58 in XrdClientPhyConnection::BuildMessage (this=0x2b6f7810afc0, IgnoreTimeouts=true, Enqueue=true) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:462 #5 0x00002b6f61c1579a in SocketReaderThread (arg=0x2b6f7810afc0, thr=) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdClient/XrdClientPhyConnection.cc:79 #6 0x00002b6f61e6508f in XrdSysThread_Xeq (myargs=0x2b6f7810ac90) at /nfs/cern.ch/asis/xrootd-3.3.2/src/XrdSys/XrdSysPthread.cc:86 #7 0x00002b6f5e67fdd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00002b6f5f3b4ead in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x2b6f7678c700 (LWP 247)): #0 0x00002b6f5e683965 in pthread_cond_wait GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002b6f5eb4982c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x0000000000f0988b in async_filebuf::underflow (this=0x2b6f78003c60) at libraries/DAQ/async_filebuf.cc:140 #3 0x0000000000f088b8 in async_filebuf::xsgetn (this=0x2b6f78003c60, s=0x2b6f76785350 "8\343\331^o+", n=60) at libraries/DAQ/async_filebuf.cc:260 #4 0x00002b6f5eb177cb in std::istream::read(char*, long) () from /lib64/libstdc++.so.6 #5 0x0000000000ec8c27 in HDEVIO::readNoFileBuff (this=0x2b6f78004710, user_buff=0x2b6f90000fc0, user_buff_len=78678, allow_swap=allow_swap entry=false) at libraries/DAQ/HDEVIO.cc:527 #6 0x0000000000efc04b in JEventSource_EVIOpp::Dispatcher (this=0x2b6f7810a6b0) at libraries/DAQ/JEventSource_EVIOpp.cc:354 #7 0x00002b6f5eb4d070 in ?? () from /lib64/libstdc++.so.6 #8 0x00002b6f5e67fdd5 in start_thread () from /lib64/libpthread.so.0 #9 0x00002b6f5f3b4ead in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x2b6f7698d700 (LWP 248)): #0 0x00002b6f5e683d12 in pthread_cond_timedwait GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000f1ced4 in __gthread_cond_timedwait (__abs_timeout=0x2b6f769863f0, __mutex=, __cond=0x2b6f78006670) at /usr/include/c++/4.8.2/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x2b6f78006670) at /usr/include/c++/4.8.2/condition_variable:160 #3 wait_until > > (__atime=..., __lock=..., this=0x2b6f78006670) at /usr/include/c++/4.8.2/condition_variable:100 #4 wait_for > (__rtime=..., __lock=..., this=0x2b6f78006670) at /usr/include/c++/4.8.2/condition_variable:132 #5 DEVIOWorkerThread::Run (this=0x2b6f780065b0) at libraries/DAQ/DEVIOWorkerThread.cc:96 #6 0x00002b6f5eb4d070 in ?? () from /lib64/libstdc++.so.6 #7 0x00002b6f5e67fdd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00002b6f5f3b4ead in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x2b6f76b8e700 (LWP 249)): #0 0x00002b6f5e683965 in pthread_cond_wait GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00002b6f5eb4982c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x0000000000f092ea in async_filebuf::readloop (this=0x2b6f78003c60) at libraries/DAQ/async_filebuf.cc:104 #3 0x00002b6f5eb4d070 in ?? () from /lib64/libstdc++.so.6 #4 0x00002b6f5e67fdd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00002b6f5f3b4ead in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x2b6f57c989c0 (LWP 230)): #0 0x00002b6f5e686e3d in nanosleep () from /lib64/libpthread.so.0 #1 0x000000000108214d in jana::JApplication::Run (this=this entry=0x7ffea47a4ba0, proc=proc entry=0x0, Nthreads=, Nthreads entry=1) at src/JANA/JApplication.cc:1613 #2 0x000000000064162f in main (narg=17, argv=) at programs/Analysis/hd_ana/hd_ana.cc:24 =========================================================== The lines below might hint at the cause of the crash. You may get help by asking at the ROOT forum http://root.cern.ch/forum. Only if you are really convinced it is a bug in ROOT then please submit a report at http://root.cern.ch/bugs. Please post the ENTIRE stack trace from above as an attachment in addition to anything else that might help us fixing this issue. =========================================================== #6 clear (this=) at /usr/include/c++/4.8.2/bits/stl_vector.h:1126 #7 CopyTo (data=std::vector of length 0, capacity 0, this=0x3a0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JFactory.h:435 #8 DParsedEvent::CopyToFactories (this=this entry=0x2b6f982bf390, loop=loop entry=0x2b6f880008c0) at libraries/DAQ/DParsedEvent.h:260 #9 0x0000000000efeb13 in JEventSource_EVIOpp::GetObjects (this=0x2b6f7810a6b0, event=..., factory=) at libraries/DAQ/JEventSource_EVIOpp.cc:599 #10 0x0000000000aa57c7 in jana::JEvent::GetObjects (this=this entry=0x2b6f880008c8, t=std::vector of length 0, capacity 0, factory=factory entry=0x2b6f881a7040) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEvent.h:97 #11 0x0000000000aa5ae0 in GetFromSource (factory=0x2b6f881a7040, t=std::vector of length 0, capacity 0, this=0x2b6f880008c0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:477 #12 jana::JEventLoop::GetFromFactory (this=this entry=0x2b6f880008c0, t=std::vector of length 0, capacity 0, tag=tag entry=0x2b6f7557ad8f "", data_source= 0x2b6f76382e20: jana::JEventLoop::DATA_FROM_FACTORY, allow_deftag=allow_deftag entry=true) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:427 #13 0x0000000000aa5dc3 in jana::JEventLoop::Get (this=0x2b6f880008c0, t=std::vector of length 0, capacity 0, tag=0x2b6f7557ad8f "", allow_deftag=) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:309 #14 0x00002b6f7555a8ac in GetSingle (exception_if_not_one=true, tag=0x2b6f7557ad8f "", t=, this=0x2b6f880008c0) at /group/halld/Software/builds/Linux_CentOS7-x86_64-gcc4.8.5-cntr/jana/jana_0.7.9p1^ccdb166/Linux_CentOS7-x86_64-gcc4.8.5-cntr/include/JANA/JEventLoop.h:239 #15 JEventProcessor_TPOL_tree::evnt (this=0x4efdbc0, loop=0x2b6f880008c0, eventnumber=389400255) at plugins/Analysis/TPOL_tree/JEventProcessor_TPOL_tree.cc:169 #16 0x00000000010a0b5a in jana::JEventLoop::OneEvent (this=this entry=0x2b6f880008c0) at src/JANA/JEventLoop.cc:693 #17 0x00000000010a1af4 in jana::JEventLoop::Loop (this=this entry=0x2b6f880008c0) at src/JANA/JEventLoop.cc:496 #18 0x0000000001078daa in LaunchThread (arg=0x7ffea47a4ba0) at src/JANA/JApplication.cc:1382 #19 0x00002b6f5e67fdd5 in start_thread () from /lib64/libpthread.so.0 #20 0x00002b6f5f3b4ead in clone () from /lib64/libc.so.6 ===========================================================