Hi Fabrice, Fabrice Jammes wrote on 2015-03-13: > > int QservOss::Init(XrdSysLogger* log, const char* cfgFn) { > + XrdSsi::Log.logger(log); // Logger is the arg in XrdOssGetSS() Move this line inside XrdOssGetStorageSystem(), right before QservOss::getInstance(); (and change log argument to Logger). Andy. > _log = LOG_GET("QservOss"); > if(!cfgFn) { > _cfgFn.assign(""); > diff --git a/core/modules/xrdoss/SConscript.test > b/core/modules/xrdoss/SConscript.test index d2e152c..691090f 100644 --- > a/core/modules/xrdoss/SConscript.test +++ > b/core/modules/xrdoss/SConscript.test @@ -11,7 +11,7 @@ import itertools > > programs = [] > unbuntuDeps = ["pthread"] > -extDeps = unbuntuDeps + "boost_regex boost_system boost_thread > mysqlclient_r protobuf ssl crypto log log4cxx XrdClient > XrdUtils".split() +extDeps = unbuntuDeps + "boost_regex boost_system > boost_thread mysqlclient_r protobuf ssl crypto log log4cxx XrdClient > XrdSsi XrdUtils".split() > > modDeps = 'mysql obsolete sql wbase wconfig wpublish util'.split() # > deps on other modules deps = itertools.chain(*map(lambda m: > defaultTgts[m], modDeps)) > > Part2: > -------- > > [qserv@ccqserv101 ~]$ gdb > /qserv/stack/Linux64/xrootd/xssi-1.0.0/bin/cmsd -c core.25849 GNU gdb > (GDB) Red Hat Enterprise Linux 7.6.1-51.el7 Copyright (C) 2013 Free > Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later > <http://gnu.org/licenses/gpl.html> This is free software: you are free > to change and redistribute it. There is NO WARRANTY, to the extent > permitted by law. Type "show copying" and "show warranty" for details. > This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting > instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... > Reading symbols from /qserv/stack/Linux64/xrootd/xssi- > 1.0.0/bin/cmsd...done. [New LWP 25849] [New LWP 25854] [New LWP 25856] > [New LWP 25853] [New LWP 25858] [New LWP 25852] [New LWP 25857] [New LWP > 25859] [New LWP 25855] [Thread debugging using libthread_db enabled] > Using host libthread_db library "/lib64/libthread_db.so.1". Core was > generated by `/qserv/stack/Linux64/xrootd/xssi-1.0.0/bin/cmsd -c > /qserv/qserv-run/etc/lsp.cf'. Program terminated with signal 11, > Segmentation fault. #0 0x00007f19e82bcc10 in pthread_mutex_lock () from > /lib64/libpthread.so.0 Missing separate debuginfos, use: > debuginfo-install expat-2.1.0-8.el7.x86_64 glibc-2.17-55.el7_0.5.x86_64 > keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.11.3-49.el7.x86_64 > libcom_err-1.42.9-4.el7.x86_64 libgcc-4.8.2-16.2.el7_0.x86_64 > libicu-50.1.2-11.el7.x86_64 libselinux-2.2.2-6.el7.x86_64 > libstdc++-4.8.2-16.2.el7_0.x86_64 > nss-softokn-freebl-3.16.2.3-1.el7_0.x86_64 > openssl-libs-1.0.1e-34.el7_0.7.x86_64 pcre-8.32-12.el7.x86_64 > xz-libs-5.1.2-9alpha.el7.x86_64 zlib-1.2.7-13.el7.x86_64 (gdb) where #0 > 0x00007f19e82bcc10 in pthread_mutex_lock () from /lib64/libpthread.so.0 > #1 0x000000000041ed54 in XrdSysMutex::Lock (this=0x10) at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/./XrdSys/XrdSysPthread.hh:149 #2 0x00007f19e85082e6 in > XrdSysLogger::Put (this=0x0, iovcnt=2, iov=0x7fff6c28ae50) > at /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/XrdSys/XrdSysLogger.cc:270 > #3 0x00007f19e8507817 in XrdSysError::Say (this=0x7f19e4cdaee0 > <XrdSsi::Log>, > txt1=0x19012d8 "0313 16:15:37.271 [0x7f19e8c81740] INFO QservOss > (build/xrdoss/QservOss.cc:267) - QservOss Init", txt2=0x0, txt3=0x0, > txt4=0x0, txt5=0x0, txt6=0x0) > at /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/XrdSys/XrdSysError.cc:163 > #4 0x00007f19e4acb81d in XrdSsiLogger::Msg (pfx=0x0, txt1=0x19012d8 "0313 > 16:15:37.271 [0x7f19e8c81740] INFO QservOss > (build/xrdoss/QservOss.cc:267) - QservOss Init", txt2=0x0, > txt3=0x0) at /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi- > 1.0.0/xrootd-xssi-1.0.0/src/XrdSsi/XrdSsiLogger.cc:66 #5 > 0x00007f19e6755a9d in > lsst::qserv::wlog::XrootdAppender::append(this=0x18e79d0, event=..., > p=...) at build/wlog/XrootdAppender.cc:57 #6 0x00007f19e5a08d1e in > log4cxx::AppenderSkeleton::doAppend(this=0x18e79d0, event=..., > pool1=...) at appenderskeleton.cpp:142 #7 0x00007f19e5a06df4 in > log4cxx::helpers::AppenderAttachableImpl::appendLoopOnAppenders(this=0x18 > e 8340, event=..., p=...) at appenderattachableimpl.cpp:69 #8 > 0x00007f19e5a4c6d5 in > log4cxx::Logger::callAppenders(this=this@entry=0x18f5300, event=..., > p=...) at logger.cpp:93 #9 0x00007f19e5a4ca1f in > log4cxx::Logger::forcedLog (this=0x18f5300, level1=..., > message="QservOss Init", location=...) at logger.cpp:124 #10 > 0x00007f19e5d1d412 in lsst::log::Log::vlog (logger=..., level=..., > filename="build/xrdoss/QservOss.cc", > funcname="virtual int > lsst::qserv::xrdoss::QservOss::Init(XrdSysLogger*, const char*)", > lineno=lineno@entry=267, fmt=fmt@entry=0x7f19e677789b "QservOss Init", > args=args@entry=0x7fff6c28b698) at src/Log.cc:357 > #11 0x00007f19e5d1d734 in lsst::log::Log::log (logger=..., level=..., > filename="build/xrdoss/QservOss.cc", > funcname="virtual int > lsst::qserv::xrdoss::QservOss::Init(XrdSysLogger*, const char*)", > lineno=267, fmt=0x7f19e677789b "QservOss Init") at src/Log.cc:386 #12 > 0x00007f19e676b555 in > lsst::qserv::xrdoss::QservOss::Init(this=0x18f5690, log=0x0, cfgFn=0x0) > at build/xrdoss/QservOss.cc:267 #13 0x00007f19e676a587 in > lsst::qserv::xrdoss::QservOss::QservOss(this=0x18f5690) at > build/xrdoss/QservOss.cc:141 #14 0x00007f19e676a384 in > lsst::qserv::xrdoss::QservOss::getInstance () at > build/xrdoss/QservOss.cc:107 #15 0x00007f19e676b9a1 in > XrdOssGetStorageSystem (native_oss=0x7f19e8a70100 > <XrdOssGetSS(XrdSysLogger*, char const*, char const*, char const*, > XrdOucEnv*, XrdVersionInfo&)::myOssSys>, > Logger=0x65b3b8 <XrdMain::Config+248>, config_fn=0x18c8c40 > "/qserv/qserv-run/etc/lsp.cf", parms=0x0) at build/xrdoss/QservOss.cc:301 > #16 0x00007f19e881184d in XrdOssGetSS (Logger=0x65b3b8 > <XrdMain::Config+248>, config_fn=0x18c8c40 > "/qserv/qserv-run/etc/lsp.cf", OssLib=0x18c8f70 "libxrdoss.so", > OssParms=0x0, > envP=0x0, urVer=...) at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/XrdOss/XrdOssApi.cc:135 #17 0x000000000042f08d in > XrdCmsConfig::ConfigOSS (this=0x65d340 <XrdCms::Config>) > at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/XrdCms/XrdCmsConfig.cc:777 #18 0x000000000042dbc8 in > XrdCmsConfig::Configure2 (this=0x65d340 <XrdCms::Config>) > at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/XrdCms/XrdCmsConfig.cc:408 #19 0x0000000000443754 in > XrdgetProtocol (pname=0x18b9920 "cmsd", parms=0x0, pi=0x65b2c0 > <XrdMain::Config>) > at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/XrdCms/XrdCmsProtocol.cc:121 #20 0x000000000041de24 in > XrdProtLoad::getProtocol (lname=0x0, pname=0x18b9920 "cmsd", parms=0x0, > pi=0x65b2c0 <XrdMain::Config>) > at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/Xrd/XrdProtLoad.cc:247 #21 0x000000000041d8b4 in > XrdProtLoad::Load (lname=0x0, pname=0x18b9920 "cmsd", parms=0x0, > pi=0x65b2c0 <XrdMain::Config>) > at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/Xrd/XrdProtLoad.cc:101 #22 0x000000000041a5b7 in > XrdConfig::Setup (this=0x65b2c0 <XrdMain::Config>, dfltp=0x7fff6c28dd37 > "cmsd") > at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/Xrd/XrdConfig.cc:1028 #23 0x00000000004187de in > XrdConfig::Configure (this=0x65b2c0 <XrdMain::Config>, argc=9, > argv=0x7fff6c28c998) > at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/Xrd/XrdConfig.cc:509 #24 0x000000000041f13b in main (argc=9, > argv=0x7fff6c28c998) at > /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- > 1.0.0/src/Xrd/XrdMain.cc:179 (gdb) > > > > > On 03/12/2015 10:54 PM, Andrew Hanushevsky wrote: >> Hi Andy, >> >> Just run what Fabrice ran and it will core right away since >> XrdSysError wasn't told about XrdSysLogger and eventually the message >> is routed to that object. >> >> Andy >> >> On Fri, 13 Mar 2015, Salnikov, Andrei A. wrote: >> >>> Hi Andy, >>> >>> I'm confused, the little piece of code below is the fix, right? >>> Is it possible for me to reproduce crash? I'll try to remember >>> tomorrow what I did for logging, maybe I could see it better. >>> >>> Cheers, >>> Andy >>> >>> >>> Andrew Hanushevsky wrote on 2015-03-12: >>>> Hi Andy, >>>> >>>> I don't think there is a fix for this. Could you fix it? I already >>>> have a >>>> Jira entry for rewriting that portion of the code once I get a >>>> handle on >>>> how to deal with the repo. >>>> >>>> Andy >>>> >>>> On Thu, 12 Mar 2015, Salnikov, Andrei A. wrote: >>>> >>>>> Hi Andy, >>>>> >>>>> sorry for the mess I created. I do not remember exactly what is >>>>> involved there, but if you say that it fixes it then I'm happy. >>>>> Are you going to make that fix or do you want me to do it? Is >>>>> there a ticket for it already? >>>>> >>>>> Cheers, >>>>> Andy >>>>> >>>>> Andrew Hanushevsky wrote on 2015-03-12: >>>>>> Hi Andy, >>>>>> >>>>>> Daniel told me that you did the log4cxx integration for this part >>>>>> of qserv. Unfortunately, the underlying xrootd object that send >>>>>> messages to the log was not initialized when the >>>>>> XrdOssGetSS(Logger, ...) is called. This results in a SEGV when a >>>>>> message is issued. I know this is old code and now there is a >>>>>> better way of doing all of this (and I will do that later) for now >>>>>> the way to fix this is simply insert this code snippedin the >>>>>> initalization section: >>>>>> >>>>>> #include "XrdSys/XrdSysError.hh" >>>>>> >>>>>> namespace XrdSsi >>>>>> { >>>>>> extern XrdSysError Log; >>>>>> }; >>>>>> >>>>>> XrdSsi::Log.logger(Logger); // Logger is the arg in XrdOssGetSS() >>>>>> >>>>>> OK? Like I said this will become far simpler on the next iteration. >>>>>> >>>>>> Andy >>>>>> >>>>>> ---------- Forwarded message ---------- >>>>>> Date: Thu, 12 Mar 2015 14:14:12 -0700 >>>>>> From: Fabrice Jammes <[log in to unmask]> >>>>>> To: Andrew Hanushevsky <[log in to unmask]> >>>>>> Subject: cmsd core log >>>>>> >>>>>> Hi Andy, >>>>>> >>>>>> We introduced logging in this module a few months ago but our >>>>>> mono-node integration test doesn't launch cmsd, so it was never >>>>>> tested. It may be our logging library (lsst::log which relies on >>>>>> log4cxx) which is faulty? >>>>>> >>>>>> Thanks, >>>>>> >>>>>> Fabrice >>>>>> >>>>>> Program terminated with signal 11, Segmentation fault. #0 >>>>>> 0x00007fdb86a9fc10 in pthread_mutex_lock () from >>>>>> /lib64/libpthread.so.0 >>>>>> Missing separate debuginfos, use: debuginfo-install expat-2.1.0- >>>>>> 8.el7.x86_64 glibc-2.17-55.el7_0.5.x86_64 >>>>>> keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.11.3-49.el7.x86_64 >>>>>> libcom_err-1.42.9-4.el7.x86_64 libgcc-4.8.2-16.2.el7_0.x86_64 >>>>>> libicu-50.1.2-11.el7.x86_64 libselinux-2.2.2-6.el7.x86_64 >>>>>> libstdc++-4.8.2-16.2.el7_0.x86_64 >>>>>> nss-softokn-freebl-3.16.2.3-1.el7_0.x86_64 >>>>>> openssl-libs-1.0.1e-34.el7_0.7.x86_64 pcre-8.32-12.el7.x86_64 >>>>>> xz-libs-5.1.2-9alpha.el7.x86_64 zlib-1.2.7-13.el7.x86_64 (gdb) >>>>>> where #0 >>>>>> 0x00007fdb86a9fc10 in pthread_mutex_lock () from >>>>>> /lib64/libpthread.so.0 >>>>>> #1 0x000000000041ed54 in XrdSysMutex::Lock (this=0x10) at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/./XrdSys/XrdSysPthread.hh:149 #2 0x00007fdb86ceb2e6 in >>>>>> XrdSysLogger::Put (this=0x0, iovcnt=2, iov=0x7fff589f1800) at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/XrdSys/XrdSysLogger.cc:270 #3 0x00007fdb86cea817 in >>>>>> XrdSysError::Say (this=0x7fdb7f3aa7e0 <XrdSsi::Log>, txt1=0x1f1efb8 >>>>>> "0312 20:09:50.451 [0x7fdb87464740] INFO QservOss >>>>>> (build/xrdoss/QservOss.cc:260) - QservOss Init", txt2=0x0, >>>>>> txt3=0x0, txt4=0x0, txt5=0x0, txt6=txt6@entry=0x0) at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/XrdSys/XrdSysError.cc:163 #4 0x00007fdb7f19e638 in >>>>>> XrdSsiLogger::Msg (pfx=<optimized out>, txt1=<optimized out>, >>>>>> txt2=<optimized out>, txt3=<optimized out>) >>>>>> at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-4.0.0rc3- >>>> qsClient2.lsst1/xrootd- >>>>>> 4.0.0rc3-qsClient2.lsst1/src/XrdSsi/XrdSsiLogger.cc:66 #5 >>>>>> 0x00007fdb84f38a0d in lsst::qserv::wlog::XrootdAppender::append >>>>>> (this=0x1f2b9e0, event=..., p=...) at > build/wlog/XrootdAppender.cc:57 >>>> #6 >>>>>> 0x00007fdb841ebd1e in log4cxx::AppenderSkeleton::doAppend >>>>>> (this=0x1f2b9e0, event=..., pool1=...) at appenderskeleton.cpp:142 >>>>>> #7 0x00007fdb841e9df4 in >>>>>> log4cxx::helpers::AppenderAttachableImpl::appendLoopOnAppenders >>>>>> (this=0x1f2bd20, event=..., p=...) at appenderattachableimpl.cpp:69 >>>>>> #8 0x00007fdb8422f6d5 in log4cxx::Logger::callAppenders >>>>>> (this=this@entry=0x1f2bd90, event=..., p=...) at logger.cpp:93 #9 >>>>>> 0x00007fdb8422fa1f in log4cxx::Logger::forcedLog (this=0x1f2bd90, >>>>>> level1=..., message="QservOss Init", location=...) at >>>>>> logger.cpp:124 #10 0x00007fdb84500412 in lsst::log::Log::vlog >>>>>> (logger=..., level=..., filename="build/xrdoss/QservOss.cc", >>>>>> funcname="virtual int >>>>>> lsst::qserv::xrdoss::QservOss::Init(XrdSysLogger*, const char*)", >>>>>> lineno=lineno@entry=260, fmt=fmt@entry=0x7fdb84f5a7bb "QservOss >>>>>> Init", args=args@entry=0x7fff589f2018) at src/Log.cc:357 #11 >>>>>> 0x00007fdb84500734 in lsst::log::Log::log (logger=..., level=..., >>>>>> filename="build/xrdoss/QservOss.cc", funcname="virtual int >>>>>> lsst::qserv::xrdoss::QservOss::Init(XrdSysLogger*, const char*)", >>>>>> lineno=260, >>>>>> fmt=0x7fdb84f5a7bb "QservOss Init") at src/Log.cc:386 >>>>>> #12 0x00007fdb84f4e4ac in lsst::qserv::xrdoss::QservOss::Init >>>>>> (this=0x1f1fab0, log=0x0, cfgFn=0x0) at build/xrdoss/QservOss.cc:260 >>>>>> #13 0x00007fdb84f4d4f7 in lsst::qserv::xrdoss::QservOss::QservOss >>>>>> รง(this=0x1f1fab0) at build/xrdoss/QservOss.cc:135 #14 >>>>>> 0x00007fdb84f4d2f4 in lsst::qserv::xrdoss::QservOss::getInstance >>>>>> () at >>>>>> build/xrdoss/QservOss.cc:101 #15 0x00007fdb84f4e8f8 in >>>>>> XrdOssGetStorageSystem (native_oss=0x7fdb87253100 >>>>>> <XrdOssGetSS(XrdSysLogger*, char const*, char const*, char const*, >>>>>> XrdOucEnv*, XrdVersionInfo&)::myOssSys>, >>>>>> Logger=0x65b3b8 <XrdMain::Config+248>, config_fn=0x1efec40 >>>>>> "/qserv/qserv-run/etc/lsp.cf", parms=0x0) at >>>>>> build/xrdoss/QservOss.cc:293 #16 0x00007fdb86ff484d in XrdOssGetSS >>>>>> (Logger=0x65b3b8 <XrdMain::Config+248>, config_fn=0x1efec40 >>>>>> "/qserv/qserv-run/etc/lsp.cf", OssLib=0x1efef70 "libxrdoss.so", >>>>>> OssParms=0x0, envP=0x0, urVer=...) >>>>>> at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/XrdOss/XrdOssApi.cc:135 #17 0x000000000042f08d in >>>>>> XrdCmsConfig::ConfigOSS (this=0x65d340 <XrdCms::Config>) at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/XrdCms/XrdCmsConfig.cc:777 #18 0x000000000042dbc8 in >>>>>> XrdCmsConfig::Configure2 (this=0x65d340 <XrdCms::Config>) at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/XrdCms/XrdCmsConfig.cc:408 #19 0x0000000000443754 in >>>>>> XrdgetProtocol (pname=0x1eef920 "cmsd", parms=0x0, pi=0x65b2c0 >>>>>> <XrdMain::Config>) >>>>>> at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/XrdCms/XrdCmsProtocol.cc:121 #20 0x000000000041de24 in >>>>>> XrdProtLoad::getProtocol (lname=0x0, pname=0x1eef920 "cmsd", >>>>>> parms=0x0, >>>>>> pi=0x65b2c0 <XrdMain::Config>) >>>>>> at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/Xrd/XrdProtLoad.cc:247 #21 0x000000000041d8b4 in >>>>>> XrdProtLoad::Load (lname=0x0, pname=0x1eef920 "cmsd", parms=0x0, >>>>>> pi=0x65b2c0 <XrdMain::Config>) >>>>>> at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/Xrd/XrdProtLoad.cc:101 #22 0x000000000041a5b7 in >>>>>> XrdConfig::Setup (this=0x65b2c0 <XrdMain::Config>, >>>>>> dfltp=0x7fff589f3d4e >>>>>> "cmsd") at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/Xrd/XrdConfig.cc:1028 #23 0x00000000004187de in >>>>>> XrdConfig::Configure (this=0x65b2c0 <XrdMain::Config>, argc=9, >>>>>> argv=0x7fff589f3318) at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/Xrd/XrdConfig.cc:509 #24 0x000000000041f13b in main >>>>>> (argc=9, >>>>>> argv=0x7fff589f3318) at >>>>>> /qserv/stack/EupsBuildDir/Linux64/xrootd-xssi-1.0.0/xrootd-xssi- >>>>>> 1.0.0/src/Xrd/XrdMain.cc:179 >>>>> >>>>> >>>>> >>> >>> >>> > > ######################################################################## > Use REPLY-ALL to reply to list > > To unsubscribe from the QSERV-L list, click the following link: > https://listserv.slac.stanford.edu/cgi-bin/wa?SUBED1=QSERV-L&A=1 ######################################################################## Use REPLY-ALL to reply to list To unsubscribe from the QSERV-L list, click the following link: https://listserv.slac.stanford.edu/cgi-bin/wa?SUBED1=QSERV-L&A=1