Author: [log in to unmask]
Date: Thu Mar 5 18:41:01 2015
New Revision: 2282
Log:
putting auger scripts in svn.
Added:
scripts/scripts/
scripts/scripts/recon/
scripts/scripts/recon/2014/
scripts/scripts/recon/2014/scripts/
scripts/scripts/recon/2014/scripts/getfilenumbers.py (with props)
scripts/scripts/recon/2014/scripts/old/
scripts/scripts/recon/2014/scripts/old/getfilenumbers.sh (with props)
scripts/scripts/recon/2014/scripts/runjob.py (with props)
scripts/scripts/recon/2014/scripts/runjobs.py (with props)
scripts/scripts/recon/2014/templates/
scripts/scripts/recon/2014/templates/allEcalOnly.xml
scripts/scripts/recon/2014/templates/dqmEcalOnly.xml
scripts/scripts/recon/2014/templates/dstEcalOnly.xml
scripts/scripts/recon/2014/templates/reconEcalOnly.xml
scripts/scripts/recon/2014/templates/reconTruncEcalOnly.xml
Added: scripts/scripts/recon/2014/scripts/getfilenumbers.py
=============================================================================
--- scripts/scripts/recon/2014/scripts/getfilenumbers.py (added)
+++ scripts/scripts/recon/2014/scripts/getfilenumbers.py Thu Mar 5 18:41:01 2015
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+import sys,os
+if len(sys.argv) < 3:
+ print sys.argv[0]+' [options] <runtype> <runnumber>'
+ print 'gets list of files (just the labels) from a run number '
+ print '<runtype> can be recon, dqm, or dst'
+ sys.exit()
+
+# mandatory arguments:
+runtype=sys.argv[len(sys.argv)-2]
+run=int(sys.argv[len(sys.argv)-1])
+
+mssdir='/mss/hallb/hps/'
+wrkdir='/work/hallb/hps'
+voldir='/volatile/hallb/hps'
+
+label='R3.2'
+passN='pass1'
+indir=mssdir+'/data'
+outdir=mssdir+'/engrun/'+passN
+
+# below are set for 'recon'
+postfixout='slcio'
+postfixin='evio'
+midin=''
+midout='_recon_'+label
+
+if runtype == 'recon':
+ outdir=outdir+'/recon'
+elif runtype == 'dst':
+ indir=outdir+'/recon'
+ outdir=outdir+'/dst'
+ postfixin='slcio'
+ postfixout='root'
+ midin='_recon_'+label
+ midout='_dst_'+label
+elif runtype == 'dqm':
+ indir=outdir+'/recon'
+ outdir=wrkdir+'/data/data_quality/'+passN+'/dqm'
+ postfixout='root'
+ postfixin = 'slcio'
+ midin='_recon_'+label
+ midout='_dqm_'+label
+else:
+ sys.exit('Invalid Run Type: '+runtype)
+
+filnos=''
+for filno in range(0,100):
+ outfile='%s/hps_%.6d.%s%s.%s'%(outdir,run,filno,midout,postfixout)
+ if postfixin == 'evio':
+ # the evio has funny label format...file extension not at end
+ infile='%s/hps_%.6d.%s.%s'%(indir,run,postfixin,filno)
+ else:
+ infile='%s/hps_%.6d.%s%s.%s'%(indir,run,filno,midin,postfixin)
+
+ if os.path.isfile(infile) and not os.path.isfile(outfile):
+ filnos += str(filno)+' '
+
+if filnos=='':
+ print 'foobar'
+else:
+ print filnos
+
Added: scripts/scripts/recon/2014/scripts/old/getfilenumbers.sh
=============================================================================
--- scripts/scripts/recon/2014/scripts/old/getfilenumbers.sh (added)
+++ scripts/scripts/recon/2014/scripts/old/getfilenumbers.sh Thu Mar 5 18:41:01 2015
@@ -0,0 +1,67 @@
+#!/bin/tcsh
+if ( $#argv != 2) then
+ echo "$0 <runtype> <runnumber>"
+ echo "gets list of files (just the labels) from a run number "
+ echo "<runtype> can be eviotolcio, recon, dqm, or dst"
+ exit
+endif
+set prefix="hps_00"
+set label="R3.2"
+set pass="pass1"
+set indir="/mss/hallb/hps/data"
+set outdir="/mss/hallb/hps/engrun/${pass}"
+# below are set for "recon"
+set postfixout="slcio"
+set postfixin="evio"
+set midin=""
+set midout="_recon_${label}"
+set runtype=$1
+set run=$2
+
+if ($1 =~ "eviotolcio") then
+ set outdir=${outdir}
+ set midout=""
+endif
+
+if ($1 =~ "recon") then
+ set outdir=${outdir}/recon
+endif
+
+if ($1 =~ "dst") then
+ set indir=${outdir}/recon
+ set outdir=${outdir}/dst
+ set postfixin="slcio"
+ set postfixout="root"
+ set midin="_recon_${label}"
+ set midout="_dst_${label}"
+endif
+
+if (($1 =~ "dqm")) then
+ set indir=$outdir/recon
+ set outdir="/home/hps/hps-work/data/data_quality/${pass}/recon"
+ set postfixout="root"
+ set postfixin = "slcio"
+ set midin="_recon_${label}"
+ set midout="_dqm_${label}"
+endif
+
+
+set n=0
+set cnt=0
+set max=100
+while ($n<$max)
+ set infileName=$indir/${prefix}${run}.${n}${midin}.${postfixin}
+ if ($postfixin =~ "evio") then # the evio has funny label format...file extension not at end
+ set infileName=$indir/${prefix}${run}.${postfixin}.${n}
+ endif
+ set outfileName=$outdir/${prefix}${run}.${n}${midout}.${postfixout}
+ if (! (-f $outfileName) && (-f $infileName)) then
+ printf "${n} "
+ set cnt=`expr $cnt + 1`
+ endif
+ set n=`expr $n + 1`
+end
+
+if ($cnt == 0) then
+ printf "foobar"
+endif
Added: scripts/scripts/recon/2014/scripts/runjob.py
=============================================================================
--- scripts/scripts/recon/2014/scripts/runjob.py (added)
+++ scripts/scripts/recon/2014/scripts/runjob.py Thu Mar 5 18:41:01 2015
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+import shutil,sys,os
+from subprocess import Popen, PIPE
+
+if len(sys.argv) < 4:
+ print 'runjob.py [options] <xml template> <runtype> <run number>'
+ print '<runtype> can be recon, dqm, dst'
+ print '[options]:'
+ print '\t -s = submit on the fly (else just create all xml submission scripts)'
+ print '\t -d = debug (never submit, the more d\'s the more debug info)'
+ sys.exit()
+
+# mandatory arguments:
+xmltemplate=sys.argv[len(sys.argv)-3]
+runtype=sys.argv[len(sys.argv)-2]
+run=sys.argv[len(sys.argv)-1]
+
+if not os.path.isfile(xmltemplate):
+ sys.exit('Template DNE: '+xmltemplate)
+
+# interpret options:
+submit=False
+debug=0
+for xx in sys.argv[1:len(sys.argv)-2]:
+ if xx=='-s':
+ submit=True
+ elif xx=='-d':
+ debug += 1
+
+runtype2=runtype.replace('Trunc','')
+
+# get the missing jobs
+cmd ='./scripts/getfilenumbers.py ' +runtype2 +' '+str(run)
+p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
+nums = p.stdout.readline().rstrip()
+sys.stdout.flush()
+if nums == 'foobar':
+ if debug>5:
+ print 'Nothing to run: '+run
+ sys.stdout.flush()
+ sys.exit()
+
+# choose output xml file name:
+if submit:
+ xmlfile = 'tmp.xml'
+else:
+ odir='jsubs/'+runtype
+ xmlfile = '%s/%s.xml'%(odir,run)
+ if not os.path.isdir(odir):
+ os.mkdirs(odir)
+
+# parse the xml template
+with open(xmltemplate,'r') as file:
+ lines = file.readlines()
+with open(xmlfile,'w') as file:
+ for line in lines:
+ line=line.replace('__FILENUM__',str(nums))
+ line=line.replace('__RUN__',str(run))
+ file.write(line)
+
+# submit the job to the batch farm:
+if submit:
+ if debug:
+ if debug>1:
+ for xx in nums.rstrip().split():
+ print '/mss/hallb/hps/data/hps_%.6d.evio.%d'%(int(run),int(xx))
+ else:
+ print 'Would have submitted '+run+' '+nums
+
+ else:
+ os.system('jsub -xml '+xmlfile)
+
+
Added: scripts/scripts/recon/2014/scripts/runjobs.py
=============================================================================
--- scripts/scripts/recon/2014/scripts/runjobs.py (added)
+++ scripts/scripts/recon/2014/scripts/runjobs.py Thu Mar 5 18:41:01 2015
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+import os,sys,subprocess
+
+firstRun = 3183
+finalRun = 3461
+
+if len(sys.argv) < 2:
+ print 'runjobs.py [options] <runtype>'
+ print '<runtype> can be recon, dqm, dst'
+ print '[options]:'
+ print '\t -s = submit on the fly (else just create all xml submission scripts)'
+ print '\t -d = debug (never submit, the more d\'s the more debug info)'
+ sys.exit()
+
+# mandatory arguments:
+runtype = sys.argv[len(sys.argv)-1]
+
+# pass these to the next script:
+opts = ' '.join(sys.argv[1:len(sys.argv)-1])
+
+# choose xml template based on runtype:
+xmltemplate = 'templates/%sEcalOnly.xml'%(runtype)
+if not os.path.isfile(xmltemplate):
+ sys.exit('XML Template DNE: '+xmltemplate)
+
+for runNumber in range(firstRun,finalRun+1):
+ cmd = './scripts/runjob.py '+opts+' '+xmltemplate +' '+runtype+' '+str(runNumber)
+ subprocess.call(cmd,shell=True)
+
Added: scripts/scripts/recon/2014/templates/allEcalOnly.xml
=============================================================================
--- scripts/scripts/recon/2014/templates/allEcalOnly.xml (added)
+++ scripts/scripts/recon/2014/templates/allEcalOnly.xml Thu Mar 5 18:41:01 2015
@@ -0,0 +1,39 @@
+<!--Run with runjob_XXXto1.sh-->
+<Request>
+ <List name="filenum">666</List>
+ <Name name="recon_RUNNO"/>
+ <Project name="hps"/>
+ <Track name="reconstruction"/>
+ <Email email="[log in to unmask]" request="false" job="false" />
+ <Memory space="4000" unit="MB"/>
+ <DiskSpace space="5" unit="GB"/>
+ <OS name="centos65"/>
+ <Variable name="run" value="RUNNO"/>
+ <Variable name="pass" value="pass1"/>
+ <Variable name="reltag" value="R3.2"/>
+ <Variable name="out_file" value="hps_00"/>
+ <Variable name="detector" value="HPS-ECalCommissioning-v2"/>
+ <Variable name="hps-java" value="/u/group/hps/hps_soft/hps-java/hps-distribution-3.2-SNAPSHOT-bin.jar"/>
+ <Variable name="log_dir" value="file:/work/hallb/hps/data/logs"/>
+ <Variable name="out_dir" value="mss:/mss/hallb/hps/engrun"/>
+ <Variable name="dq_dir" value="file:/work/hallb/hps/data/data_quality"/>
+ <Variable name="steering" value="/org/hps/steering/recon/EngineeringRun2014ECalRecon_Pass1.lcsim"/>
+ <ForEach list="filenum">
+ <Job>
+ <Input src="mss:/mss/hallb/hps/data/hps_00${run}.evio.${filenum}" dest="in.evio"/>
+ <Command><![CDATA[
+ echo "Auger ID: $AUGER_ID"
+ /apps/scicomp/java/jdk1.7/bin/java -cp ${hps-java} org.hps.evio.EvioToLcio -x ${steering} -r -d ${detector} -R ${run} -DoutputFile=out in.evio
+ /apps/scicomp/java/jdk1.7/bin/java -XX:+UseSerialGC -Xmx100m -jar ${hps-java} -r /org/hps/steering/production/DataQuality.lcsim -i out.slcio > data_quality.txt
+ /apps/scicomp/java/jdk1.7/bin/java -jar ${hps-java} -r ${steering} -DoutputFile=dqm.out -i out.slcio
+ /u/group/hps/hps_soft/hps-dst/build/bin/dst_maker -g -b0.5 -o dst.root out.slcio -e
+ ]]></Command>
+ <Output src="out.slcio" dest="${out_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.slcio"/>
+ <Output src="data_quality.txt" dest="${dq_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.txt"/>
+ <Output src="out.root" dest="${dq_dir}/${pass}/dqm/${out_file}${run}.${filenum}_dqm_${reltag}.root"/>
+ <Output src="out.root" dest="${out_dir}/${pass}/dst/${out_file}${run}.${filenum}_dst_${reltag}.root"/>
+ <Stderr dest="${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.err" />
+ <Stdout dest="${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.out" />
+ </Job>
+ </ForEach>
+</Request>
Added: scripts/scripts/recon/2014/templates/dqmEcalOnly.xml
=============================================================================
--- scripts/scripts/recon/2014/templates/dqmEcalOnly.xml (added)
+++ scripts/scripts/recon/2014/templates/dqmEcalOnly.xml Thu Mar 5 18:41:01 2015
@@ -0,0 +1,30 @@
+<Request>
+ <List name="filenum">__FILENUM__</List>
+ <Name name="dqm__RUN__"/>
+ <Project name="hps"/>
+ <Track name="reconstruction"/>
+ <Email email="[log in to unmask]" request="false" job="false" />
+ <Memory space="2000" unit="MB"/>
+ <DiskSpace space="5" unit="GB"/>
+ <OS name="centos65"/>
+ <Variable name="run" value="__RUN__"/>
+ <Variable name="reltag" value="R3.2" />
+ <Variable name="pass" value="pass1" />
+ <Variable name="hps-java" value="/u/group/hps/hps_soft/hps-java/hps-distribution-3.2-SNAPSHOT-bin.jar"/>
+ <Variable name="log_dir" value="file:/work/hallb/hps/data/logs"/>
+ <Variable name="dq_dir" value="file:/work/hallb/hps/data/data_quality"/>
+ <Variable name="out_file" value="hps_00"/>
+ <Variable name="steering" value="/org/hps/steering/production/DataQualityEcalOnlyEngineeringRun.lcsim"/>
+ <ForEach list="filenum">
+ <Job>
+ <Input src="mss:/mss/hallb/hps/engrun/${pass}/recon/hps_00${run}.${filenum}_recon_${reltag}.slcio" dest="in.slcio"/>
+ <Command><![CDATA[
+ echo "Auger ID: $AUGER_ID"
+ /apps/scicomp/java/jdk1.7/bin/java -jar ${hps-java} -r ${steering} -DoutputFile=out -i in.slcio
+ ]]></Command>
+ <Output src="out.root" dest="${dq_dir}/${pass}/dqm/${out_file}${run}.${filenum}_dqm_${reltag}.root"/>
+ <Stderr dest="${log_dir}/${pass}/dqm/${out_file}${run}.${filenum}_dqm_${reltag}.err" />
+ <Stdout dest="${log_dir}/${pass}/dqm/${out_file}${run}.${filenum}_dqm_${reltag}.out" />
+ </Job>
+ </ForEach>
+</Request>
Added: scripts/scripts/recon/2014/templates/dstEcalOnly.xml
=============================================================================
--- scripts/scripts/recon/2014/templates/dstEcalOnly.xml (added)
+++ scripts/scripts/recon/2014/templates/dstEcalOnly.xml Thu Mar 5 18:41:01 2015
@@ -0,0 +1,29 @@
+<Request>
+ <List name="filenum">__FILENUM__</List>
+ <Name name="dst__RUN__"/>
+ <Project name="hps"/>
+ <Track name="reconstruction"/>
+ <Email email="[log in to unmask]" request="false" job="false" />
+ <Memory space="2048" unit="MB"/>
+ <DiskSpace space="5" unit="GB"/>
+ <OS name="centos65"/>
+ <Variable name="pass" value="pass1"/>
+ <Variable name="run" value="__RUN__"/>
+ <Variable name="reltag" value="R3.2"/>
+ <Variable name="log_dir" value="file:/work/hallb/hps/data/logs"/>
+ <Variable name="out_dir" value="mss:/mss/hallb/hps/engrun"/>
+ <Variable name="out_file" value="hps_00"/>
+ <ForEach list="filenum">
+ <Job>
+ <Input src="mss:/mss/hallb/hps/engrun/${pass}/recon/hps_00${run}.${filenum}_recon_${reltag}.slcio" dest="in.slcio"/>
+ <Command><![CDATA[
+ echo "Auger ID: $AUGER_ID"
+ /u/group/hps/hps_soft/hps-dst/build/bin/dst_maker -g -b0.5 -o out.root in.slcio -e
+ ls -lh
+ ]]></Command>
+ <Output src="out.root" dest="${out_dir}/${pass}/dst/${out_file}${run}.${filenum}_dst_${reltag}.root"/>
+ <Stderr dest="${log_dir}/${pass}/dst/${out_file}${run}.${filenum}_dst_${reltag}.err" />
+ <Stdout dest="${log_dir}/${pass}/dst/${out_file}${run}.${filenum}_dst_${reltag}.out" />
+ </Job>
+ </ForEach>
+</Request>
Added: scripts/scripts/recon/2014/templates/reconEcalOnly.xml
=============================================================================
--- scripts/scripts/recon/2014/templates/reconEcalOnly.xml (added)
+++ scripts/scripts/recon/2014/templates/reconEcalOnly.xml Thu Mar 5 18:41:01 2015
@@ -0,0 +1,37 @@
+<Request>
+ <List name="filenum">__FILENUM__</List>
+ <Name name="recon__RUN__"/>
+ <Project name="hps"/>
+ <Track name="reconstruction"/>
+ <Email email="[log in to unmask]" request="false" job="false" />
+ <Memory space="4000" unit="MB"/>
+ <DiskSpace space="5" unit="GB"/>
+ <OS name="centos65"/>
+ <Variable name="run" value="__RUN__"/>
+ <Variable name="pass" value="pass1"/>
+ <Variable name="reltag" value="R3.2"/>
+ <Variable name="out_file" value="hps_00"/>
+ <Variable name="detector" value="HPS-ECalCommissioning-v2"/>
+ <Variable name="hps-java" value="/u/group/hps/hps_soft/hps-java/hps-distribution-3.2-SNAPSHOT-bin.jar"/>
+ <Variable name="log_dir" value="file:/work/hallb/hps/data/logs"/>
+ <Variable name="out_dir" value="mss:/mss/hallb/hps/engrun"/>
+ <Variable name="dq_dir" value="file:/work/hallb/hps/data/data_quality"/>
+ <Variable name="steering" value="/org/hps/steering/recon/EngineeringRun2014ECalRecon_Pass1.lcsim"/>
+ <ForEach list="filenum">
+ <Job>
+ <Input src="mss:/mss/hallb/hps/data/hps_00${run}.evio.${filenum}" dest="in.evio"/>
+ <Command><![CDATA[
+ echo "Auger ID: $AUGER_ID"
+ /apps/scicomp/java/jdk1.7/bin/java -cp ${hps-java} org.hps.evio.EvioToLcio -x ${steering} -r -d ${detector} -R ${run} -DoutputFile=out in.evio
+ /apps/scicomp/java/jdk1.7/bin/java -XX:+UseSerialGC -Xmx100m -jar ${hps-java} -r /org/hps/steering/production/DataQuality.lcsim -i out.slcio > data_quality.txt
+ rm -f ${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.err
+ rm -f ${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.out
+ rm -f ${dq_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.txt
+ ]]></Command>
+ <Output src="out.slcio" dest="${out_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.slcio"/>
+ <Output src="data_quality.txt" dest="${dq_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.txt"/>
+ <Stderr dest="${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.err" />
+ <Stdout dest="${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.out" />
+ </Job>
+ </ForEach>
+</Request>
Added: scripts/scripts/recon/2014/templates/reconTruncEcalOnly.xml
=============================================================================
--- scripts/scripts/recon/2014/templates/reconTruncEcalOnly.xml (added)
+++ scripts/scripts/recon/2014/templates/reconTruncEcalOnly.xml Thu Mar 5 18:41:01 2015
@@ -0,0 +1,39 @@
+<Request>
+ <List name="filenum">__FILENUM__</List>
+ <Name name="recon__RUN__"/>
+ <Project name="hps"/>
+ <Track name="reconstruction"/>
+ <Email email="[log in to unmask]" request="false" job="false" />
+ <Memory space="4000" unit="MB"/>
+ <DiskSpace space="5" unit="GB"/>
+ <OS name="centos65"/>
+ <Variable name="run" value="__RUN__"/>
+ <Variable name="pass" value="pass1"/>
+ <Variable name="reltag" value="R3.2"/>
+ <Variable name="out_file" value="hps_00"/>
+ <Variable name="eviocopy" value="/u/group/hps/hps_soft/evio/bin/eviocopy"/>
+ <Variable name="detector" value="HPS-ECalCommissioning-v2"/>
+ <Variable name="hps-java" value="/u/group/hps/hps_soft/hps-java/hps-distribution-3.2-SNAPSHOT-bin.jar-DBFIX"/>
+ <Variable name="log_dir" value="file:/work/hallb/hps/data/logs"/>
+ <Variable name="out_dir" value="mss:/mss/hallb/hps/engrun"/>
+ <Variable name="dq_dir" value="file:/work/hallb/hps/data/data_quality"/>
+ <Variable name="steering" value="/org/hps/steering/recon/EngineeringRun2014ECalRecon_Pass1.lcsim"/>
+ <ForEach list="filenum">
+ <Job>
+ <Input src="mss:/mss/hallb/hps/data/hps_00${run}.evio.${filenum}" dest="in.evio"/>
+ <Command><![CDATA[
+ echo "Auger ID: $AUGER_ID"
+ ${eviocopy} in.evio out.evio ; mv -f out.evio in.evio
+ /apps/scicomp/java/jdk1.7/bin/java -cp ${hps-java} org.hps.evio.EvioToLcio -x ${steering} -r -d ${detector} -R ${run} -DoutputFile=out in.evio
+ /apps/scicomp/java/jdk1.7/bin/java -XX:+UseSerialGC -Xmx100m -jar ${hps-java} -r /org/hps/steering/production/DataQuality.lcsim -i out.slcio > data_quality.txt
+ rm -f ${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.err
+ rm -f ${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.out
+ rm -f ${dq_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.txt
+ ]]></Command>
+ <Output src="out.slcio" dest="${out_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.slcio"/>
+ <Output src="data_quality.txt" dest="${dq_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.txt"/>
+ <Stderr dest="${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.err" />
+ <Stdout dest="${log_dir}/${pass}/recon/${out_file}${run}.${filenum}_recon_${reltag}.out" />
+ </Job>
+ </ForEach>
+</Request>
