Author: [log in to unmask]
Date: Thu Feb 19 13:13:55 2015
New Revision: 2167

Log:
update for dqm, future run, slac locations

Modified:
    java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_metadata.py
    java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_register.py
    java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_util.py

Modified: java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_metadata.py
 =============================================================================
--- java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_metadata.py	(original)
+++ java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_metadata.py	Thu Feb 19 13:13:55 2015
@@ -64,7 +64,7 @@
     if isADC==1:
 
       # ignore SLOT lines:
-      if re.search('SLOT',key)!=None:
+      if key.find('SLOT')>=0:
         continue
 
       # may have SVT FADC later, so specify:
@@ -78,7 +78,7 @@
         mtd['SSP_HPS_'+__IOSRC[xx[1]]]=1
 
     # special case #2, interpret threshold:
-    elif isADC and re.search('ALLCH_PED',key)!=None:
+    elif isADC and key.find('ALLCH_PED')>=0:
 
       if prevthresh=='0':
         mtd['ECALFADC_THRESH']=0
@@ -86,7 +86,7 @@
         mtd['ECALFADC_THRESH']=int('%.0f'%(float(prevthresh)-float(xx[0])+1))
 
     # special case #3, save threshold:
-    elif isADC and re.search('ALLCH_TET',key)!=None:
+    elif isADC and key.find('ALLCH_TET')>=0:
       prevthresh=xx[0]
 
     # 2-column lines, only one possibility:

Modified: java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_register.py
 =============================================================================
--- java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_register.py	(original)
+++ java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_register.py	Thu Feb 19 13:13:55 2015
@@ -2,17 +2,15 @@
 import re,os,sys
 import engrun_util as ERU
 import engrun_metadata as ERM
+
+ERU.SetRunPeriod("engrun2014")
 
 USAGE='Usage:  engrun_register.py filepath [outputfile]'
 
 DEBUG=0
 #DEBUG=1
 
-# no sshing, just print commands:
 SSHREG=0
-
-# one ssh for each file to register:
-# (do this for automating)
 #SSHREG=1
 
 if len(sys.argv)!=3 and len(sys.argv)!=2:
@@ -28,7 +26,6 @@
   else:
     OUTFILE=open(OUTFILE, 'w')
 
-
 # DATA CATALOG PATH MUST ALREADY EXIST (MAKE IT MANUALLY)
 DCPATH=ERU.GetDCPath(FILEPATH)
 
@@ -36,7 +33,7 @@
 
 # THIS WILL NEED CHANGING TO RUN LOCALLY AT SLAC
 # WE CAN CHOOSE WHETHER TO SSH BASED ON FILEPATH, or HOSTNAME (see Jeremy's util.py)
-DCLISTCMD=ERU.__SSH+' '+ERU.__DATACAT+' find '+DCOPTS+' '+ERU.__DCSRCHOPTS+' '+DCPATH
+DCLISTCMD=ERU.__SSH+' '+ERU.__DATACAT+' find '+DCOPTS+' '+ERU.__DCSRCHOPTS+' '+'/HPS'#DCPATH
 
 DCREGCMD=ERU.__DATACAT+' registerDataset '+DCOPTS
 
@@ -46,6 +43,7 @@
 # Get list of files to register:
 FILELIST=ERU.ListRealFiles(FILEPATH)
 
+NFILES2CATALOG=0;
 
 if len(FILELIST)==0:
   sys.exit('No Files:  '+FILEPATH)
@@ -74,6 +72,8 @@
     if DEBUG>1:
       sys.stderr.write('Invalid File Number for '+filename+'\n')
     continue
+
+  NFILES2CATALOG += 1
 
   metadata={}
   metadata['Run']=runno
@@ -149,6 +149,9 @@
     print >> OUTFILE, cmd
 
 
+if NFILES2CATALOG==0:
+  print 'All Files Already Cataloged'
+
 if DEBUG!=0:
   ERU.DumpTable()
   ERU.DumpNames()

Modified: java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_util.py
 =============================================================================
--- java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_util.py	(original)
+++ java/trunk/datacat/src/main/python/hpsdatacat/engrun/engrun_util.py	Thu Feb 19 13:13:55 2015
@@ -6,10 +6,7 @@
 
 __SSH='ssh [log in to unmask]
 __DATACAT='~srs/datacat/prod/datacat-hps'
-__DCSRCHOPTS='--search-groups --show-non-ok-locations'
-
-__RUNMIN=2000
-__RUNMAX=4000
+__DCSRCHOPTS='--recurse --search-groups --show-non-ok-locations'
 
 __ALLRUNS=[]
 __ALLMETADATA=[]
@@ -17,6 +14,23 @@
 
 __RAWFILENAMEFORMAT='hps_%.6d.evio.%d'
 
+__HPSRUNPERIOD=None
+
+__HPSRUNPERIODS={'simulation':[-9999,100],
+                 'testrun2012':[100,2000],
+                 'engrun2014':[2000,4000],
+                 'engrun2015':[4000,9999]}
+
+def SetRunPeriod(runperiod):
+  global __HPSRUNPERIOD
+  if not __HPSRUNPERIODS.has_key(runperiod):
+    sys.exit('SetRunPeriod:  Missing run period:  '+runperiod)
+  __HPSRUNPERIOD=''.join(runperiod)
+
+#
+# For Reconstructed Files on Tape/Disk, path must match /pass\d+/.*
+# Everything before that will be ignored.
+#
 
 ##################################
 # CATALOG STRUCTURE (terminal directories are "GROUPS")
@@ -42,7 +56,7 @@
   return runlist
 
 
-# THIS NEEDS FIXING FOR OTHER TYPES (e.g. DQM):
+# Data Catalog Groups (RAW,RECON,DQM,DST,...):
 def GetGroup(filename):
   dt=GetDataType(filename)
   if dt==None:
@@ -51,50 +65,80 @@
     return 'RAW'
   elif dt[1]=='SLCIO':
     return 'RECON'
-  elif dt[1]=='TEST':
-    return 'DST'
+  elif filename.lower().endswith('.root'):
+    if filename.lower().find('dqm')>=0:
+      return 'DQM'
+    elif filename.lower().find('dst')>=0:
+      return 'DST'
   else:
     sys.exit('GetGroup:  Not Ready for this DataType:  '+filename)
 
-# THESE WILL HAVE TO CHANGE FOR THE NEXT RUN
-# SHOULD DO IT AUTOMATICALLY BASED ON RUN NUMBER
-def GetDCRunPeriod(filepath):
-#  runno=GetRunNumber(filepath)
-  return 'engrun2014'
-def GetMSSRunPeriod(filepath):
-  return 'engrun'
-
-# ONLY READY FOR FILES AT JLAB, UPDATE FOR SLAC:
+
+## path stub on tape at JLab:
+#def GetMSSRunPeriod(filepath):
+#  runno=GetRunNumber(filepath)[0]
+#  if runno<10:
+#    return 'simulation'
+#  if runno<2500:
+#    return 'testrun2012'
+#  elif runno<4000:
+#    return 'engrun'
+#  else:
+#    return 'engrun2015'
+
+## path stub for the Data Catalog
+#def GetDCRunPeriod(filepath):
+#  runno=GetRunNumber(filepath)[0]
+#  hpsrp=__HPSRUNPERIODS
+#  for xx in hpsrp.keys():
+#    if runno>=hpsrp[xx][0] and runno<hpsrp[xx][1]:
+#      return xx
+#  return None
+
+# full path for the Data Catalog:
 def GetDCPath(filepath):
-  if re.match(__HPSMSSPATH,filepath)==None:
-    sys.exit('GetDCPath:  Must start with '+__HPSMSSPATH+':  '+filepath)
+
+  print __HPSRUNPERIOD
+  prefix=__HPSDCPATH+'/'+__HPSRUNPERIOD #GetDCRunPeriod(filepath)
+
   if not os.path.isdir(filepath):
     sys.exit('Directory does not exist:  '+filepath)
-  filepath=filepath.lstrip(__HPSMSSPATH).lstrip('/').rstrip('/')
-  prefix=__HPSDCPATH+'/'+GetDCRunPeriod(filepath)
-  if filepath=='data':
-    return prefix
-  else:
-    subdirs=filepath.split('/')
-    subdirs.reverse()
-    if subdirs.pop().find('engrun')>=0:
-      npass=GetPass(filepath)
-      if npass==None:
-        sys.exit('GetDCPath:  Unresolved Pass:  '+filepath)
-      return prefix+'/pass%d'%(npass)
-  sys.exit('GetDCPath: Not ready for this:  '+filepath)
+
+  # it's got to be a raw EVIO file:
+  if filepath.rstrip().rstrip('/') == __HPSMSSPATH+'/data':
+    # fix name for 2014 run:
+    if prefix.endswith('engrun2014'):
+      return __HPSDCPATH+'/engrun'
+
+  # throw away everything before '/pass#/'
+  tmp=re.search('/(pass\d+)/(.*)',filepath)
+  if (tmp == None):
+    sys.exit('GetDCPath:  Not raw data on tape, Not a pass#:\n'+filepath)
+
+  passN=tmp.group(1)
+
+  # No longer used:
+  #pathstub=tmp.group(2).lstrip('/').rstrip('/')
+  #subdirs=pathstub.split('/').reverse()
+  #if subdirs.pop()=='engrun':
+  #  return prefix+'/'+passN
+  #sys.exit('GetDCPath: Not ready for this:  '+filepath)
+
+  return prefix+'/'+passN+'/'
+
+
 
 # UPDATE THIS IF WE GET NEW DATATYPES AVAILABLE IN THE CATALOG:
 # Returns [fileformat,datatype] REQUIRED by the catalog
 def GetDataType(filename):
-  if re.search('evio',os.path.basename(filename))!=None:
+  basename=os.path.basename(filename).lower()
+  if basename.find('.evio')>0:
     return ['evio','EVIO']
-  elif re.search('slcio',os.path.basename(filename))!=None:
+  elif basename.endswith('.slcio')>0 or basename.endswith('.lcio')>0:
     return ['slcio','SLCIO']
-  elif re.search('dst',os.path.basename(filename))!=None:
+  elif basename.endswith('.root'):
     return ['root','TEST']
-  else:
-    return ['unspecified','TEST']
+  return ['unspecified','TEST']
 
 def GetPass(filename):
   match=re.search('pass\d+',filename)
@@ -104,6 +148,8 @@
 
 def GetSite(filepath):
   if re.match('/mss',filepath)!=None:
+    return 'JLAB'
+  elif re.match('/work',filepath)!=None:
     return 'JLAB'
   else:
     return 'SLAC'
@@ -133,7 +179,8 @@
     if integers[ii].lstrip('0')=='':
       continue
     xx=int(integers[ii].lstrip('0'))
-    if xx>__RUNMIN and xx<__RUNMAX:
+    if xx>=__HPSRUNPERIODS[__HPSRUNPERIOD][0] and \
+       xx< __HPSRUNPERIODS[__HPSRUNPERIOD][1]:
       runno=xx
       if ii<len(integers)-1:
         if integers[ii+1].lstrip('0')=='':