Author: [log in to unmask]
Date: Thu Aug 20 12:50:12 2015
New Revision: 3380
Log:
Major overhaul of record-util module for supporting run database. HPSJAVA-575, HPSJAVA-576
Added:
java/trunk/record-util/src/main/java/org/hps/record/epics/Epics20sVariables.java
java/trunk/record-util/src/main/java/org/hps/record/epics/Epics2sVariables.java
java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsRunProcessor.java
- copied, changed from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EventCountProcessor.java
- copied, changed from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventCountProcessor.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileFilter.java
- copied, changed from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileMetaData.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileMetaDataReader.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileSequenceComparator.java
- copied, changed from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileSequenceComparator.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java
- copied, changed from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EvioLoop.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EvioLoopAdapter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryMap.java
- copied, changed from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java
java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataDao.java
java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataDaoImpl.java
java/trunk/record-util/src/main/java/org/hps/record/run/EvioFilesDao.java
java/trunk/record-util/src/main/java/org/hps/record/run/EvioFilesDaoImpl.java
java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryDao.java
java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryDaoImpl.java
java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataDao.java
java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataDaoImpl.java
java/trunk/record-util/src/main/java/org/hps/record/run/package-info.java
Removed:
java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsCsvExporter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventCountProcessor.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileSequenceComparator.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java
java/trunk/record-util/src/main/java/org/hps/record/run/AbstractRunDatabaseReader.java
java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataReader.java
java/trunk/record-util/src/main/java/org/hps/record/run/EvioFileListReader.java
java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryReader.java
java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataReader.java
Modified:
java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsData.java
java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsEvioProcessor.java
java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsGenericObject.java
java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsHeader.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EvioEventConstants.java
java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileSource.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
java/trunk/record-util/src/main/java/org/hps/record/run/RunManager.java
java/trunk/record-util/src/main/java/org/hps/record/run/RunSummary.java
java/trunk/record-util/src/main/java/org/hps/record/scalers/ScalerData.java
java/trunk/record-util/src/main/java/org/hps/record/scalers/ScalersEvioProcessor.java
Added: java/trunk/record-util/src/main/java/org/hps/record/epics/Epics20sVariables.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/epics/Epics20sVariables.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/epics/Epics20sVariables.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,54 @@
+package org.hps.record.epics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Static list of variable names contained in 20s EPICS data block from Eng Run 2015 data along
+ * with a brief description (if known).
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public final class Epics20sVariables {
+
+ /**
+ * Map with variable names and description.
+ */
+ static final Map<String, String> VARIABLES = new HashMap<String, String>();
+
+ /**
+ * Variable definitions.
+ */
+ static {
+ VARIABLES.put("beam_stop.RBV", "beam stop motor position");
+ VARIABLES.put("hps:svt_bot:motor.RBV", "SVT bottom motor position");
+ VARIABLES.put("hps:svt_top:motor.RBV", "SVT top motor position");
+ VARIABLES.put("hps:target:motor.RBV", "target motor position");
+ VARIABLES.put("hps_collimator.RBV", "collimator motor position");
+ VARIABLES.put("scalerS10b", "DWN-B beamline counter");
+ VARIABLES.put("scalerS11b", "DWN-R beamline counter");
+ VARIABLES.put("scalerS8b", "DWN-T beamline counter");
+ VARIABLES.put("scalerS9b", "DWN-L beamline counter");
+ VARIABLES.put("scaler_cS3b", "UPS-L beamline counter");
+ VARIABLES.put("scaler_cS4b", "UPS-R beamline counter");
+ VARIABLES.put("scaler_cS5b", "TAG-L beamline counter");
+ VARIABLES.put("scaler_cS6b", "TAG-T beamline counter");
+ VARIABLES.put("scaler_cS7b", "TAG-T2 beamline counter");
+ VARIABLES.put("SMRPOSB", "");
+ }
+
+ /**
+ * Get the variable map with the names and descriptions.
+ *
+ * @return the variable map
+ */
+ public static Map<String, String> getVariables() {
+ return VARIABLES;
+ }
+
+ /**
+ * Do not allow class instantiation.
+ */
+ private Epics20sVariables() {
+ }
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/epics/Epics2sVariables.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/epics/Epics2sVariables.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/epics/Epics2sVariables.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,65 @@
+package org.hps.record.epics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * List of EPICS variables and their descriptions contained in the 2s data bank from Eng Run 2015 data.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public final class Epics2sVariables {
+
+ /**
+ * Map with variable names and description.
+ */
+ static final Map<String, String> VARIABLES = new HashMap<String, String>();
+
+ /**
+ * Variable definitions.
+ */
+ static {
+ VARIABLES.put("MBSY2C_energy", "Beam energy according to Hall B BSY dipole string");
+ VARIABLES.put("PSPECIRBCK", "Pair Spectrometer Current Readback");
+ VARIABLES.put("HPS:LS450_2:FIELD", "Frascati probe field");
+ VARIABLES.put("HPS:LS450_1:FIELD", "Pair Spectrometer probe field");
+ VARIABLES.put("MTIRBCK", "Frascati Current Readback");
+ VARIABLES.put("VCG2C21 2C21", "Vacuum gauge pressure");
+ VARIABLES.put("VCG2C21A", "2C21A Vacuum gauge pressure");
+ VARIABLES.put("VCG2C24A", "2C24A Vacuum gauge pressure");
+ VARIABLES.put("VCG2H00A", "2H00 Vacuum gauge pressure");
+ VARIABLES.put("VCG2H01A", "2H01 Vacuum gauge pressure");
+ VARIABLES.put("VCG2H02A", "2H02 Vacuum gauge pressure");
+ VARIABLES.put("scaler_calc1", "Faraday cup current");
+ VARIABLES.put("scalerS12b", "HPS-Left beam halo count");
+ VARIABLES.put("scalerS13b", "HPS-Right beam halo count");
+ VARIABLES.put("scalerS14b", "HPS-Top beam halo count");
+ VARIABLES.put("scalerS15b", "HPS-SC beam halo count");
+ VARIABLES.put("hallb_IPM2C21A_XPOS", "Beam position X at 2C21");
+ VARIABLES.put("hallb_IPM2C21A_YPOS", "Beam position Y at 2C21");
+ VARIABLES.put("hallb_IPM2C21A_CUR", "Current at 2C21");
+ VARIABLES.put("hallb_IPM2C24A_XPOS", "Beam position X at 2C24");
+ VARIABLES.put("hallb_IPM2C24A_YPOS", "Beam position Y at 2C24");
+ VARIABLES.put("hallb_IPM2C24A_CUR", "Current at 2C24");
+ VARIABLES.put("hallb_IPM2H00_XPOS", "Beam position X at 2H00");
+ VARIABLES.put("hallb_IPM2H00_YPOS", "Beam position Y at 2H00");
+ VARIABLES.put("hallb_IPM2H00_CUR", "Current at 2H00");
+ VARIABLES.put("hallb_IPM2H02_YPOS", "Beam position X at 2H02");
+ VARIABLES.put("hallb_IPM2H02_XPOS", "Beam position Y at 2H02");
+ }
+
+ /**
+ * Get the variable map.
+ *
+ * @return the variable map
+ */
+ public static Map<String, String> getVariables() {
+ return VARIABLES;
+ }
+
+ /**
+ * Do not allow class instantiation.
+ */
+ private Epics2sVariables() {
+ }
+}
Modified: java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsData.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsData.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsData.java Thu Aug 20 12:50:12 2015
@@ -42,66 +42,6 @@
private static final String EPICS_VARIABLE_NAMES = "EPICS_VARIABLE_NAMES";
/**
- * This map contains the list of EPICS keys and their descriptions from the<br/>
- * <a href="https://confluence.slac.stanford.edu/display/hpsg/EVIO+Data+Format">EVIO Data Format Confluence Page</a>
- */
- private final static Map<String, String> VARIABLES = new HashMap<String, String>();
-
- /**
- * List of possible EPICS keys.
- */
- static {
- VARIABLES.put("MBSY2C_energy", "Beam energy according to Hall B BSY dipole string");
- VARIABLES.put("PSPECIRBCK", "Pair Spectrometer Current Readback");
- VARIABLES.put("HPS:LS450_2:FIELD", "Frascati probe field");
- VARIABLES.put("HPS:LS450_1:FIELD", "Pair Spectrometer probe field");
- VARIABLES.put("MTIRBCK", "Frascati Current Readback");
- VARIABLES.put("VCG2C21 2C21", "Vacuum gauge pressure");
- VARIABLES.put("VCG2C21A", "2C21A Vacuum gauge pressure");
- VARIABLES.put("VCG2C24A", "2C24A Vacuum gauge pressure");
- VARIABLES.put("VCG2H00A", "2H00 Vacuum gauge pressure");
- VARIABLES.put("VCG2H01A", "2H01 Vacuum gauge pressure");
- VARIABLES.put("VCG2H02A", "2H02 Vacuum gauge pressure");
- VARIABLES.put("scaler_calc1", "Faraday cup current");
- VARIABLES.put("scalerS12b", "HPS-Left beam halo count");
- VARIABLES.put("scalerS13b", "HPS-Right beam halo count");
- VARIABLES.put("scalerS14b", "HPS-Top beam halo count");
- VARIABLES.put("scalerS15b", "HPS-SC beam halo count");
- VARIABLES.put("hallb_IPM2C21A_XPOS", "Beam position X at 2C21");
- VARIABLES.put("hallb_IPM2C21A_YPOS", "Beam position Y at 2C21");
- VARIABLES.put("hallb_IPM2C21A_CUR", "Current at 2C21");
- VARIABLES.put("hallb_IPM2C24A_XPOS", "Beam position X at 2C24");
- VARIABLES.put("hallb_IPM2C24A_YPOS", "Beam position Y at 2C24");
- VARIABLES.put("hallb_IPM2C24A_CUR", "Current at 2C24");
- VARIABLES.put("hallb_IPM2H00_XPOS", "Beam position X at 2H00");
- VARIABLES.put("hallb_IPM2H00_YPOS", "Beam position Y at 2H00");
- VARIABLES.put("hallb_IPM2H00_CUR", "Current at 2H00");
- VARIABLES.put("hallb_IPM2H02_YPOS", "Beam position X at 2H02");
- VARIABLES.put("hallb_IPM2H02_XPOS", "Beam position Y at 2H02");
- }
-
- /**
- * Get the description of a named EPICS variable.
- *
- * @param name the name of the variable
- */
- public static String getVariableDescription(final String name) {
- return VARIABLES.get(name);
- }
-
- /**
- * Get the static list of all available EPICs variable names.
- * <p>
- * This could be different than the variable names which were actually written into the collection header. For this,
- * instead use the method {@link #getKeys()} method.
- *
- * @return the set of default EPICS variable names
- */
- public static Set<String> getVariableNames() {
- return VARIABLES.keySet();
- };
-
- /**
* <p>
* Read data into this object from an LCIO event using the default collection name.
* <p>
@@ -109,7 +49,7 @@
* {@link org.lcsim.util.Driver#process(EventHeader)} method.
*
* @param event the LCIO event
- * @return the EPICS data from the event
+ * @return the EPICS data from the event or null if none exists
*/
public static EpicsData read(final EventHeader event) {
if (event.hasCollection(GenericObject.class, EpicsData.DEFAULT_COLLECTION_NAME)) {
@@ -210,9 +150,6 @@
/**
* Get the list of EPICS variables used by this object.
- * <p>
- * This could potentially be different than the list of default names from {@link #getVariableNames()} as not all
- * variables are included in every EPICS event.
*
* @return the list of used EPICS variable names
*/
@@ -243,14 +180,15 @@
*
* @param epicsHeader the {@link EpicsHeader} object
*/
- void setEpicsHeader(final EpicsHeader epicsHeader) {
+ public void setEpicsHeader(final EpicsHeader epicsHeader) {
this.epicsHeader = epicsHeader;
}
/**
- * Set a double value by name.
- *
- * @return the value from the key
+ * Set a variable's value.
+ *
+ * @param name the name of the variable
+ * @param value the new value of the variable
*/
public void setValue(final String name, final double value) {
this.dataMap.put(name, value);
@@ -268,7 +206,7 @@
newObject.setKeys(new String[this.dataMap.size()]);
newObject.setValues(new double[this.dataMap.size()]);
-
+
int index = 0;
for (final String key : this.dataMap.keySet()) {
newObject.setKey(index, key);
@@ -278,10 +216,8 @@
// Write header information into the object's int array.
if (epicsHeader != null) {
- final int[] headerData = new int[] {
- epicsHeader.getRun(),
- epicsHeader.getSequence(),
- epicsHeader.getTimeStamp()};
+ final int[] headerData = new int[] {epicsHeader.getRun(), epicsHeader.getSequence(),
+ epicsHeader.getTimestamp()};
newObject.setHeaderData(headerData);
}
return newObject;
Modified: java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsEvioProcessor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsEvioProcessor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsEvioProcessor.java Thu Aug 20 12:50:12 2015
@@ -50,7 +50,7 @@
// Is this an EPICS event?
if (EventTagConstant.EPICS.isEventTag(evio)) {
- LOGGER.info("processing EPICS event " + evio.getEventNumber());
+ LOGGER.fine("processing EPICS event " + evio.getEventNumber());
// Find the bank with the EPICS data string.
final BaseStructure epicsBank = EvioBankTag.EPICS_STRING.findBank(evio);
@@ -73,7 +73,7 @@
} else {
// This is an error because the string data bank should always be present in EPICS events.
- final RuntimeException x = new RuntimeException("No EPICS data bank found in EPICS event.");
+ final RuntimeException x = new RuntimeException("No data bank found in EPICS event.");
LOGGER.log(Level.SEVERE, x.getMessage(), x);
throw x;
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsGenericObject.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsGenericObject.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsGenericObject.java Thu Aug 20 12:50:12 2015
@@ -36,11 +36,23 @@
return this.values[index];
}
+ /**
+ * Dummy implementation.
+ *
+ * @param index the array index
+ * @return always returns 0
+ */
@Override
public float getFloatVal(final int index) {
return 0;
}
+ /**
+ * Get an int value which is used to store the EPICS header information.
+ *
+ * @param index the array index
+ * @return the int value at <code>index</code>
+ */
@Override
public int getIntVal(final int index) {
return headerData[index];
@@ -75,11 +87,21 @@
return this.values.length;
}
+ /**
+ * Dummy implementation.
+ *
+ * @return always returns 0
+ */
@Override
public int getNFloat() {
return 0;
}
+ /**
+ * Get the number of int values which is the length of the data header.
+ *
+ * @return the number of int values
+ */
@Override
public int getNInt() {
return this.headerData.length;
Modified: java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsHeader.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsHeader.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsHeader.java Thu Aug 20 12:50:12 2015
@@ -1,7 +1,7 @@
package org.hps.record.epics;
/**
- * Representation of EPICs header data (run, sequence, time stamp).
+ * Representation of EPICs header data from the EVIO bank (run, sequence, time stamp).
*
* @author Jeremy McCormick, SLAC
*/
@@ -31,14 +31,15 @@
private final int sequence;
/**
- * The time stamp in seconds (Unix).
+ * The unix time in seconds.
*/
private final int timestamp;
/**
* Class constructor.
* <p>
- * The data array should be length 3 and usually will come from the int data of a <code>GenericObject</code>.
+ * The data array should be length 3 and usually will come from the int data of a <code>GenericObject</code>. In
+ * order, it should contain the run, sequence and timestamp values.
*
* @param data the header data with length 3
*/
@@ -74,7 +75,7 @@
*
* @return the time stamp
*/
- public int getTimeStamp() {
+ public int getTimestamp() {
return timestamp;
}
}
Copied: java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsRunProcessor.java (from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java)
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/epics/EpicsRunProcessor.java Thu Aug 20 12:50:12 2015
@@ -1,25 +1,19 @@
-package org.hps.record.evio.crawler;
+package org.hps.record.epics;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
-import org.hps.record.epics.EpicsData;
-import org.hps.record.epics.EpicsEvioProcessor;
import org.hps.record.evio.EvioEventProcessor;
import org.jlab.coda.jevio.EvioEvent;
/**
- * Create a summary log of EPICS information found in EVIO events.
+ * Creates a list of EPICS data found in EVIO events across an entire job.
*
* @author Jeremy McCormick, SLAC
*/
-final class EpicsLog extends EvioEventProcessor {
-
- /**
- * A count of how many times a given EPICS variable is found in the input for computing the mean value across the
- * entire run.
- */
- private final Map<String, Integer> counts = new HashMap<String, Integer>();
+public final class EpicsRunProcessor extends EvioEventProcessor {
/**
* The current EPICS data block from the EVIO events (last one that was found).
@@ -27,9 +21,11 @@
private EpicsData currentEpicsData;
/**
- * The summary information for the variables computed from the mean values across the whole run.
+ * Collection of the EPICS data accumulated during the job.
+ * <p>
+ * A set is used here to avoid adding duplicate objects.
*/
- private final EpicsData logData = new EpicsData();
+ private Set<EpicsData> epicsDataSet;
/**
* The processor for extracting the EPICS information from EVIO events.
@@ -39,31 +35,16 @@
/**
* Create an EPICs log.
*/
- EpicsLog() {
+ public EpicsRunProcessor() {
}
/**
- * End of job hook which computes the mean values for all EPICS variables found in the run.
+ * Get the EPICS data from the job.
+ *
+ * @return the EPICS data from the job
*/
- @Override
- public void endJob() {
- System.out.println(this.logData);
-
- // Compute means for all EPICS variables.
- for (final String name : this.logData.getKeys()) {
- final double total = this.logData.getValue(name);
- final double mean = total / this.counts.get(name);
- this.logData.setValue(name, mean);
- }
- }
-
- /**
- * Get the {@link org.hps.record.epics.EpicsData} which contains mean values for the run.
- *
- * @return the {@link org.hps.record.epics.EpicsData} for the run
- */
- EpicsData getEpicsData() {
- return this.logData;
+ public List<EpicsData> getEpicsData() {
+ return new ArrayList<EpicsData>(this.epicsDataSet);
}
/**
@@ -71,33 +52,22 @@
*/
@Override
public void process(final EvioEvent evioEvent) {
+
+ // Call the processor that will load EPICS data if it exists in the event.
this.processor.process(evioEvent);
this.currentEpicsData = this.processor.getEpicsData();
- this.update();
+
+ // Add EPICS data to the collection.
+ if (this.currentEpicsData != null) {
+ this.epicsDataSet.add(this.currentEpicsData);
+ }
}
/**
- * Update state from the current EPICS data.
- * <p>
- * If the current data is <code>null</code>, this method does nothing.
+ * Start of job hook (reset the list of EPICS data).
*/
- private void update() {
- if (this.currentEpicsData != null) {
- for (final String name : this.currentEpicsData.getKeys()) {
- if (!this.logData.getKeys().contains(name)) {
- this.logData.setValue(name, 0.);
- }
- if (!this.counts.keySet().contains(name)) {
- this.counts.put(name, 0);
- }
- int count = this.counts.get(name);
- count += 1;
- this.counts.put(name, count);
- final double value = this.logData.getValue(name) + this.currentEpicsData.getValue(name);
- this.logData.setValue(name, value);
- // System.out.println(name + " => added " + this.currentData.getValue(name) + "; total = " + value +
- // "; mean = " + value / count);
- }
- }
+ @Override
+ public void startJob() {
+ epicsDataSet = new HashSet<EpicsData>();
}
}
Copied: java/trunk/record-util/src/main/java/org/hps/record/evio/EventCountProcessor.java (from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventCountProcessor.java)
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventCountProcessor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EventCountProcessor.java Thu Aug 20 12:50:12 2015
@@ -1,12 +1,8 @@
-package org.hps.record.evio.crawler;
+package org.hps.record.evio;
import java.util.HashMap;
import java.util.Map;
-import org.hps.record.evio.EventTagBitMask;
-import org.hps.record.evio.EventTagConstant;
-import org.hps.record.evio.EvioEventProcessor;
-import org.hps.record.evio.EvioEventUtilities;
import org.jlab.coda.jevio.EvioEvent;
/**
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/EvioEventConstants.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/EvioEventConstants.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EvioEventConstants.java Thu Aug 20 12:50:12 2015
@@ -16,29 +16,16 @@
* EPICS bank tag.
*/
public static final int EPICS_BANK_TAG = 57620;
-
-
- /**
- * Tag of bank containing the EPICS data bank.
- */
- public static final int EPICS_PARENT_BANK_TAG = 129;
-
- /**
- * EPICS 20 second event tag.
- */
- // FIXME: This is unused and not handled in event processing.
- public static final int EPICS_BANK_TAG_20s = -1;
-
- /**
- * EPICS 2 second event tag.
- */
- // FIXME: This is unused and not handled in event processing.
- public static final int EPICS_BANK_TAG_2s = -1;
/**
* EPICS event tag.
*/
public static final int EPICS_EVENT_TAG = 31;
+
+ /**
+ * Tag of bank containing the EPICS data bank.
+ */
+ public static final int EPICS_PARENT_BANK_TAG = 129;
/**
* Event ID bank tag.
@@ -58,7 +45,6 @@
/**
* Pause event tag.
*/
- // FIXME: Not generally handled or used in event processing.
public static final int PAUSE_EVENT_TAG = 19;
/**
Copied: java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileFilter.java (from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java)
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileFilter.java Thu Aug 20 12:50:12 2015
@@ -1,27 +1,29 @@
-package org.hps.record.evio.crawler;
+package org.hps.record.evio;
import java.io.File;
import java.io.FileFilter;
/**
- * This is a simple file filter that will accept EVIO files with a certain convention to their naming which looks like
- * <i>FILENAME.evio.SEQUENCE</i>. This matches the convention used by the CODA DAQ software.
+ * This is a simple file filter that will accept EVIO files with a certain pattern to their file names:<br/>
+ * <i>FILENAME.evio.SEQUENCE</i>.
+ * <p>
+ * This matches the convention used by the CODA DAQ software.
*
* @author Jeremy McCormick, SLAC
*/
-final class EvioFileFilter implements FileFilter {
+public final class EvioFileFilter implements FileFilter {
/**
- * Return <code>true</code> if file is an EVIO file with correct file name convention.
+ * Return <code>true</code> if file is an EVIO file with the correct file naming convention.
*
- * @return <code>true</code> if file is an EVIO file with correct file name convention
+ * @return <code>true</code> if file is an EVIO file with correct file naming convention
*/
@Override
public boolean accept(final File pathname) {
final boolean isEvio = pathname.getName().contains(".evio");
boolean hasSeqNum = false;
try {
- EvioFileUtilities.getSequence(pathname);
+ EvioFileUtilities.getSequenceFromName(pathname);
hasSeqNum = true;
} catch (final Exception e) {
}
Added: java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileMetaData.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileMetaData.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileMetaData.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,315 @@
+package org.hps.record.evio;
+
+import java.io.File;
+import java.util.Date;
+
+/**
+ * Meta data that can be extracted from EVIO files.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public final class EvioFileMetaData {
+
+ /**
+ * The number of bad events in the file that are unreadable.
+ */
+ private int badEventCount;
+
+ /**
+ * The total number of bytes in the file.
+ */
+ private long byteCount;
+
+ /**
+ * The end date of the file which will be taken from the <i>END</i> event or the last physics event.
+ */
+ private Date endDate;
+
+ /**
+ * The last event number in the file.
+ */
+ private int endEvent;
+
+ /**
+ * The number of events in the file.
+ */
+ private int eventCount;
+
+ /**
+ * The EVIO file which is set at class construction time and cannot be changed.
+ */
+ private final File evioFile;
+
+ /**
+ * <code>true</code> if there is an <i>END</i> event in this file.
+ */
+ private boolean hasEnd;
+
+ /**
+ * <code>true</code> if there is a <i>PRESTART</i> event in this file.
+ */
+ private boolean hasPrestart;
+
+ /**
+ * The run number.
+ */
+ private int run;
+
+ /**
+ * The file sequence number.
+ */
+ private int sequence;
+
+ /**
+ * The start date which comes from the <i>PRESTART</i> event or the first physics event.
+ */
+ private Date startDate;
+
+ /**
+ * The first event number in the file.
+ */
+ private int startEvent;
+
+ /**
+ * Create a meta data object.
+ *
+ * @param evioFile the EVIO file to which the meta data applies
+ */
+ public EvioFileMetaData(final File evioFile) {
+ if (evioFile == null) {
+ throw new IllegalArgumentException("The EVIO file argument is null.");
+ }
+ if (!evioFile.exists()) {
+ throw new IllegalArgumentException("The file " + evioFile.getPath()
+ + " does not exist or it is inaccessible.");
+ }
+ this.evioFile = evioFile;
+ }
+
+ /**
+ * Get the bad event count.
+ *
+ * @return the bad event count
+ */
+ public int getBadEventCount() {
+ return badEventCount;
+ }
+
+ /**
+ * Get the byte count.
+ *
+ * @return the byte count
+ */
+ public long getByteCount() {
+ return byteCount;
+ }
+
+ /**
+ * Get the end date.
+ *
+ * @return the end date
+ */
+ public Date getEndDate() {
+ return endDate;
+ }
+
+ /**
+ * Get the end event number.
+ *
+ * @return the end event number
+ */
+ public int getEndEvent() {
+ return endEvent;
+ }
+
+ /**
+ * Get the number of events in the file (all types).
+ *
+ * @return the number of events in the file
+ */
+ public int getEventCount() {
+ return eventCount;
+ }
+
+ /**
+ * Get the EVIO file.
+ *
+ * @return the EVIO file
+ */
+ public File getEvioFile() {
+ return evioFile;
+ }
+
+ /**
+ * Get the run number.
+ *
+ * @return the run number
+ */
+ public Integer getRun() {
+ return run;
+ }
+
+ /**
+ * Get the file sequence number, numbered from 0.
+ *
+ * @return the file sequence number
+ */
+ public int getSequence() {
+ return sequence;
+ }
+
+ /**
+ * Get the start date.
+ *
+ * @return the start date
+ */
+ public Date getStartDate() {
+ return startDate;
+ }
+
+ /**
+ * Get the first event number in the file.
+ *
+ * @return the first event number in the file
+ */
+ public int getStartEvent() {
+ return startEvent;
+ }
+
+ /**
+ * Return <code>true</code> if the file has an EVIO <i>END</i> event.
+ *
+ * @return <code>true</code> if the file has an EVIO <i>END</i> event
+ */
+ public boolean hasEnd() {
+ return hasEnd;
+ }
+
+ /**
+ * Return <code>true</code> if the file has an EVIO <i>PRESTART</i> event.
+ *
+ * @return <code>true</code> if the file has an EVIO <i>PRESTART</i> event
+ */
+ public boolean hasPrestart() {
+ return hasPrestart;
+ }
+
+ /**
+ * Set the bad event count.
+ *
+ * @param badEventCount the bad event count
+ */
+ void setBadEventCount(final int badEventCount) {
+ if (badEventCount < 0) {
+ throw new IllegalArgumentException("badEventCount");
+ }
+ this.badEventCount = badEventCount;
+ }
+
+ /**
+ * Set the byte count.
+ *
+ * @param byteCount the byte count
+ */
+ void setByteCount(final long byteCount) {
+ if (byteCount < 0) {
+ throw new IllegalArgumentException("byteCount");
+ }
+ this.byteCount = byteCount;
+ }
+
+ /**
+ * Set the end date.
+ *
+ * @param endDate the end date
+ */
+ void setEndDate(final Date endDate) {
+ this.endDate = endDate;
+ }
+
+ /**
+ * Set the end event number.
+ *
+ * @param endEvent the end event number
+ */
+ void setEndEvent(final int endEvent) {
+ this.endEvent = endEvent;
+ }
+
+ /**
+ * Set the event count.
+ *
+ * @param eventCount the event count
+ */
+ void setEventCount(final int eventCount) {
+ this.eventCount = eventCount;
+ }
+
+ /**
+ * Set whether the file has an EVIO <i>END</i> event.
+ *
+ * @param hasEnd <code>true</code> if file has an EVIO <i>END</i> event
+ */
+ void setHasEnd(final boolean hasEnd) {
+ this.hasEnd = hasEnd;
+ }
+
+ /**
+ * Set whether the file has an EVIO <i>PRESTART</i> event.
+ *
+ * @param hasPrestart <code>true</code> if file has an EVIO <i>PRESTART</i> event
+ */
+ void setHasPrestart(final boolean hasPrestart) {
+ this.hasPrestart = hasPrestart;
+ }
+
+ /**
+ * Set the run number.
+ *
+ * @param run the run number
+ */
+ void setRun(final int run) {
+ this.run = run;
+ }
+
+ /**
+ * Set the sequence number
+ *
+ * @param sequence the sequence number
+ */
+ void setSequence(final int sequence) {
+ this.sequence = sequence;
+ }
+
+ /**
+ * Set the start date.
+ *
+ * @param startDate the start date
+ */
+ void setStartDate(final Date startDate) {
+ this.startDate = startDate;
+ }
+
+ /**
+ * Set the start event number.
+ *
+ * @param startEvent the start event number
+ */
+ void setStartEvent(final int startEvent) {
+ this.startEvent = startEvent;
+ }
+
+ /**
+ * Convert this object to a human readable string.
+ *
+ * @return this object converted to a string
+ */
+ @Override
+ public String toString() {
+ return "EvioFileMetaData { evioFile: " + this.evioFile + ", startDate: " + this.startDate + "endDate: "
+ + this.endDate + "badEventCount: " + this.badEventCount + ", byteCount: " + this.byteCount
+ + ", eventCount: " + this.eventCount + ", hasPrestart: " + this.hasPrestart + ", hasEnd: "
+ + this.hasEnd + ", run: " + this.run + ", fileNumber: " + sequence + ", startEvent: "
+ + this.startEvent + ", endEvent: " + endEvent + " }";
+ }
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileMetaDataReader.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileMetaDataReader.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileMetaDataReader.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,151 @@
+package org.hps.record.evio;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.jlab.coda.jevio.BaseStructure;
+import org.jlab.coda.jevio.EvioEvent;
+import org.jlab.coda.jevio.EvioException;
+import org.jlab.coda.jevio.EvioReader;
+import org.lcsim.util.log.DefaultLogFormatter;
+import org.lcsim.util.log.LogUtil;
+
+/**
+ * Read {@link EvioFileMetaData} from an EVIO file.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public class EvioFileMetaDataReader {
+
+ /**
+ * The class logger.
+ */
+ private static Logger LOGGER = LogUtil.create(EvioFileMetaDataReader.class, new DefaultLogFormatter(), Level.ALL);
+
+ /**
+ * Get meta data from the EVIO file.
+ *
+ * @param evioFile the EVIO file
+ * @return the file's meta data
+ */
+ public EvioFileMetaData getMetaData(final File evioFile) {
+
+ LOGGER.info("getting meta data for " + evioFile.getPath());
+
+ final EvioFileMetaData metaData = new EvioFileMetaData(evioFile);
+
+ EvioEvent evioEvent = null;
+ Date startDate = null;
+ Date endDate = null;
+ int badEventCount = 0;
+ int eventCount = 0;
+ int byteCount = 0;
+ boolean hasPrestart = false;
+ boolean hasEnd = false;
+ int[] headBankData = null;
+ int[] eventIdData = null;
+ Integer run = null;
+ Integer endEvent = null;
+ Integer startEvent = null;
+
+ final int fileNumber = EvioFileUtilities.getSequenceFromName(evioFile);
+ LOGGER.info("set file number to " + fileNumber);
+
+ try {
+ final EvioReader evioReader = EvioFileUtilities.open(evioFile, false);
+ LOGGER.getHandlers()[0].flush();
+ eventCount = evioReader.getEventCount();
+ while (true) {
+ try {
+ evioEvent = evioReader.parseNextEvent();
+ if (evioEvent == null) {
+ break;
+ }
+ } catch (final Exception e) {
+ badEventCount++;
+ LOGGER.log(Level.WARNING, "got bad EVIO event", e);
+ continue;
+ }
+ byteCount += evioEvent.getTotalBytes();
+ if (EventTagConstant.PRESTART.isEventTag(evioEvent)) {
+ LOGGER.info("found PRESTART");
+ hasPrestart = true;
+ final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent);
+ final long timestamp = controlEventData[0] * 1000L;
+ startDate = new Date(timestamp);
+ LOGGER.info("set start date to " + startDate);
+ if (run == null) {
+ run = controlEventData[1];
+ LOGGER.info("set run to " + run);
+ }
+ } else if (EventTagConstant.END.isEventTag(evioEvent)) {
+ LOGGER.info("found END");
+ hasEnd = true;
+ final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent);
+ final long timestamp = controlEventData[0] * 1000L;
+ endDate = new Date(timestamp);
+ LOGGER.info("set end date to " + endDate);
+ if (run == null) {
+ run = controlEventData[1];
+ LOGGER.info("set run to " + run);
+ }
+ } else if (EventTagBitMask.PHYSICS.isEventTag(evioEvent)) {
+ final BaseStructure headBank = EvioEventUtilities.getHeadBank(evioEvent);
+ if (headBank != null) {
+ headBankData = headBank.getIntData();
+ if (startDate == null) {
+ startDate = new Date(headBankData[3] * 1000);
+ LOGGER.info("set start date to " + endDate + " from physics event");
+ }
+ if (run == null) {
+ run = headBankData[1];
+ LOGGER.info("set run to " + run + " from physics event");
+ }
+ }
+ eventIdData = EvioEventUtilities.getEventIdData(evioEvent);
+ if (eventIdData != null) {
+ if (startEvent == null) {
+ startEvent = eventIdData[0];
+ LOGGER.info("set start event " + startEvent);
+ }
+ }
+ }
+ LOGGER.getHandlers()[0].flush();
+ }
+
+ if (endDate == null) {
+ if (headBankData != null) {
+ endDate = new Date(headBankData[3] * 1000);
+ LOGGER.info("set end date to " + endDate + " from last head bank");
+ }
+ }
+ if (eventIdData != null) {
+ endEvent = eventIdData[0];
+ LOGGER.info("set end event " + endEvent);
+ }
+ } catch (EvioException | IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ metaData.setStartDate(startDate);
+ metaData.setEndDate(endDate);
+ metaData.setBadEventCount(badEventCount);
+ metaData.setByteCount(byteCount);
+ metaData.setEventCount(eventCount);
+ metaData.setHasPrestart(hasPrestart);
+ metaData.setHasEnd(hasEnd);
+ metaData.setRun(run);
+ metaData.setSequence(fileNumber);
+ if (endEvent != null) {
+ metaData.setEndEvent(endEvent);
+ }
+ if (startEvent != null) {
+ metaData.setStartEvent(startEvent);
+ }
+
+ return metaData;
+ }
+}
Copied: java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileSequenceComparator.java (from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileSequenceComparator.java)
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileSequenceComparator.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileSequenceComparator.java Thu Aug 20 12:50:12 2015
@@ -1,4 +1,4 @@
-package org.hps.record.evio.crawler;
+package org.hps.record.evio;
import java.io.File;
import java.util.Comparator;
@@ -8,7 +8,7 @@
*
* @author Jeremy McCormick, SLAC
*/
-final class EvioFileSequenceComparator implements Comparator<File> {
+public final class EvioFileSequenceComparator implements Comparator<File> {
/**
* Compare two EVIO files by their sequence numbers.
@@ -17,8 +17,8 @@
*/
@Override
public int compare(final File o1, final File o2) {
- final Integer sequenceNumber1 = EvioFileUtilities.getSequence(o1);
- final Integer sequenceNumber2 = EvioFileUtilities.getSequence(o2);
+ final Integer sequenceNumber1 = EvioFileUtilities.getSequenceFromName(o1);
+ final Integer sequenceNumber2 = EvioFileUtilities.getSequenceFromName(o2);
return sequenceNumber1.compareTo(sequenceNumber2);
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileSource.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileSource.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileSource.java Thu Aug 20 12:50:12 2015
@@ -12,12 +12,12 @@
import org.jlab.coda.jevio.EvioReader;
/**
- * A basic implementation of an <tt>AbstractRecordSource</tt> for supplying <tt>EvioEvent</tt> objects to a loop from
- * EVIO files.
+ * A basic implementation of an <tt>AbstractRecordSource</tt> for supplying <tt>EvioEvent</tt> objects to a loop from a
+ * list of EVIO files.
* <p>
* Unlike the LCIO record source, it has no rewind or indexing capabilities.
*
- * @author <a href="mailto:[log in to unmask]">Jeremy McCormick</a>
+ * @author Jeremy McCormick, SLAC
*/
public final class EvioFileSource extends AbstractRecordSource {
@@ -48,7 +48,7 @@
*/
public EvioFileSource(final File file) {
this.files.add(file);
- openReader();
+ this.openReader();
}
/**
@@ -58,7 +58,7 @@
*/
public EvioFileSource(final List<File> files) {
this.files.addAll(files);
- openReader();
+ this.openReader();
}
/**
@@ -89,6 +89,15 @@
@Override
public Object getCurrentRecord() throws IOException {
return this.currentEvent;
+ }
+
+ /**
+ * Get the list of files.
+ *
+ * @return the list of files
+ */
+ public List<File> getFiles() {
+ return this.files;
}
/**
@@ -130,10 +139,10 @@
throw new IOException(e);
}
if (this.currentEvent == null) {
- closeReader();
+ this.closeReader();
this.fileIndex++;
- if (!endOfFiles()) {
- openReader();
+ if (!this.endOfFiles()) {
+ this.openReader();
continue;
} else {
throw new NoSuchRecordException();
@@ -151,7 +160,7 @@
private void openReader() {
try {
System.out.println("Opening reader for file " + this.files.get(this.fileIndex) + " ...");
- this.reader = new EvioReader(this.files.get(this.fileIndex), false,true);
+ this.reader = new EvioReader(this.files.get(this.fileIndex), false, true);
System.out.println("Done opening file.");
} catch (EvioException | IOException e) {
throw new RuntimeException(e);
@@ -167,4 +176,5 @@
public boolean supportsNext() {
return true;
}
+
}
Copied: java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java (from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java)
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java Thu Aug 20 12:50:12 2015
@@ -1,26 +1,24 @@
-package org.hps.record.evio.crawler;
+package org.hps.record.evio;
import java.io.File;
import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
import java.util.logging.Logger;
-import org.hps.record.evio.EvioEventConstants;
-import org.hps.record.evio.EvioEventUtilities;
-import org.jlab.coda.jevio.BaseStructure;
-import org.jlab.coda.jevio.EvioEvent;
import org.jlab.coda.jevio.EvioException;
import org.jlab.coda.jevio.EvioReader;
import org.lcsim.util.log.LogUtil;
/**
- * A miscellaneous collection of EVIO file utility methods used by the crawler package.
+ * A miscellaneous collection of EVIO file utility methods.
*
* @author Jeremy McCormick, SLAC
*/
-final class EvioFileUtilities {
+public final class EvioFileUtilities {
/**
- * Setup logger.
+ * Setup class logger.
*/
private static final Logger LOGGER = LogUtil.create(EvioFileUtilities.class);
@@ -36,7 +34,7 @@
* @return the cached file path (prepends "/cache" to the path)
* @throws IllegalArgumentException if the file is not on the MSS (e.g. path does not start with "/mss")
*/
- static File getCachedFile(final File file) {
+ public static File getCachedFile(final File file) {
if (!isMssFile(file)) {
throw new IllegalArgumentException("File " + file.getPath() + " is not on the JLab MSS.");
}
@@ -47,80 +45,13 @@
}
/**
- * Get the end date
- *
- * @param evioReader the <code>EvioReader</code>
- * @return the run end date
- */
- static Long getEndTimestamp(final EvioReader evioReader) {
-
- // Date endDate = null;
- Long timestamp = null;
-
- try {
- // Search for the last physics event in the last 5 events of the file.
- System.out.println("going to event " + (evioReader.getEventCount() - 5) + " / "
- + evioReader.getEventCount() + " to find end date");
- evioReader.gotoEventNumber(evioReader.getEventCount() - 5);
- EvioEvent evioEvent = null;
- EvioEvent lastPhysicsEvent = null;
-
- // Find last physics event.
- while ((evioEvent = evioReader.parseNextEvent()) != null) {
- if (EvioEventUtilities.isPhysicsEvent(evioEvent)) {
- lastPhysicsEvent = evioEvent;
- }
- }
-
- // If there is no physics event found this is an error.
- if (lastPhysicsEvent == null) {
- throw new RuntimeException("No physics event found.");
- }
-
- // Get the timestamp from the head bank of the physics event.
- LOGGER.info("getting head bank date from " + lastPhysicsEvent.getEventNumber());
- final Long eventTimestamp = getHeadBankTimestamp(lastPhysicsEvent);
- if (eventTimestamp != null) {
- LOGGER.info("found end timestamp " + eventTimestamp);
- timestamp = eventTimestamp;
- } else {
- throw new RuntimeException("No timestamp found in head bank.");
- }
-
- } catch (EvioException | IOException e) {
- throw new RuntimeException(e);
- }
- return timestamp;
- }
-
- /**
- * Get the date from the head bank.
- *
- * @param event the EVIO file
- * @return the date from the head bank or null if not found
- */
- static Long getHeadBankTimestamp(final EvioEvent event) {
- // Date date = null;
- Long timestamp = null;
- final BaseStructure headBank = EvioEventUtilities.getHeadBank(event);
- if (headBank != null) {
- final int[] data = headBank.getIntData();
- final long time = data[3];
- if (time != 0L) {
- timestamp = time * MILLISECONDS;
- }
- }
- return timestamp;
- }
-
- /**
* Get the run number from the file name.
*
* @param file the EVIO file
* @return the run number
* @throws Exception if there is a problem parsing out the run number
*/
- static Integer getRun(final File file) {
+ public static Integer getRunFromName(final File file) {
final String name = file.getName();
final int startIndex = name.lastIndexOf("_") + 1;
final int endIndex = name.indexOf(".");
@@ -134,65 +65,9 @@
* @return the file's sequence number
* @throws Exception if there is an error parsing out the sequence number
*/
- static Integer getSequence(final File file) {
+ public static Integer getSequenceFromName(final File file) {
final String name = file.getName();
return Integer.parseInt(name.substring(name.lastIndexOf(".") + 1));
- }
-
- /**
- * Get the start date from the first physics event.
- *
- * @param evioReader the <code>EvioReader</code>
- * @return the run start date
- */
- static Long getStartTimestamp(final EvioReader evioReader) {
-
- Long timestamp = null;
-
- // Read events until there is a physics event and return its timestamp.
- try {
- EvioEvent event = null;
- while ((event = evioReader.parseNextEvent()) != null) {
- if (EvioEventUtilities.isPhysicsEvent(event)) {
- if ((timestamp = getHeadBankTimestamp(event)) != null) {
- break;
- }
- }
- }
- } catch (EvioException | IOException e) {
- throw new RuntimeException(e);
- }
- return timestamp;
- }
-
- /**
- * Get the date from the control bank of an EVIO event.
- *
- * @param file the EVIO file
- * @param eventTag the event tag on the bank
- * @param gotoEvent an event to start the scanning
- * @return the control bank date or null if not found
- */
- static Long getTimestamp(final EvioReader evioReader, final int eventTag, final int gotoEvent) {
- Long timestamp = null;
- try {
- EvioEvent evioEvent;
- if (gotoEvent > 0) {
- evioReader.gotoEventNumber(gotoEvent);
- } else if (gotoEvent < 0) {
- evioReader.gotoEventNumber(evioReader.getEventCount() + gotoEvent);
- }
- while ((evioEvent = evioReader.parseNextEvent()) != null) {
- if (evioEvent.getHeader().getTag() == eventTag) {
- final int[] data = EvioEventUtilities.getControlEventData(evioEvent);
- timestamp = (long) (data[0] * MILLISECONDS);
- break;
- }
- }
- } catch (EvioException | IOException e) {
- throw new RuntimeException(e);
- }
- return timestamp;
}
/**
@@ -201,35 +76,8 @@
* @param file the file
* @return <code>true</code> if the file is a cached file
*/
- static boolean isCachedFile(final File file) {
+ public static boolean isCachedFile(final File file) {
return file.getPath().startsWith("/cache");
- }
-
- /**
- * Return <code>true</code> if a valid CODA <i>END</i> event can be located in the <code>EvioReader</code>'s current
- * file.
- *
- * @param reader the EVIO reader
- * @return <code>true</code> if valid END event is located
- * @throws Exception if there are IO problems using the reader
- */
- static boolean isEndOkay(final EvioReader reader) throws Exception {
- LOGGER.info("checking is END okay ...");
-
- boolean endOkay = false;
-
- // Go to second to last event for searching.
- reader.gotoEventNumber(reader.getEventCount() - 2);
-
- // Look for END event.
- EvioEvent event = null;
- while ((event = reader.parseNextEvent()) != null) {
- if (event.getHeader().getTag() == EvioEventConstants.END_EVENT_TAG) {
- endOkay = true;
- break;
- }
- }
- return endOkay;
}
/**
@@ -238,7 +86,7 @@
* @param file the file
* @return <code>true</code> if the file is on the MSS
*/
- static boolean isMssFile(final File file) {
+ public static boolean isMssFile(final File file) {
return file.getPath().startsWith("/mss");
}
@@ -250,7 +98,7 @@
* @throws IOException if there is an IO problem
* @throws EvioException if there is an error reading the EVIO data
*/
- static EvioReader open(final File file) throws IOException, EvioException {
+ public static EvioReader open(final File file) throws IOException, EvioException {
return open(file, false);
}
@@ -263,7 +111,7 @@
* @throws IOException if there is an IO problem
* @throws EvioException if there is an error reading the EVIO data
*/
- static EvioReader open(final File file, final boolean sequential) throws IOException, EvioException {
+ public static EvioReader open(final File file, final boolean sequential) throws IOException, EvioException {
File openFile = file;
if (isMssFile(file)) {
openFile = getCachedFile(file);
@@ -284,12 +132,21 @@
* @throws IOException if there is an IO problem
* @throws EvioException if there is an error reading the EVIO data
*/
- static EvioReader open(final String path) throws IOException, EvioException {
+ public static EvioReader open(final String path) throws IOException, EvioException {
return open(new File(path), false);
}
/**
- * Present class instantiation.
+ * Sort a list of EVIO files by their sequence numbers.
+ *
+ * @param evioFileList the list of files to sort
+ */
+ public static void sortBySequence(final List<File> evioFileList) {
+ Collections.sort(evioFileList, new EvioFileSequenceComparator());
+ }
+
+ /**
+ * Prevent class instantiation.
*/
private EvioFileUtilities() {
}
Added: java/trunk/record-util/src/main/java/org/hps/record/evio/EvioLoop.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/EvioLoop.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EvioLoop.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,58 @@
+package org.hps.record.evio;
+
+import org.freehep.record.loop.DefaultRecordLoop;
+
+/**
+ * Implementation of a Freehep <code>RecordLoop</code> for EVIO data.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public class EvioLoop extends DefaultRecordLoop {
+
+ /**
+ * The record adapter.
+ */
+ private final EvioLoopAdapter adapter = new EvioLoopAdapter();
+
+ /**
+ * Create a new record loop.
+ */
+ public EvioLoop() {
+ this.addLoopListener(adapter);
+ this.addRecordListener(adapter);
+ }
+
+ /**
+ * Add an EVIO event processor to the adapter which will be activated for every EVIO event that is processed.
+ *
+ * @param evioEventProcessor the EVIO processor to add
+ */
+ public void addEvioEventProcessor(final EvioEventProcessor evioEventProcessor) {
+ adapter.addEvioEventProcessor(evioEventProcessor);
+ }
+
+ /**
+ * Loop over events from the source.
+ *
+ * @param number the number of events to process or -1L for all events from the source
+ * @return the number of records that were processed
+ */
+ public long loop(final long number) {
+ if (number < 0L) {
+ this.execute(Command.GO, true);
+ } else {
+ this.execute(Command.GO_N, number, true);
+ this.execute(Command.STOP);
+ }
+ return this.getSupplied();
+ }
+
+ /**
+ * Set the EVIO data source.
+ *
+ * @param evioFileSource the EVIO data source
+ */
+ public void setEvioFileSource(final EvioFileSource evioFileSource) {
+ this.setRecordSource(evioFileSource);
+ }
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/evio/EvioLoopAdapter.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/EvioLoopAdapter.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/EvioLoopAdapter.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,96 @@
+package org.hps.record.evio;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.freehep.record.loop.AbstractLoopListener;
+import org.freehep.record.loop.LoopEvent;
+import org.freehep.record.loop.RecordEvent;
+import org.freehep.record.loop.RecordListener;
+import org.jlab.coda.jevio.EvioEvent;
+import org.lcsim.util.log.DefaultLogFormatter;
+import org.lcsim.util.log.LogUtil;
+
+/**
+ * A loop adapter for the {@link EvioLoop} which manages and activates a list of {@link EvioEventProcessor} objects.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public final class EvioLoopAdapter extends AbstractLoopListener implements RecordListener {
+
+ /**
+ * Setup class logger.
+ */
+ private final Logger LOGGER = LogUtil.create(EvioLoopAdapter.class, new DefaultLogFormatter(), Level.ALL);
+
+ /**
+ * List of event processors to activate.
+ */
+ private final List<EvioEventProcessor> processors = new ArrayList<EvioEventProcessor>();
+
+ /**
+ * Create a new loop adapter.
+ */
+ EvioLoopAdapter() {
+ }
+
+ /**
+ * Add an EVIO processor to the adapter.
+ *
+ * @param processor the EVIO processor to add to the adapter
+ */
+ void addEvioEventProcessor(final EvioEventProcessor processor) {
+ LOGGER.info("adding " + processor.getClass().getName() + " to EVIO processors");
+ this.processors.add(processor);
+ }
+
+ /**
+ * Implementation of the finish hook which activates the {@link EvioEventProcessor#endJob()} method of all
+ * registered processors.
+ */
+ @Override
+ protected void finish(final LoopEvent event) {
+ LOGGER.info("executing finish hook");
+ for (final EvioEventProcessor processor : processors) {
+ processor.endJob();
+ }
+ }
+
+ /**
+ * Primary event processing method that activates the {@link EvioEventProcessor#process(EvioEvent)} method of all
+ * registered processors.
+ *
+ * @param recordEvent the record event to process which should have an EVIO event
+ * @throws IllegalArgumentException if the record is the wrong type
+ */
+ @Override
+ public void recordSupplied(final RecordEvent recordEvent) {
+ final Object record = recordEvent.getRecord();
+ if (record instanceof EvioEvent) {
+ final EvioEvent evioEvent = EvioEvent.class.cast(record);
+ for (final EvioEventProcessor processor : processors) {
+ try {
+ processor.process(evioEvent);
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ } else {
+ throw new IllegalArgumentException("The supplied record has the wrong type: " + record.getClass());
+ }
+ }
+
+ /**
+ * Implementation of the start hook which activates the {@link EvioEventProcessor#startJob()} method of all
+ * registered processors.
+ */
+ @Override
+ protected void start(final LoopEvent event) {
+ LOGGER.info("executing start hook");
+ for (final EvioEventProcessor processor : processors) {
+ processor.startJob();
+ }
+ }
+}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java Thu Aug 20 12:50:12 2015
@@ -7,6 +7,7 @@
import java.nio.file.attribute.BasicFileAttributes;
import java.sql.Connection;
import java.sql.SQLException;
+import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashSet;
@@ -20,6 +21,8 @@
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.hps.conditions.database.ConnectionParameters;
+import org.hps.record.run.RunSummary;
+import org.hps.record.run.RunSummaryDaoImpl;
import org.lcsim.util.log.DefaultLogFormatter;
import org.lcsim.util.log.LogUtil;
@@ -50,20 +53,20 @@
* Statically define the command options.
*/
static {
- OPTIONS.addOption("a", "runs", true, "list of run numbers to accept (others will be excluded)");
+ // TODO: add -f argument with file name to include; others would be excluded if they do not match
OPTIONS.addOption("b", "min-date", true, "min date for a file (example \"2015-03-26 11:28:59\")");
OPTIONS.addOption("c", "cache", false, "automatically cache files from MSS to cache disk (JLAB only)");
OPTIONS.addOption("C", "connection-properties", true, "database connection properties file (required)");
OPTIONS.addOption("d", "directory", true, "root directory to start crawling (default is current dir)");
OPTIONS.addOption("E", "evio-processor", true, "class name of an EvioEventProcessor to execute");
OPTIONS.addOption("h", "help", false, "print help and exit (overrides all other arguments)");
- OPTIONS.addOption("m", "max-files", true, "max number of files to process per run (mostly for debugging)");
- OPTIONS.addOption("p", "print", true, "set event printing interval during EVIO processing");
- OPTIONS.addOption("r", "insert", false, "insert information into the run database (not done by default)");
+ OPTIONS.addOption("i", "insert", false, "insert information into the run database (not done by default)");
+ OPTIONS.addOption("L", "log-level", true, "set the log level (INFO, FINE, etc.)");
+ OPTIONS.addOption("r", "run", true, "add a run number to accept (when used others will be excluded)");
OPTIONS.addOption("t", "timestamp-file", true, "existing or new timestamp file name");
OPTIONS.addOption("w", "max-cache-wait", true, "total time to allow for file caching (seconds)");
- OPTIONS.addOption("L", "log-level", true, "set the log level (INFO, FINE, etc.)");
- OPTIONS.addOption("u", "update", false, "allow overriding existing data in the run db (not allowed by default)");
+ OPTIONS.addOption("u", "update", false,
+ "allow replacement of existing data in the run db (not allowed by default)");
OPTIONS.addOption("x", "max-depth", true, "max depth to crawl in the directory tree");
}
@@ -179,9 +182,9 @@
}
// List of one or more runs to accept in the job.
- if (cl.hasOption("a")) {
+ if (cl.hasOption("r")) {
final Set<Integer> acceptRuns = new HashSet<Integer>();
- for (final String runString : cl.getOptionValues("a")) {
+ for (final String runString : cl.getOptionValues("r")) {
final Integer acceptRun = Integer.parseInt(runString);
acceptRuns.add(acceptRun);
LOGGER.config("added run filter " + acceptRun);
@@ -189,7 +192,7 @@
config.setAcceptRuns(acceptRuns);
}
- // Enable run log updating (off by default).
+ // Enable updating of run database.
if (cl.hasOption("r")) {
config.setUpdateRunLog(true);
LOGGER.config("inserting into run database is enabled");
@@ -208,21 +211,7 @@
LOGGER.config("max time for file caching set to " + config.waitTime());
}
- // Max files to process per run; mostly just here for debugging purposes.
- if (cl.hasOption("m")) {
- final int maxFiles = Integer.parseInt(cl.getOptionValue("m"));
- config.setMaxFiles(maxFiles);
- LOGGER.config("max files set to " + maxFiles);
- }
-
- // Event printing interval when doing EVIO event processing.
- if (cl.hasOption("p")) {
- final int eventPrintInterval = Integer.parseInt(cl.getOptionValue("p"));
- config.setEventPrintInterval(eventPrintInterval);
- LOGGER.config("event print interval set to " + eventPrintInterval);
- }
-
- // Flag to allow replacement of existing records in the database; not allowed by default.
+ // Allow deletion and replacement of records in run database.
if (cl.hasOption("u")) {
config.setAllowUpdates(true);
LOGGER.config("deletion and replacement of existing runs in the database is enabled");
@@ -267,7 +256,13 @@
throw new RuntimeException("Error parsing options.", e);
}
+ // Configure the max wait time for file caching operations.
+ if (config.waitTime() != null && config.waitTime() > 0L) {
+ cacheManager.setWaitTime(config.waitTime());
+ }
+
LOGGER.info("done parsing command line options");
+ LOGGER.getHandlers()[0].flush();
return this;
}
@@ -293,31 +288,32 @@
LOGGER.info("running Crawler job");
// Create the file visitor for crawling the root directory with the given date filter.
+ LOGGER.info("creating file visitor");
+ LOGGER.getHandlers()[0].flush();
final EvioFileVisitor visitor = new EvioFileVisitor(config.timestamp());
// Walk the file tree using the visitor.
+ LOGGER.info("walking the dir tree");
+ LOGGER.getHandlers()[0].flush();
this.walk(visitor);
// Get the list of run data created by the visitor.
- final RunLog runs = visitor.getRunLog();
-
- // Print the run numbers that were found.
- runs.printRunNumbers();
-
- // Sort the files on their sequence numbers.
- runs.sortFiles();
+ final RunSummaryMap runs = visitor.getRunMap();
// Process all the files, performing caching from the MSS if necessary.
+ LOGGER.info("processing all runs");
RunProcessor.processAllRuns(this.cacheManager, runs, config);
-
- // Print the summary information after the run processing is done.
- runs.printRunSummaries();
+ LOGGER.getHandlers()[0].flush();
// Execute the run database update.
+ LOGGER.info("updating run database");
this.updateRunDatabase(runs);
+ LOGGER.getHandlers()[0].flush();
// Update the timestamp output file.
+ LOGGER.info("updating the timestamp");
this.updateTimestamp();
+ LOGGER.getHandlers()[0].flush();
LOGGER.info("Crawler job is done!");
}
@@ -328,21 +324,26 @@
* @param runs the list of runs to update
* @throws SQLException if there is a database query error
*/
- private void updateRunDatabase(final RunLog runs) throws SQLException {
+ private void updateRunDatabase(final RunSummaryMap runs) throws SQLException {
// Insert the run information into the database.
if (config.updateRunLog()) {
+ LOGGER.info("updating run database");
+
// Open a DB connection.
final Connection connection = config.connectionParameters().createConnection();
- // Create and configure RunLogUpdater which updates the run log for all runs found in the crawl job.
- final RunLogUpdater runUpdater = new RunLogUpdater(connection, runs, config.allowUpdates());
-
- // Update the DB.
- runUpdater.insert();
+ // Insert all run summaries into the database.
+ new RunSummaryDaoImpl(connection).insertFullRunSummaries(new ArrayList<RunSummary>(runs.getRunSummaries()),
+ config.allowUpdates());
// Close the DB connection.
connection.close();
+
+ LOGGER.info("done updating run database");
+
+ } else {
+ LOGGER.info("run database will not be updated");
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java Thu Aug 20 12:50:12 2015
@@ -13,7 +13,7 @@
import org.hps.record.evio.EvioEventProcessor;
/**
- * Full configuration information for the {@link Crawler class}.
+ * Full configuration information for the {@link Crawler} class.
* <p>
* Method chaining of setters is supported.
*
@@ -38,19 +38,9 @@
private boolean allowUpdates = false;
/**
- * The database connection parameters which must be provided by command line argument.
+ * The database connection parameters which must be provided by a command line argument.
*/
private ConnectionParameters connectionParameters;
-
- /**
- * Default event print interval.
- */
- private final int DEFAULT_EVENT_PRINT_INTERVAL = 1000;
-
- /**
- * Interval for printing out event number while running EVIO processors.
- */
- private int eventPrintInterval = DEFAULT_EVENT_PRINT_INTERVAL;
/**
* The maximum depth to crawl.
@@ -151,15 +141,6 @@
}
/**
- * Get the event print interval.
- *
- * @return the event print interval
- */
- int eventPrintInterval() {
- return this.eventPrintInterval;
- }
-
- /**
* Get the max depth in the directory tree to crawl.
*
* @return the max depth
@@ -231,17 +212,6 @@
}
/**
- * Set the interval for printing the EVIO event numbers during processing.
- *
- * @param eventPrintInterval the event print interval
- * @return this object
- */
- CrawlerConfig setEventPrintInterval(final int eventPrintInterval) {
- this.eventPrintInterval = eventPrintInterval;
- return this;
- }
-
- /**
* Set the max depth.
*
* @param maxDepth the max depth
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java Thu Aug 20 12:50:12 2015
@@ -12,11 +12,13 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.hps.record.evio.EvioFileFilter;
+import org.hps.record.evio.EvioFileUtilities;
import org.lcsim.util.log.DefaultLogFormatter;
import org.lcsim.util.log.LogUtil;
/**
- * A file visitor that crawls directories for EVIO files and returns the information as a {@link RunLog}.
+ * A file visitor that crawls directories for EVIO files and returns the information as a {@link RunSummaryMap}.
*
* @author Jeremy McCormick, SLAC
*/
@@ -35,8 +37,8 @@
/**
* The run log containing information about files from each run.
*/
- private final RunLog runs = new RunLog();
-
+ private final RunSummaryMap runs = new RunSummaryMap();
+
/**
* Create a new file visitor.
*
@@ -83,7 +85,7 @@
*
* @return the run log
*/
- RunLog getRunLog() {
+ RunSummaryMap getRunMap() {
return this.runs;
}
@@ -99,10 +101,10 @@
if (this.accept(file)) {
// Get the run number from the file name.
- final Integer run = EvioFileUtilities.getRun(file);
+ final Integer run = EvioFileUtilities.getRunFromName(file);
// Get the sequence number from the file name.
- final Integer seq = EvioFileUtilities.getSequence(file);
+ final Integer seq = EvioFileUtilities.getSequenceFromName(file);
LOGGER.info("accepted file " + file.getPath() + " with run " + run + " and seq " + seq);
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java Thu Aug 20 12:50:12 2015
@@ -13,6 +13,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.hps.record.evio.EvioFileUtilities;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.input.SAXBuilder;
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java Thu Aug 20 12:50:12 2015
@@ -3,6 +3,8 @@
import java.io.File;
import java.io.FileFilter;
import java.util.Set;
+
+import org.hps.record.evio.EvioFileUtilities;
/**
* A filter which rejects files with run numbers not in a specified set.
@@ -36,6 +38,6 @@
*/
@Override
public boolean accept(final File file) {
- return this.acceptRuns.contains(EvioFileUtilities.getRun(file));
+ return this.acceptRuns.contains(EvioFileUtilities.getRunFromName(file));
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java Thu Aug 20 12:50:12 2015
@@ -1,19 +1,16 @@
package org.hps.record.evio.crawler;
import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.hps.record.evio.EvioEventConstants;
-import org.hps.record.evio.EvioEventProcessor;
+import org.hps.record.epics.EpicsRunProcessor;
+import org.hps.record.evio.EvioFileMetaData;
+import org.hps.record.evio.EvioFileMetaDataReader;
+import org.hps.record.evio.EvioFileSource;
+import org.hps.record.evio.EvioLoop;
import org.hps.record.run.RunSummary;
import org.hps.record.scalers.ScalersEvioProcessor;
-import org.jlab.coda.jevio.EvioEvent;
-import org.jlab.coda.jevio.EvioException;
-import org.jlab.coda.jevio.EvioReader;
import org.lcsim.util.log.DefaultLogFormatter;
import org.lcsim.util.log.LogUtil;
@@ -40,35 +37,24 @@
* @param runs the run log containing the list of run summaries
* @throws Exception if there is an error processing one of the runs
*/
- static void processAllRuns(final JCacheManager cacheManager, final RunLog runs, final CrawlerConfig config)
+ static void processAllRuns(final JCacheManager cacheManager, final RunSummaryMap runs, final CrawlerConfig config)
throws Exception {
- // Configure the max wait time for file caching operations.
- if (config.waitTime() != null && config.waitTime() > 0L) {
- cacheManager.setWaitTime(config.waitTime());
- }
-
// Process all of the runs that were found.
- for (final int run : runs.getSortedRunNumbers()) {
-
- // Get the run summary.
- final RunSummary runSummary = runs.getRunSummary(run);
+ for (final RunSummary runSummary : runs.getRunSummaries()) {
// Clear the cache manager.
if (config.useFileCache()) {
+ LOGGER.info("clearing file cache");
cacheManager.clear();
}
// Create a processor to process all the EVIO events in the run.
+ LOGGER.info("creating run processor for " + runSummary.getRun());
final RunProcessor runProcessor = new RunProcessor(cacheManager, runSummary, config);
- // Add extra processors.
- for (final EvioEventProcessor processor : config.processors()) {
- runProcessor.addProcessor(processor);
- LOGGER.config("added extra EVIO processor " + processor.getClass().getName());
- }
-
// Process all of the run's files.
+ LOGGER.info("processing run " + runSummary.getRun());
runProcessor.processRun();
}
}
@@ -81,27 +67,17 @@
/**
* Processor for extracting EPICS information.
*/
- private final EpicsLog epicsLog;
-
- /**
- * Processor for extracting event type counts (sync, physics, trigger types, etc.).
- */
- private final EventCountProcessor eventCountProcessor;
-
- /**
- * The event printing interval when processing EVIO files.
- */
- private int eventPrintInterval = 1000;
-
- /**
- * Max total files to read (default is unlimited).
- */
- private int maxFiles = -1;
-
- /**
- * The list of EVIO processors to run on the files that are found.
- */
- private final List<EvioEventProcessor> processors = new ArrayList<EvioEventProcessor>();
+ private final EpicsRunProcessor epicsLog;
+
+ /**
+ * The data source with the list of EVIO files to process.
+ */
+ private final EvioFileSource evioFileSource;
+
+ /**
+ * The EVIO event processing loop.
+ */
+ private final EvioLoop evioLoop = new EvioLoop();
/**
* The run summary information updated by running this processor.
@@ -119,9 +95,9 @@
private boolean useFileCache;
/**
- * Create the processor for a single run.
- *
- * @param runSummary the run summary for the run
+ * Create a run processor.
+ *
+ * @param runSummary the run summary object for the run
* @return the run processor
*/
RunProcessor(final JCacheManager cacheManager, final RunSummary runSummary, final CrawlerConfig config) {
@@ -129,39 +105,22 @@
this.runSummary = runSummary;
this.cacheManager = cacheManager;
- // EPICS processor.
- epicsLog = new EpicsLog();
- this.addProcessor(epicsLog);
-
- // Scaler data processor.
+ // Setup record loop.
+ runSummary.sortFiles();
+ evioFileSource = new EvioFileSource(runSummary.getEvioFileList());
+ evioLoop.setEvioFileSource(evioFileSource);
+
+ // Add EPICS processor.
+ epicsLog = new EpicsRunProcessor();
+ evioLoop.addEvioEventProcessor(epicsLog);
+
+ // Add Scaler data processor.
scalersProcessor = new ScalersEvioProcessor();
scalersProcessor.setResetEveryEvent(false);
- this.addProcessor(scalersProcessor);
-
- // Event log processor.
- eventCountProcessor = new EventCountProcessor();
- this.addProcessor(eventCountProcessor);
-
- // Max files.
- if (config.maxFiles() != -1) {
- this.setMaxFiles(config.maxFiles());
- }
-
- // Enable file caching.
+ evioLoop.addEvioEventProcessor(scalersProcessor);
+
+ // Set whether file caching from MSS is enabled.
this.useFileCache(config.useFileCache());
-
- // Set event printing interval.
- this.setEventPrintInterval(config.eventPrintInterval());
- }
-
- /**
- * Add a processor of EVIO events.
- *
- * @param processor the EVIO event processor
- */
- void addProcessor(final EvioEventProcessor processor) {
- this.processors.add(processor);
- LOGGER.config("added processor " + processor.getClass().getSimpleName());
}
/**
@@ -172,10 +131,10 @@
*/
private void cacheFiles() {
- LOGGER.info("caching files from run " + this.runSummary.getRun() + " ...");
+ LOGGER.info("caching files from run " + this.runSummary.getRun());
// Cache all the files and wait for the operation to complete (it will take awhile!).
- this.cacheManager.cache(this.getFiles());
+ this.cacheManager.cache(this.runSummary.getEvioFileList());
final boolean cached = this.cacheManager.waitForCache();
// If the files weren't cached then die.
@@ -187,245 +146,82 @@
}
/**
- * Find the end date in the EVIO events.
- *
- * @param evioReader the open <code>EvioReader</code>
- */
- private void findEndDate(final EvioReader evioReader) {
-
- // Try to get end date from END event.
- Long endTimestamp = EvioFileUtilities.getTimestamp(evioReader, EvioEventConstants.END_EVENT_TAG, -5);
-
- if (endTimestamp != null) {
- // Flag end okay for the run.
- this.runSummary.setEndOkay(true);
- } else {
- // Try to find the end date from the last physics event.
- endTimestamp = EvioFileUtilities.getEndTimestamp(evioReader);
- this.runSummary.setEndOkay(false);
- }
-
- if (endTimestamp == null) {
- // Not finding the end date is a fatal error.
- throw new RuntimeException("Failed to find end date.");
- }
-
- LOGGER.info("found end timestamp " + endTimestamp);
- this.runSummary.setEndTimeUtc(endTimestamp);
- }
-
- /**
- * Find the start date in the EVIO events.
- *
- * @param evioReader the open <code>EvioReader</code>
- */
- private void findStartDate(final EvioReader evioReader) {
-
- // First try to find the start date in the PRESTART event.
- Long startTimestamp = EvioFileUtilities.getTimestamp(evioReader, EvioEventConstants.PRESTART_EVENT_TAG, 0);
-
- if (startTimestamp == null) {
- // Search for start date in first physics event.
- startTimestamp = EvioFileUtilities.getStartTimestamp(evioReader);
- }
-
- if (startTimestamp == null) {
- // Not finding the start date is a fatal error.
- throw new RuntimeException("Failed to find start date.");
- }
-
- LOGGER.fine("got run start " + startTimestamp);
- this.runSummary.setStartTimeUtc(startTimestamp);
- }
-
- /**
- * Get the list of files to process, which will be limited by the {@link #maxFiles} value if it is set.
- *
- * @return the files to process
- */
- private List<File> getFiles() {
- // Get the list of files to process, taking into account the max files setting.
- List<File> files = this.runSummary.getEvioFileList();
- if (this.maxFiles != -1) {
- int toIndex = this.maxFiles;
- if (toIndex > files.size()) {
- toIndex = files.size();
- }
- files = files.subList(0, toIndex);
- }
- return files;
- }
-
- /**
- * Get the list of EVIO processors.
- *
- * @return the list of EVIO processors
- */
- List<EvioEventProcessor> getProcessors() {
- return this.processors;
- }
-
- /**
- * Return <code>true</code> if the file is the first one in the list for the run.
- *
- * @param file the EVIO <code>File</code>
- * @return <code>true</code> if the file is the first one in the list for the run
- */
- private boolean isFirstFile(final File file) {
- return file.equals(this.runSummary.getEvioFileList().first());
- }
-
- /**
- * Return <code>true</code> if the file is the last one in the list for the run.
- *
- * @param file the EVIO <code>File</code>
- * @return <code>true</code> if the file is the last one in the list for the run
- */
- private boolean isLastFile(final File file) {
- return file.equals(this.getFiles().get(this.getFiles().size() - 1));
- }
-
- /**
- * Process events using the list of EVIO processors.
- *
- * @param evioReader the open <code>EvioReader</code>
- * @throws IOException if there is a file IO error
- * @throws EvioException if there is an EVIO error
- * @throws Exception if there is some other error
- */
- private void processEvents(final EvioReader evioReader) throws IOException, EvioException, Exception {
- LOGGER.finer("running EVIO processors ...");
- evioReader.gotoEventNumber(0);
- int nProcessed = 0;
- if (!this.processors.isEmpty()) {
- EvioEvent event = null;
- while ((event = evioReader.parseNextEvent()) != null) {
- for (final EvioEventProcessor processor : this.processors) {
- processor.process(event);
- }
- ++nProcessed;
- if (nProcessed % this.eventPrintInterval == 0) {
- LOGGER.finer("processed " + nProcessed + " EVIO events");
- }
- }
- LOGGER.info("done running EVIO processors");
- }
- }
-
- /**
- * Process a single EVIO file from the run.
- *
- * @param file the EVIO file
- * @throws EvioException if there is an EVIO error
- * @throws IOException if there is some kind of IO error
- * @throws Exception if there is a generic error thrown by event processing
- */
- private void processFile(final File file) throws EvioException, IOException, Exception {
-
- LOGGER.fine("processing file " + file.getPath() + " ...");
-
- EvioReader evioReader = null;
- try {
-
- // Open file for reading (flag should be true for sequential or false for mem map).
- evioReader = EvioFileUtilities.open(file, true);
-
- // If this is the first file then get the start date.
- if (this.isFirstFile(file)) {
- LOGGER.fine("getting run start from first file " + file.getPath() + " ...");
- this.findStartDate(evioReader);
- }
-
- // Go back to the first event and process the events using the list of EVIO processors.
- this.processEvents(evioReader);
-
- // Find end date from last file in the run.
- if (this.isLastFile(file)) {
- LOGGER.fine("getting run end from last file " + file.getPath() + " ...");
- this.findEndDate(evioReader);
- }
-
- } finally {
- // Close the EvioReader for the current file.
- if (evioReader != null) {
- evioReader.close();
- }
- }
- LOGGER.fine("done processing " + file.getPath());
- }
-
- /**
- * Process the run by executing the registered {@link org.hps.record.evio.EvioEventProcessor}s extracting the start
- * and end dates.
+ * Process the run by executing the registered {@link org.hps.record.evio.EvioEventProcessor}s and extracting the
+ * start and end dates.
* <p>
- * This method will also activate file caching, if enabled by the {@link #useFileCache} option.
+ * This method will also execute file caching from MSS, if enabled by the {@link #useFileCache} option.
*
* @throws Exception if there is an error processing a file
*/
void processRun() throws Exception {
- LOGGER.info("processing run " + this.runSummary.getRun() + " ...");
-
- // First cache all the files we will process, if necessary.
+ LOGGER.info("processing " + this.runSummary.getEvioFileList().size() + " files from run "
+ + this.runSummary.getRun());
+
+ // Cache files from MSS if this is enabled.
if (this.useFileCache) {
+ LOGGER.info("caching files from MSS");
this.cacheFiles();
}
- // Run the start of job hooks.
- for (final EvioEventProcessor processor : this.processors) {
- processor.startJob();
- }
-
- // Get the list of files, limited by max files setting.
- final List<File> files = this.getFiles();
-
- LOGGER.info("processing " + files.size() + " from run " + this.runSummary.getRun());
-
- // Process all the files.
- for (final File file : files) {
- this.processFile(file);
- }
-
- // Run the end job hooks.
- LOGGER.info("running end of job hooks on EVIO processors ...");
- for (final EvioEventProcessor processor : this.processors) {
- processor.endJob();
- }
-
+ // Run processors over all files.
+ LOGGER.info("looping over all events");
+ evioLoop.loop(-1);
+
+ // Get run start date.
+ LOGGER.info("setting run start date");
+ this.setRunStartDate();
+
+ // Get run end date.
+ LOGGER.info("setting run end date");
+ this.setRunEndDate();
+
+ // Update run summary from processors.
+ LOGGER.info("updating run summary");
+ this.updateRunSummary();
+
+ LOGGER.info("done processing run " + this.runSummary.getRun());
+ }
+
+ /**
+ * Set the run end date by getting meta data from the last file and copying it to the run summary.
+ */
+ private void setRunEndDate() {
+ final File lastEvioFile = runSummary.getEvioFileList().get(runSummary.getEvioFileList().size() - 1);
+ LOGGER.info("getting meta data for " + lastEvioFile.getPath());
+ final EvioFileMetaDataReader metaDataReader = new EvioFileMetaDataReader();
+ final EvioFileMetaData metaData = metaDataReader.getMetaData(lastEvioFile);
+ LOGGER.info(metaData.toString());
+ LOGGER.info("setting unix end time to " + metaData.getEndDate().getTime() + " from meta data");
+ runSummary.setEndTimeUtc(metaData.getEndDate().getTime());
+ runSummary.setEndOkay(metaData.hasEnd());
+ }
+
+ /**
+ * Set the run start date by getting meta data from the first file and copying it to the run summary.
+ */
+ private void setRunStartDate() {
+ final File firstEvioFile = runSummary.getEvioFileList().get(0);
+ LOGGER.info("getting meta data for " + firstEvioFile.getPath());
+ final EvioFileMetaDataReader metaDataReader = new EvioFileMetaDataReader();
+ final EvioFileMetaData metaData = metaDataReader.getMetaData(firstEvioFile);
+ LOGGER.info(metaData.toString());
+ LOGGER.info("setting unix start time to " + metaData.getStartDate().getTime() + " from meta data");
+ runSummary.setStartTimeUtc(metaData.getStartDate().getTime());
+ }
+
+ /**
+ * Update the current run summary by copying data to it from the EVIO processors.
+ */
+ private void updateRunSummary() {
// Put scaler data from EVIO processor into run summary.
runSummary.setScalerData(this.scalersProcessor.getScalerData());
- // Set the counts of event types on the run summary.
- runSummary.setEventTypeCounts(eventCountProcessor.getEventCounts());
-
// Set total number of events on the run summary from the event counter.
- runSummary.setTotalEvents(this.eventCountProcessor.getTotalEventCount());
+ runSummary.setTotalEvents((int) evioLoop.getTotalCountableConsumed());
// Set EpicsData for the run.
runSummary.setEpicsData(this.epicsLog.getEpicsData());
-
- LOGGER.info("done processing run " + this.runSummary.getRun());
- }
-
- /**
- * Set the event print interval when running the EVIO processors.
- *
- * @param eventPrintInterval the event print interval when running the EVIO processors
- */
- void setEventPrintInterval(final int eventPrintInterval) {
- this.eventPrintInterval = eventPrintInterval;
- }
-
- /**
- * Set the maximum number of files to process.
- * <p>
- * This is intended primarily for debugging.
- *
- * @param maxFiles the maximum number of files to process
- */
- void setMaxFiles(final int maxFiles) {
- this.maxFiles = maxFiles;
- LOGGER.config("max files set to " + maxFiles);
}
/**
Copied: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryMap.java (from r3364, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java)
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryMap.java Thu Aug 20 12:50:12 2015
@@ -1,32 +1,33 @@
package org.hps.record.evio.crawler;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.HashMap;
import java.util.logging.Logger;
import org.hps.record.run.RunSummary;
import org.lcsim.util.log.LogUtil;
/**
- * This class contains information about a series of runs which each have a {@link RunSummary} object.
+ * This class maps run numbers to {@link RunSummary} objects.
*
* @author Jeremy McCormick, SLAC
*/
-final class RunLog {
+@SuppressWarnings("serial")
+final class RunSummaryMap extends HashMap<Integer, RunSummary> {
/**
* Setup logging.
*/
- private static final Logger LOGGER = LogUtil.create(RunLog.class);
+ private static final Logger LOGGER = LogUtil.create(RunSummaryMap.class);
/**
- * A map between run numbers and the run summary information.
+ * Get the collection of {@link RunSummary} objects.
+ *
+ * @return the collection of {@link RunSummary} objects
*/
- private final Map<Integer, RunSummary> runs = new LinkedHashMap<Integer, RunSummary>();
+ public Collection<RunSummary> getRunSummaries() {
+ return this.values();
+ }
/**
* Get a {@link RunSummary} by its run number.
@@ -37,63 +38,10 @@
* @return the <code>RunSummary</code> for the run number
*/
public RunSummary getRunSummary(final int run) {
- if (!this.runs.containsKey(run)) {
+ if (!this.containsKey(run)) {
LOGGER.info("creating new RunSummary for run " + run);
- this.runs.put(run, new RunSummary(run));
+ this.put(run, new RunSummary(run));
}
- return this.runs.get(run);
+ return this.get(run);
}
-
- /**
- * Get the collection of {@link RunSummary} objects.
- *
- * @return the collection of {@link RunSummary} objects
- */
- public Collection<RunSummary> getRunSummaries() {
- return this.runs.values();
- }
-
- /**
- * Get a list of sorted run numbers from this run log.
- * <p>
- * This is a copy of the keys from the map, so modifying it will have no effect on the original.
- *
- * @return the list of sorted run numbers
- */
- List<Integer> getSortedRunNumbers() {
- final List<Integer> runList = new ArrayList<Integer>(this.runs.keySet());
- Collections.sort(runList);
- return runList;
- }
-
- /**
- * Print out each {@link RunSummary} to <code>System.out</code>.
- */
- void printRunSummaries() {
- for (final int run : this.runs.keySet()) {
- this.runs.get(run).printOut(System.out);
- }
- }
-
- /**
- * Sort the file list for each run in place by EVIO sequence numbers.
- */
- void sortFiles() {
- for (final Integer run : this.runs.keySet()) {
- this.runs.get(run).sortFiles();
- }
- }
-
- /**
- * Print the run numbers to the log.
- */
- void printRunNumbers() {
- // Print the list of runs that were found.
- final StringBuffer sb = new StringBuffer();
- for (final Integer run : getSortedRunNumbers()) {
- sb.append(run + " ");
- }
- LOGGER.info("found EVIO files from runs: " + sb.toString());
- }
-
}
Added: java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataDao.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataDao.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataDao.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,65 @@
+package org.hps.record.run;
+
+import java.util.List;
+
+import org.hps.record.epics.EpicsData;
+
+/**
+ * Database Access Object (DAO) API for EPICS data from the run database.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public interface EpicsDataDao {
+
+ /**
+ * Delete EPICS data from the database.
+ *
+ * @param epicsData the EPICS data to delete
+ */
+ void deleteEpicsData(EpicsData epicsData);
+
+ /**
+ * Delete all EPICS data for a run from the database.
+ *
+ * @param run the run number
+ */
+ void deleteEpicsData(int run);
+
+ /**
+ * Get all the EPICS data in the database.
+ *
+ * @return the list of EPICS data
+ */
+ List<EpicsData> getAllEpicsData();
+
+ /**
+ * Get EPICS data by run.
+ *
+ * @param run the run number
+ * @return the EPICS data
+ */
+ List<EpicsData> getEpicsData(int run);
+
+ /**
+ * Get the list of unique variables names used in the database records.
+ *
+ * @return the list of unique variable names
+ */
+ List<String> getVariableNames();
+
+ /**
+ * Insert a list of EPICS data into the database.
+ * <p>
+ * The run number comes from the header information.
+ *
+ * @param epicsDataList the list of EPICS data
+ */
+ void insertEpicsData(List<EpicsData> epicsDataList);
+
+ /**
+ * Updates EPICS data in the database.
+ *
+ * @param epicsData the EPICS data to update
+ */
+ void updateEpicsData(EpicsData epicsData);
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataDaoImpl.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataDaoImpl.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataDaoImpl.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,362 @@
+package org.hps.record.run;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.hps.record.epics.EpicsData;
+import org.hps.record.epics.EpicsHeader;
+
+/**
+ * Implementation of database operations for EPICS data.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public class EpicsDataDaoImpl implements EpicsDataDao {
+
+ /**
+ * SQL data query strings.
+ */
+ private static class EpicsDataQuery {
+
+ /**
+ * Delete by run number.
+ */
+ private static final String DELETE_BY_RUN = "DELETE FROM run_epics WHERE run = ?";
+ /**
+ * Delete by run and sequence number.
+ */
+ private static final String DELETE_RUN_AND_SEQUENCE = "DELETE FROM run_epics WHERE run = ? and sequence = ?";
+ /**
+ * Insert a record.
+ */
+ private static final String INSERT = "INSERT INTO run_epics (run, sequence, timestamp, variable_name, value) VALUES (?, ?, ?, ?, ?)";
+ /**
+ * Select all records.
+ */
+ private static final String SELECT_ALL = "SELECT * FROM run_epics ORDER BY run, sequence";
+ /**
+ * Select by run number.
+ */
+ private static final String SELECT_RUN = "SELECT * FROM run_epics WHERE run = ? ORDER BY `sequence`";
+ /**
+ * Select unique variable names.
+ */
+ private static final String SELECT_VARIABLE_NAMES = "SELECT DISTINCT(variable_name) FROM run_epics ORDER BY variable_name";
+ /**
+ * Update a record.
+ */
+ private static final String UPDATE = "UPDATE run_epics SET run = ?, sequence = ?, timestamp = ?, variable_name = ?, value = ? WHERE run = ? and sequence = ? and variable_name = ?";
+ }
+
+ /**
+ * The database connection.
+ */
+ private final Connection connection;
+
+ /**
+ * Create a new DAO implementation for EPICS data.
+ *
+ * @param connection the database connection
+ */
+ public EpicsDataDaoImpl(final Connection connection) {
+ if (connection == null) {
+ throw new IllegalArgumentException("The connection is null.");
+ }
+ this.connection = connection;
+ }
+
+ /**
+ * Delete the record for this EPICS data object using its run and sequence number.
+ *
+ * @param run the run number
+ * @throws IllegalArgumentException if the EPICS data is missing a header object
+ */
+ @Override
+ public void deleteEpicsData(final EpicsData epicsData) {
+ PreparedStatement preparedStatement = null;
+ try {
+ final EpicsHeader epicsHeader = epicsData.getEpicsHeader();
+ if (epicsHeader == null) {
+ throw new IllegalArgumentException("The EPICS data is missing header information.");
+ }
+ preparedStatement = connection.prepareStatement(EpicsDataQuery.DELETE_RUN_AND_SEQUENCE);
+ preparedStatement.setInt(1, epicsHeader.getRun());
+ preparedStatement.setInt(2, epicsHeader.getSequence());
+ preparedStatement.executeUpdate();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ /**
+ * Delete all EPICS data for a run from the database.
+ *
+ * @param run the run number
+ */
+ @Override
+ public void deleteEpicsData(final int run) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(EpicsDataQuery.DELETE_BY_RUN);
+ preparedStatement.setInt(1, run);
+ preparedStatement.executeUpdate();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ /**
+ * Get all the EPICS data in the database.
+ *
+ * @return the list of EPICS data
+ */
+ @Override
+ public List<EpicsData> getAllEpicsData() {
+ PreparedStatement preparedStatement = null;
+ final List<EpicsData> epicsDataList = new ArrayList<EpicsData>();
+ try {
+ preparedStatement = connection.prepareStatement(EpicsDataQuery.SELECT_ALL);
+ final ResultSet resultSet = preparedStatement.executeQuery();
+ Integer currentRun = null;
+ Integer currentSequence = null;
+ EpicsData epicsData = new EpicsData();
+ while (resultSet.next()) {
+ if (currentRun == null) {
+ currentRun = resultSet.getInt("run");
+ }
+ if (currentSequence == null) {
+ currentSequence = resultSet.getInt("sequence");
+ }
+ final int run = resultSet.getInt("run");
+ final int sequence = resultSet.getInt("sequence");
+ final int timestamp = resultSet.getInt("timestamp");
+ final String variableName = resultSet.getString("variable_name");
+ final double value = resultSet.getDouble("value");
+ if (currentRun != run || currentSequence != sequence) {
+ epicsDataList.add(epicsData);
+ epicsData = new EpicsData();
+ final EpicsHeader epicsHeader = new EpicsHeader(new int[] {run, sequence, timestamp});
+ epicsData.setEpicsHeader(epicsHeader);
+ }
+ epicsData.setValue(variableName, value);
+ }
+ epicsDataList.add(epicsData);
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ return epicsDataList;
+ }
+
+ /**
+ * Get EPICS data by run.
+ *
+ * @param run the run number
+ * @return the EPICS data
+ */
+ @Override
+ public List<EpicsData> getEpicsData(final int run) {
+ PreparedStatement preparedStatement = null;
+ final List<EpicsData> epicsDataList = new ArrayList<EpicsData>();
+ try {
+ preparedStatement = connection.prepareStatement(EpicsDataQuery.SELECT_RUN);
+ preparedStatement.setInt(1, run);
+ final ResultSet resultSet = preparedStatement.executeQuery();
+ Integer currentSequence = null;
+ EpicsData epicsData = new EpicsData();
+ EpicsHeader epicsHeader = null;
+ while (resultSet.next()) {
+
+ // Get record data.
+ final int sequence = resultSet.getInt("sequence");
+ final int timestamp = resultSet.getInt("timestamp");
+ final String variableName = resultSet.getString("variable_name");
+ final double value = resultSet.getDouble("value");
+
+ // Get sequence first time.
+ if (currentSequence == null) {
+ currentSequence = resultSet.getInt("sequence");
+ }
+
+ // Create EPICS header.
+ epicsHeader = new EpicsHeader(new int[] {run, sequence, timestamp});
+
+ // First time need to set header here.
+ if (epicsData.getEpicsHeader() == null) {
+ epicsData.setEpicsHeader(epicsHeader);
+ }
+
+ // New sequence number occurred.
+ if (currentSequence != sequence) {
+
+ // Add the EPICS data to the list.
+ epicsDataList.add(epicsData);
+
+ // Use the new sequence number.
+ currentSequence = sequence;
+
+ // Create new EPICS data.
+ epicsData = new EpicsData();
+
+ // Set header from current record.
+ epicsData.setEpicsHeader(epicsHeader);
+ }
+
+ // Set the value of the variable from the current record.
+ epicsData.setValue(variableName, value);
+ }
+
+ // Add the last object which will not happen inside the loop.
+ epicsDataList.add(epicsData);
+
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ return epicsDataList;
+ }
+
+ /**
+ * Get the list of unique variables names used in the database records.
+ *
+ * @return the list of unique variable names
+ */
+ @Override
+ public List<String> getVariableNames() {
+ final List<String> variableNames = new ArrayList<String>();
+ Statement statement = null;
+ try {
+ statement = connection.createStatement();
+ final ResultSet resultSet = statement.executeQuery(EpicsDataQuery.SELECT_VARIABLE_NAMES);
+ while (resultSet.next()) {
+ variableNames.add(resultSet.getString(1));
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return variableNames;
+ }
+
+ /**
+ * Insert a list of EPICS data into the database.
+ * <p>
+ * The run number comes from the header information.
+ *
+ * @param epicsDataList the list of EPICS data
+ */
+ @Override
+ public void insertEpicsData(final List<EpicsData> epicsDataList) {
+ if (epicsDataList.isEmpty()) {
+ throw new IllegalStateException("The EPICS data list is empty.");
+ }
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(EpicsDataQuery.INSERT);
+ for (final EpicsData epicsData : epicsDataList) {
+ final EpicsHeader epicsHeader = epicsData.getEpicsHeader();
+ if (epicsHeader == null) {
+ throw new IllegalArgumentException("The EPICS data is missing header information.");
+ }
+ for (final String variableName : epicsData.getKeys()) {
+ preparedStatement.setInt(1, epicsData.getEpicsHeader().getRun());
+ preparedStatement.setInt(2, epicsData.getEpicsHeader().getSequence());
+ preparedStatement.setInt(3, epicsData.getEpicsHeader().getTimestamp());
+ preparedStatement.setString(4, variableName);
+ preparedStatement.setDouble(5, epicsData.getValue(variableName));
+ preparedStatement.executeUpdate();
+ }
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ /**
+ * Updates EPICS data in the database.
+ *
+ * @param epicsData the EPICS data to update
+ */
+ @Override
+ public void updateEpicsData(final EpicsData epicsData) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(EpicsDataQuery.UPDATE);
+ final int run = epicsData.getEpicsHeader().getRun();
+ final int sequence = epicsData.getEpicsHeader().getSequence();
+ final int timestamp = epicsData.getEpicsHeader().getTimestamp();
+ for (final String variableName : epicsData.getKeys()) {
+ preparedStatement.setInt(1, run);
+ preparedStatement.setInt(2, sequence);
+ preparedStatement.setInt(3, timestamp);
+ preparedStatement.setString(4, variableName);
+ preparedStatement.setDouble(5, epicsData.getValue(variableName));
+ preparedStatement.setInt(6, run);
+ preparedStatement.setInt(7, sequence);
+ preparedStatement.setString(8, variableName);
+ preparedStatement.executeUpdate();
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ try {
+ if (preparedStatement != null) {
+ preparedStatement.close();
+ }
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ try {
+ connection.setAutoCommit(true);
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/run/EvioFilesDao.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/EvioFilesDao.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/EvioFilesDao.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,43 @@
+package org.hps.record.run;
+
+import java.io.File;
+import java.util.List;
+
+/**
+ * Database Access Object (DAO) interface to EVIO files in the run database.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public interface EvioFilesDao {
+
+ /**
+ * Delete the EVIO file records for a run.
+ *
+ * @param run the run number
+ */
+ void deleteEvioFiles(int run);
+
+ /**
+ * Get all EVIO files from the database.
+ *
+ * @return all EVIO files from the database
+ */
+ List<File> getAllEvioFiles();
+
+ /**
+ * Get a list of EVIO files by run number.
+ *
+ * @param run the run number
+ * @return the list of EVIO files for the run
+ */
+ List<File> getEvioFiles(int run);
+
+ /**
+ * Insert the list of files for a run.
+ *
+ * @param fileList the list of files
+ * @param run the run number
+ */
+ void insertEvioFiles(List<File> fileList, int run);
+
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/run/EvioFilesDaoImpl.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/EvioFilesDaoImpl.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/EvioFilesDaoImpl.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,168 @@
+package org.hps.record.run;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Implementation of database operations for EVIO files in the run database.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public class EvioFilesDaoImpl implements EvioFilesDao {
+
+ /**
+ * SQL query strings.
+ */
+ private static final class EvioFilesQuery {
+
+ /**
+ * Delete files by run number.
+ */
+ private static final String DELETE_RUN = "DELETE FROM run_files where run = ?";
+ /**
+ * Insert files by run number.
+ */
+ private static final String INSERT_RUN = "INSERT INTO run_files (run, directory, name) VALUES(?, ?, ?)";
+ /**
+ * Select all records.
+ */
+ private static final String SELECT_ALL = "SELECT directory, name FROM run_files";
+ /**
+ * Select records by run number.
+ */
+ private static final String SELECT_RUN = "SELECT directory, name FROM run_files WHERE run = ?";
+ }
+
+ /**
+ * The database connection.
+ */
+ private final Connection connection;
+
+ public EvioFilesDaoImpl(final Connection connection) {
+ this.connection = connection;
+ }
+
+ /**
+ * Delete the EVIO file records for a run.
+ *
+ * @param run the run number
+ */
+ @Override
+ public void deleteEvioFiles(final int run) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(EvioFilesQuery.DELETE_RUN);
+ preparedStatement.setInt(1, run);
+ preparedStatement.executeUpdate();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Get all EVIO files from the database.
+ *
+ * @return all EVIO files from the database
+ */
+ @Override
+ public List<File> getAllEvioFiles() {
+ Statement statement = null;
+ final List<File> evioFileList = new ArrayList<File>();
+ try {
+ statement = this.connection.createStatement();
+ final ResultSet resultSet = statement.executeQuery(EvioFilesQuery.SELECT_ALL);
+ while (resultSet.next()) {
+ evioFileList.add(new File(resultSet.getString("directory") + File.separator
+ + resultSet.getString("name")));
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return evioFileList;
+ }
+
+ /**
+ * Get a list of EVIO files by run number.
+ *
+ * @param run the run number
+ * @return the list of EVIO files for the run
+ */
+ @Override
+ public List<File> getEvioFiles(final int run) {
+ PreparedStatement preparedStatement = null;
+ final List<File> evioFileList = new ArrayList<File>();
+ try {
+ preparedStatement = this.connection.prepareStatement(EvioFilesQuery.SELECT_RUN);
+ preparedStatement.setInt(1, run);
+ final ResultSet resultSet = preparedStatement.executeQuery();
+ while (resultSet.next()) {
+ evioFileList.add(new File(resultSet.getString("directory") + File.separator
+ + resultSet.getString("name")));
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return evioFileList;
+ }
+
+ /**
+ * Insert the list of files for a run.
+ *
+ * @param fileList the list of files
+ * @param run the run number
+ */
+ @Override
+ public void insertEvioFiles(final List<File> fileList, final int run) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(EvioFilesQuery.INSERT_RUN);
+ for (final File file : fileList) {
+ preparedStatement.setInt(1, run);
+ preparedStatement.setString(2, file.getParentFile().getPath());
+ preparedStatement.setString(3, file.getName());
+ preparedStatement.executeUpdate();
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
Modified: java/trunk/record-util/src/main/java/org/hps/record/run/RunManager.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/RunManager.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/RunManager.java Thu Aug 20 12:50:12 2015
@@ -1,9 +1,8 @@
package org.hps.record.run;
import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -14,19 +13,14 @@
import org.lcsim.util.log.LogUtil;
/**
- * Manages read-only access to the run database and creates a {@link RunSummary} object from the data for a specific
- * run.
- * <p>
- * This class converts database records into {@link RunSummary}, {@link org.hps.record.epics.EpicsData},
- * {@link org.hps.record.scalers.ScalerData}, and {@link org.hps.record.evio.crawler.EvioFileList} objects using their
- * corresponding {@link AbstractRunDatabaseReader} implementation classes.
+ * Manages read-only access to the run database and creates a {@link RunSummary} for a specific run.
*
* @author Jeremy McCormick, SLAC
*/
public final class RunManager implements ConditionsListener {
/**
- * The default connection parameters for read-only access to the run database using the standard 'hpsuser' account.
+ * The default connection parameters for read-only access to the run database.
*/
private static ConnectionParameters DEFAULT_CONNECTION_PARAMETERS = new ConnectionParameters("hpsuser",
"darkphoton", "hps_run_db", "hpsdb.jlab.org");
@@ -61,7 +55,7 @@
/**
* The database connection parameters, initially set to the default parameters.
*/
- private ConnectionParameters connectionParameters = DEFAULT_CONNECTION_PARAMETERS;
+ private final ConnectionParameters connectionParameters = DEFAULT_CONNECTION_PARAMETERS;
/**
* The run number; the -1 value indicates that this has not been set externally yet.
@@ -73,11 +67,18 @@
*/
private RunSummary runSummary = null;
- void closeConnection() {
- try {
- this.connection.close();
- } catch (final SQLException e) {
- e.printStackTrace();
+ /**
+ * Close the database connection.
+ */
+ private void closeConnection() {
+ if (!(this.connection == null)) {
+ try {
+ if (!this.connection.isClosed()) {
+ this.connection.close();
+ }
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
}
}
@@ -90,21 +91,21 @@
}
/**
- * Get the database connection.
- *
- * @return the database connection or <code>null</code> if it is not set
- */
- Connection getConnection() {
- return this.connection;
- }
-
- /**
- * Get the run number.
+ * Get the current run number.
*
* @return the run number
*/
public int getRun() {
return run;
+ }
+
+ /**
+ * Get the complete list of run numbers from the database.
+ *
+ * @return the complete list of run numbers
+ */
+ List<Integer> getRuns() {
+ return new RunSummaryDaoImpl(this.connection).getRuns();
}
/**
@@ -117,122 +118,70 @@
}
/**
- * Read information from the run database and create a {@link RunSummary} from it.
+ * Open a new database connection from the connection parameters if the current one is closed or <code>null</code>.
+ * <p>
+ * This method does nothing if the connection is already open.
*/
- private void readRun() {
-
- // Read main RunSummary object but not objects that it references.
- final RunSummaryReader runSummaryReader = new RunSummaryReader();
- runSummaryReader.setRun(this.getRun());
- runSummaryReader.setConnection(this.getConnection());
- runSummaryReader.read();
- this.setRunSummary(runSummaryReader.getData());
-
- // Read EpicsData and set on RunSummary.
- final EpicsDataReader epicsDataReader = new EpicsDataReader();
- epicsDataReader.setRun(this.getRun());
- epicsDataReader.setConnection(this.getConnection());
- epicsDataReader.read();
- this.getRunSummary().setEpicsData(epicsDataReader.getData());
-
- // Read ScalerData and set on RunSummary.
- final ScalerDataReader scalerDataReader = new ScalerDataReader();
- scalerDataReader.setRun(this.getRun());
- scalerDataReader.setConnection(this.getConnection());
- scalerDataReader.read();
- this.getRunSummary().setScalerData(scalerDataReader.getData());
-
- // Read ScalerData and set on RunSummary.
- final EvioFileListReader evioFileListReader = new EvioFileListReader();
- evioFileListReader.setRun(this.getRun());
- evioFileListReader.setConnection(this.getConnection());
- evioFileListReader.read();
- this.getRunSummary().setEvioFileList(evioFileListReader.getData());
+ private void openConnection() {
+ try {
+ if (this.connection == null || this.connection.isClosed()) {
+ LOGGER.info("creating database connection");
+ this.connection = connectionParameters.createConnection();
+ } else {
+ LOGGER.warning("connection already open");
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException("Error opening database connection.", e);
+ }
}
/**
- * Check if the current run number exists in the run database.
+ * Set the connection externally if using a database other than the default one at JLAB.
*
- * @return <code>true</code> if run exists
+ * @param connection the database connection
*/
- private boolean runExists() throws SQLException {
- PreparedStatement statement = null;
- boolean exists = false;
- try {
- statement = connection.prepareStatement("SELECT run FROM runs where run = ?");
- statement.setInt(1, this.run);
- final ResultSet resultSet = statement.executeQuery();
- exists = resultSet.next();
- } finally {
- if (statement != null) {
- try {
- statement.close();
- } catch (final SQLException e) {
- e.printStackTrace();
- }
- }
- }
- return exists;
+ public void setConnection(final Connection connection) {
+ this.connection = connection;
}
/**
- * Set the database connection parameters.
- */
- public void setConnectionParameters(final ConnectionParameters connectionParameters) {
- this.connectionParameters = connectionParameters;
- }
-
- /**
- * Set the run number and load the applicable {@link RunSummary} from the db.
+ * Set the run number and then load the applicable {@link RunSummary} from the database.
*
* @param run the run number
*/
- synchronized void setRun(final int run) {
+ public synchronized void setRun(final int run) {
+ // Check if run number is valid.
if (run < 0) {
throw new IllegalArgumentException("invalid run number: " + run);
}
- try {
+ // Setup the database connection.
+ this.openConnection();
- // Setup the database connection.
- if (this.connection == null || this.connection.isClosed()) {
- this.connection = connectionParameters.createConnection();
+ // Initialize database interface.
+ final RunSummaryDao runSummaryDao = new RunSummaryDaoImpl(this.connection);
+
+ // Set the current run number.
+ this.run = run;
+
+ // Does the current run exist in the database?
+ if (runSummaryDao.runSummaryExists(this.getRun())) {
+ LOGGER.info("run " + run + " found in database");
+ try {
+ // Read the records from the database and convert into complex Java object.
+ this.runSummary = runSummaryDao.readFullRunSummary(this.getRun());
+ } catch (final Exception e) {
+ // There was some unknown error when reading in the run records.
+ LOGGER.log(Level.SEVERE, "Error reading from run database.", e);
+ throw new RuntimeException(e);
}
+ } else {
+ // Run is not in the database.
+ LOGGER.warning("run database record does not exist for run " + run);
+ }
- // Set the current run number.
- this.run = run;
-
- // Does the current run exist in the database?
- if (this.runExists()) {
- LOGGER.info("run record found in hps_run_db for " + run);
- try {
- // Read the records from the database and convert into Java objects.
- this.readRun();
- } catch (final Exception e) {
- // There was some unknown error when reading in the run records.
- LOGGER.log(Level.SEVERE, "Error reading from run database for run: " + run, e);
- throw new RuntimeException(e);
- }
- } else {
- // Run is not in the database.
- LOGGER.warning("run database record does not exist for run " + run);
- }
-
- // Close the database connection.
- this.connection.close();
-
- } catch (final SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Set the current {@link RunSummary}.
- *
- * @param runSummary the current {@link RunSummary}
- */
- void setRunSummary(final RunSummary runSummary) {
- this.runSummary = runSummary;
+ // Close the database connection.
+ this.closeConnection();
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/run/RunSummary.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/RunSummary.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/RunSummary.java Thu Aug 20 12:50:12 2015
@@ -4,13 +4,16 @@
import java.io.PrintStream;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
+import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import org.hps.record.epics.EpicsData;
-import org.hps.record.evio.crawler.EvioFileList;
+import org.hps.record.evio.EvioFileSequenceComparator;
import org.hps.record.scalers.ScalerData;
/**
@@ -26,11 +29,10 @@
* <li>number of EVIO files in the run</li>
* <li>whether the END event was found indicating that the DAQ did not crash</li>
* <li>whether the run is considered good (all <code>true</code> for now)</li>
- * <li>list of EVIO files in the run</li>
* </ul>
- * There are also associated {@link org.hps.record.epics.EpicsData} and {@link org.hps.record.scalers.ScalerData}
- * objects representing the EPICS and scaler data summaries for the run. The EPICS data is averaged over the whole run
- * while the scalers are from the last scaler data bank that was found.
+ * <p>
+ * It also references several complex objects including lists of {@link org.hps.record.epics.EpicsData} and
+ * {@link org.hps.record.scalers.ScalerData} for the run, as well as a list of EVIO files.
*
* @author Jeremy McCormick, SLAC
*/
@@ -48,6 +50,9 @@
DATE_DISPLAY.setCalendar(new GregorianCalendar(TimeZone.getTimeZone("America/New_York")));
}
+ /**
+ * Date this record was created.
+ */
private Date created;
/**
@@ -61,9 +66,9 @@
private long endTimeUtc;
/**
- * The combined EPICS information for the run (uses the mean values for each variable).
- */
- private EpicsData epics;
+ * The EPICS data from the run.
+ */
+ private List<EpicsData> epicsDataList;
/**
* The counts of different types of events that were found.
@@ -73,7 +78,7 @@
/**
* The list of EVIO files in the run.
*/
- private EvioFileList evioFileList = new EvioFileList();
+ private List<File> evioFileList = new ArrayList<File>();
/**
* The run number.
@@ -86,9 +91,9 @@
private boolean runOkay = true;
/**
- * The scaler data from the last physics event in the run.
- */
- private ScalerData scalerData;
+ * The scaler data for the run.
+ */
+ private List<ScalerData> scalerDataList;
/**
* The run start time in UTC (milliseconds).
@@ -165,14 +170,12 @@
}
/**
- * Get the EPICS data summary.
- * <p>
- * This is computed by taking the mean of each variable for the run.
- *
- * @return the EPICS data summary
- */
- public EpicsData getEpicsData() {
- return this.epics;
+ * Get the EPICS data from the run.
+ *
+ * @return the EPICS data from the run
+ */
+ public List<EpicsData> getEpicsDataSet() {
+ return this.epicsDataList;
}
/**
@@ -202,7 +205,7 @@
*
* @return the list of EVIO files in this run
*/
- public EvioFileList getEvioFileList() {
+ public List<File> getEvioFileList() {
return this.evioFileList;
}
@@ -225,12 +228,12 @@
}
/**
- * Get the scaler data of this run (last event only).
- *
- * @return the scaler data of this run from the last event
- */
- public ScalerData getScalerData() {
- return this.scalerData;
+ * Get the scaler data of this run.
+ *
+ * @return the scaler data of this run
+ */
+ public List<ScalerData> getScalerData() {
+ return this.scalerDataList;
}
/**
@@ -295,8 +298,8 @@
public void printOut(final PrintStream ps) {
ps.println("--------------------------------------------");
ps.println("run: " + this.run);
- ps.println("first file: " + this.evioFileList.first());
- ps.println("last file: " + this.evioFileList.last());
+ ps.println("first file: " + this.evioFileList.get(0));
+ ps.println("last file: " + this.evioFileList.get(evioFileList.size() - 1));
ps.println("started: " + DATE_DISPLAY.format(this.getStartDate()));
ps.println("ended: " + DATE_DISPLAY.format(this.getEndDate()));
ps.println("total events: " + this.getTotalEvents());
@@ -333,7 +336,7 @@
/**
* Set the end date.
*
- * @param endDate the end date
+ * @param endTimeUtc the end date
*/
public void setEndTimeUtc(final long endTimeUtc) {
this.endTimeUtc = endTimeUtc;
@@ -344,8 +347,8 @@
*
* @param epics the EPICS data for the run
*/
- public void setEpicsData(final EpicsData epics) {
- this.epics = epics;
+ public void setEpicsData(final List<EpicsData> epicsDataList) {
+ this.epicsDataList = epicsDataList;
}
/**
@@ -362,7 +365,7 @@
*
* @param evioFileList the list of EVIO files for the run
*/
- public void setEvioFileList(final EvioFileList evioFileList) {
+ public void setEvioFileList(final List<File> evioFileList) {
this.evioFileList = evioFileList;
}
@@ -380,14 +383,14 @@
*
* @param scalerData the scaler data
*/
- public void setScalerData(final ScalerData scalerData) {
- this.scalerData = scalerData;
+ public void setScalerData(final List<ScalerData> scalerDataList) {
+ this.scalerDataList = scalerDataList;
}
/**
* Set the start date of the run.
*
- * @param startDate the start date of the run
+ * @param startTimeUtc the start date of the run
*/
public void setStartTimeUtc(final long startTimeUtc) {
this.startTimeUtc = startTimeUtc;
@@ -424,7 +427,7 @@
* Sort the files in the run by sequence number in place.
*/
public void sortFiles() {
- this.evioFileList.sort();
+ Collections.sort(this.evioFileList, new EvioFileSequenceComparator());
}
/**
Added: java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryDao.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryDao.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryDao.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,99 @@
+package org.hps.record.run;
+
+import java.util.List;
+
+/**
+ * Data Access Object (DAO) API for managing run summary information in the run database.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public interface RunSummaryDao {
+
+ /**
+ * Delete a run summary from the database including its referenced objects such as EPICS data.
+ *
+ * @param runSummary the run summary to delete
+ */
+ void deleteFullRunSummary(RunSummary runSummary);
+
+ /**
+ * Delete a run summary by run number.
+ *
+ * @param run the run number
+ */
+ void deleteRunSummary(int run);
+
+ /**
+ * Delete a run summary but not its objects.
+ *
+ * @param runSummary the run summary object
+ */
+ void deleteRunSummary(RunSummary runSummary);
+
+ /**
+ * Get the list of run numbers.
+ *
+ * @return the list of run numbers
+ */
+ List<Integer> getRuns();
+
+ /**
+ * Get a list of run summaries without loading their objects such as EPICS data.
+ *
+ * @return the list of run summaries
+ */
+ List<RunSummary> getRunSummaries();
+
+ /**
+ * Get a run summary by run number without loading object state.
+ *
+ * @param run the run number
+ * @return the run summary object
+ */
+ RunSummary getRunSummary(int run);
+
+ /**
+ * Insert a list of run summaries along with its referenced objects such as scaler and EPICS data.
+ *
+ * @param runSummaryList the list of run summaries
+ * @param deleteExisting <code>true</code> to allow deletion and replacement of existing run summaries
+ */
+ void insertFullRunSummaries(List<RunSummary> runSummaryList, boolean deleteExisting);
+
+ /**
+ * Insert a run summary including all its objects.
+ *
+ * @param runSummary the run summary object
+ */
+ void insertFullRunSummary(RunSummary runSummary);
+
+ /**
+ * Insert a run summary but not its objects.
+ *
+ * @param runSummary the run summary object
+ */
+ void insertRunSummary(RunSummary runSummary);
+
+ /**
+ * Read a run summary and its objects such as scaler data.
+ *
+ * @param run the run number
+ * @return the full run summary
+ */
+ RunSummary readFullRunSummary(int run);
+
+ /**
+ * Return <code>true</code> if a run summary exists in the database.
+ *
+ * @param run the run number
+ * @return <code>true</code> if <code>run</code> exists in the database
+ */
+ boolean runSummaryExists(int run);
+
+ /**
+ * Update a run summary but not its objects.
+ *
+ * @param runSummary the run summary to update
+ */
+ void updateRunSummary(RunSummary runSummary);
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryDaoImpl.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryDaoImpl.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryDaoImpl.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,488 @@
+package org.hps.record.run;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.lcsim.util.log.DefaultLogFormatter;
+import org.lcsim.util.log.LogUtil;
+
+/**
+ * Implementation of database operations for {@link RunSummary} objects in the run database.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public class RunSummaryDaoImpl implements RunSummaryDao {
+
+ /**
+ * SQL query strings.
+ */
+ private static final class RunSummaryQuery {
+
+ /**
+ * Delete by run number.
+ */
+ private static final String DELETE_RUN = "DELETE FROM runs WHERE run = ?";
+ /**
+ * Insert a record for a run.
+ */
+ private static final String INSERT = "INSERT INTO runs (run, start_time_utc, end_time_utc, nevents, nfiles, end_ok, created) VALUES(?, ?, ?, ?, ?, ?, NOW())";
+ /**
+ * Select all records.
+ */
+ private static final String SELECT_ALL = "SELECT * from runs";
+ /**
+ * Select record by run number.
+ */
+ private static final String SELECT_RUN = "SELECT run, start_time_utc, end_time_utc, nevents, nfiles, end_ok, run_ok, updated, created FROM runs WHERE run = ?";
+ /**
+ * Update information for a run.
+ */
+ private static final String UPDATE_RUN = "UPDATE runs SET start_time_utc, end_time_utc, nevents, nfiles, end_ok, run_ok WHERE run = ?";
+ }
+
+ /**
+ * Setup class logging.
+ */
+ private static final Logger LOGGER = LogUtil.create(RunSummaryDaoImpl.class, new DefaultLogFormatter(), Level.ALL);
+
+ /**
+ * The database connection.
+ */
+ private final Connection connection;
+
+ /**
+ * The database API for EPICS data.
+ */
+ private EpicsDataDao epicsDataDao = null;
+
+ /**
+ * The database API for EVIO file information.
+ */
+ private EvioFilesDao evioFilesDao = null;
+
+ /**
+ * The database API for scaler data.
+ */
+ private ScalerDataDao scalerDataDao = null;
+
+ /**
+ * Create a new DAO object for run summary information.
+ *
+ * @param connection the database connection
+ */
+ public RunSummaryDaoImpl(final Connection connection) {
+ // Set the connection.
+ if (connection == null) {
+ throw new IllegalArgumentException("The connection is null.");
+ }
+ this.connection = connection;
+
+ // Setup DAO API objects for managing complex object state.
+ epicsDataDao = new EpicsDataDaoImpl(this.connection);
+ scalerDataDao = new ScalerDataDaoImpl(this.connection);
+ evioFilesDao = new EvioFilesDaoImpl(this.connection);
+ }
+
+ /**
+ * Delete a run summary from the database including its referenced objects such as EPICS data.
+ *
+ * @param runSummary the run summary to delete
+ */
+ @Override
+ public void deleteFullRunSummary(final RunSummary runSummary) {
+ // Delete EPICS log.
+ epicsDataDao.deleteEpicsData(runSummary.getRun());
+
+ // Delete scaler data.
+ scalerDataDao.deleteScalerData(runSummary.getRun());
+
+ // Delete file list.
+ evioFilesDao.deleteEvioFiles(runSummary.getRun());
+
+ // Finally delete the run summary information.
+ this.deleteRunSummary(runSummary.getRun());
+ }
+
+ /**
+ * Delete a run summary by run number.
+ *
+ * @param run the run number
+ */
+ @Override
+ public void deleteRunSummary(final int run) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(RunSummaryQuery.DELETE_RUN);
+ preparedStatement.setInt(1, run);
+ preparedStatement.executeUpdate();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ /**
+ * Delete a run summary but not its objects.
+ *
+ * @param runSummary the run summary object
+ */
+ @Override
+ public void deleteRunSummary(final RunSummary runSummary) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(RunSummaryQuery.DELETE_RUN);
+ preparedStatement.setInt(1, runSummary.getRun());
+ preparedStatement.executeUpdate();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ /**
+ * Get the list of run numbers.
+ *
+ * @return the list of run numbers
+ */
+ @Override
+ public List<Integer> getRuns() {
+ final List<Integer> runs = new ArrayList<Integer>();
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = this.connection.prepareStatement("SELECT distinct(run) FROM runs ORDER BY run");
+ final ResultSet resultSet = preparedStatement.executeQuery();
+ while (resultSet.next()) {
+ final Integer run = resultSet.getInt(1);
+ runs.add(run);
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return runs;
+ }
+
+ /**
+ * Get a list of run summaries without loading their objects such as EPICS data.
+ *
+ * @return the list of run summaries
+ */
+ @Override
+ public List<RunSummary> getRunSummaries() {
+ PreparedStatement statement = null;
+ final List<RunSummary> runSummaries = new ArrayList<RunSummary>();
+ try {
+ statement = this.connection.prepareStatement(RunSummaryQuery.SELECT_ALL);
+ final ResultSet resultSet = statement.executeQuery();
+ while (resultSet.next()) {
+ final RunSummary runSummary = new RunSummary(resultSet.getInt("run"));
+ runSummary.setStartTimeUtc(resultSet.getLong("start_time_utc"));
+ runSummary.setEndTimeUtc(resultSet.getLong("end_time_utc"));
+ runSummary.setTotalEvents(resultSet.getInt("nevents"));
+ runSummary.setTotalFiles(resultSet.getInt("nfiles"));
+ runSummary.setEndOkay(resultSet.getBoolean("end_ok"));
+ runSummary.setRunOkay(resultSet.getBoolean("run_ok"));
+ runSummary.setUpdated(resultSet.getTimestamp("updated"));
+ runSummary.setCreated(resultSet.getTimestamp("created"));
+ runSummaries.add(runSummary);
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return runSummaries;
+ }
+
+ /**
+ * Get a run summary by run number without loading object state.
+ *
+ * @param run the run number
+ * @return the run summary object
+ */
+ @Override
+ public RunSummary getRunSummary(final int run) {
+ PreparedStatement statement = null;
+ RunSummary runSummary = null;
+ try {
+ statement = this.connection.prepareStatement(RunSummaryQuery.SELECT_RUN);
+ statement.setInt(1, run);
+ final ResultSet resultSet = statement.executeQuery();
+ if (!resultSet.next()) {
+ throw new IllegalArgumentException("No record exists for run " + run + " in database.");
+ }
+
+ runSummary = new RunSummary(run);
+ runSummary.setStartTimeUtc(resultSet.getLong("start_time_utc"));
+ runSummary.setEndTimeUtc(resultSet.getLong("end_time_utc"));
+ runSummary.setTotalEvents(resultSet.getInt("nevents"));
+ runSummary.setTotalFiles(resultSet.getInt("nfiles"));
+ runSummary.setEndOkay(resultSet.getBoolean("end_ok"));
+ runSummary.setRunOkay(resultSet.getBoolean("run_ok"));
+ runSummary.setUpdated(resultSet.getTimestamp("updated"));
+ runSummary.setCreated(resultSet.getTimestamp("created"));
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return runSummary;
+ }
+
+ /**
+ * Insert a list of run summaries along with their complex state such as referenced scaler and EPICS data.
+ *
+ * @param runSummaryList the list of run summaries
+ * @param deleteExisting <code>true</code> to allow deletion and replacement of existing run summaries
+ */
+ @Override
+ public void insertFullRunSummaries(final List<RunSummary> runSummaryList, final boolean deleteExisting) {
+
+ if (runSummaryList == null) {
+ throw new IllegalArgumentException("The run summary list is null.");
+ }
+ if (runSummaryList.isEmpty()) {
+ throw new IllegalArgumentException("The run summary list is empty.");
+ }
+
+ LOGGER.info("inserting " + runSummaryList.size() + " run summaries into database");
+
+ // Turn off auto commit.
+ try {
+ LOGGER.info("turning off auto commit");
+ this.connection.setAutoCommit(false);
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ }
+
+ // Loop over all runs found while crawling.
+ for (final RunSummary runSummary : runSummaryList) {
+
+ final int run = runSummary.getRun();
+
+ LOGGER.info("inserting run summary for run " + run + " into database");
+
+ // Does the run exist in the database already?
+ if (this.runSummaryExists(run)) {
+ // Is deleting existing rows allowed?
+ if (deleteExisting) {
+ LOGGER.info("deleting existing run summary");
+ // Delete the existing rows.
+ this.deleteFullRunSummary(runSummary);
+ } else {
+ // Rows exist but updating is disallowed which is a fatal error.
+ throw new IllegalStateException("Run " + runSummary.getRun()
+ + " already exists and updates are disallowed.");
+ }
+ }
+
+ // Insert full run summary information including sub-objects.
+ LOGGER.info("inserting run summary");
+ this.insertFullRunSummary(runSummary);
+ LOGGER.info("run summary for " + run + " inserted successfully");
+
+ try {
+ // Commit the transaction for the run.
+ LOGGER.info("committing transaction");
+ this.connection.commit();
+ } catch (final SQLException e1) {
+ try {
+ LOGGER.severe("rolling back transaction");
+ // Rollback the transaction if there was an error.
+ this.connection.rollback();
+ } catch (final SQLException e2) {
+ throw new RuntimeException(e2);
+ }
+ }
+
+ LOGGER.info("done inserting run summary " + run);
+
+ LOGGER.getHandlers()[0].flush();
+ }
+
+ try {
+ LOGGER.info("turning auto commit on");
+ // Turn auto commit back on.
+ this.connection.setAutoCommit(true);
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+
+ LOGGER.info("done inserting run summaries");
+ }
+
+ /**
+ * Insert a run summary including all its objects.
+ *
+ * @param runSummary the run summary object to insert
+ */
+ @Override
+ public void insertFullRunSummary(final RunSummary runSummary) {
+
+ // Insert basic run log info.
+ this.insertRunSummary(runSummary);
+
+ // Insert list of files.
+ LOGGER.info("inserting EVIO " + runSummary.getEvioFileList().size() + " files");
+ evioFilesDao.insertEvioFiles(runSummary.getEvioFileList(), runSummary.getRun());
+
+ // Insert EPICS data.
+ LOGGER.info("inserting " + runSummary.getEpicsDataSet().size() + " EPICS records");
+ epicsDataDao.insertEpicsData(runSummary.getEpicsDataSet());
+
+ // Insert scaler data.
+ LOGGER.info("inserting " + runSummary.getScalerData().size() + " scaler data records");
+ scalerDataDao.insertScalerData(runSummary.getScalerData(), runSummary.getRun());
+ }
+
+ /**
+ * Insert a run summary but not its objects.
+ *
+ * @param runSummary the run summary object
+ */
+ @Override
+ public void insertRunSummary(final RunSummary runSummary) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(RunSummaryQuery.INSERT);
+ preparedStatement.setInt(1, runSummary.getRun());
+ preparedStatement.setLong(2, runSummary.getStartTimeUtc());
+ preparedStatement.setLong(3, runSummary.getEndTimeUtc());
+ preparedStatement.setInt(4, runSummary.getTotalEvents());
+ preparedStatement.setInt(5, runSummary.getEvioFileList().size());
+ preparedStatement.setBoolean(6, runSummary.getEndOkay());
+ preparedStatement.executeUpdate();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ /**
+ * Read a run summary and its objects such as scaler data.
+ *
+ * @param run the run number
+ * @return the full run summary
+ */
+ @Override
+ public RunSummary readFullRunSummary(final int run) {
+
+ // Read main run summary but not referenced objects.
+ final RunSummary runSummary = this.getRunSummary(run);
+
+ // Read EPICS data and set on RunSummary.
+ runSummary.setEpicsData(epicsDataDao.getEpicsData(run));
+
+ // Read scaler data and set on RunSummary.
+ runSummary.setScalerData(scalerDataDao.getScalerData(run));
+
+ // Read EVIO file list and set on RunSummary.
+ runSummary.setEvioFileList(evioFilesDao.getEvioFiles(run));
+
+ return runSummary;
+ }
+
+ /**
+ * Return <code>true</code> if a run summary exists in the database for the run number.
+ *
+ * @param run the run number
+ * @return <code>true</code> if run exists in the database
+ */
+ @Override
+ public boolean runSummaryExists(final int run) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement("SELECT run FROM runs where run = ?");
+ preparedStatement.setInt(1, run);
+ final ResultSet rs = preparedStatement.executeQuery();
+ return rs.first();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ /**
+ * Update a run summary but not its complex state.
+ *
+ * @param runSummary the run summary to update
+ */
+ @Override
+ public void updateRunSummary(final RunSummary runSummary) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(RunSummaryQuery.UPDATE_RUN);
+ preparedStatement.setLong(1, runSummary.getStartTimeUtc());
+ preparedStatement.setLong(2, runSummary.getEndTimeUtc());
+ preparedStatement.setInt(3, runSummary.getTotalEvents());
+ preparedStatement.setInt(4, runSummary.getEvioFileList().size());
+ preparedStatement.setBoolean(5, runSummary.getEndOkay());
+ preparedStatement.setBoolean(6, runSummary.getRunOkay());
+ preparedStatement.setInt(7, runSummary.getRun());
+ preparedStatement.executeUpdate();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataDao.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataDao.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataDao.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,36 @@
+package org.hps.record.run;
+
+import java.util.List;
+
+import org.hps.record.scalers.ScalerData;
+
+/**
+ * Database Access Object (DAO) for scaler data in the run database.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public interface ScalerDataDao {
+
+ /**
+ * Delete scaler data for the run.
+ *
+ * @param run the run number
+ */
+ void deleteScalerData(int run);
+
+ /**
+ * Get scaler data for a run.
+ *
+ * @param run the run number
+ * @return the scaler data for the run
+ */
+ List<ScalerData> getScalerData(int run);
+
+ /**
+ * Insert scaler data for a run.
+ *
+ * @param scalerData the list of scaler data
+ * @param run the run number
+ */
+ void insertScalerData(List<ScalerData> scalerData, int run);
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataDaoImpl.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataDaoImpl.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataDaoImpl.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,175 @@
+package org.hps.record.run;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.hps.record.scalers.ScalerData;
+
+/**
+ * Implementation of database API for {@link org.hps.record.scalers.ScalerData} in the run database.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
+public class ScalerDataDaoImpl implements ScalerDataDao {
+
+ /**
+ * SQL query strings.
+ */
+ private static final class ScalerDataQuery {
+
+ /**
+ * Delete by run.
+ */
+ private static final String DELETE_RUN = "DELETE FROM run_scalers WHERE run = ?";
+ /**
+ * Insert a record.
+ */
+ private static final String INSERT = "INSERT INTO run_scalers (run, event, idx, value) VALUES (?, ?, ?, ?)";
+ /**
+ * Select by run.
+ */
+ private static final String SELECT_RUN = "SELECT event, idx, value FROM run_scalers WHERE run = ? ORDER BY event, idx";
+ }
+
+ /**
+ * The database connection.
+ */
+ private final Connection connection;
+
+ /**
+ * Create object for managing scaler data in the run database.
+ *
+ * @param connection the database connection
+ */
+ public ScalerDataDaoImpl(final Connection connection) {
+ if (connection == null) {
+ throw new IllegalArgumentException("The connection is null.");
+ }
+ this.connection = connection;
+ }
+
+ /**
+ * Delete scaler data for the run.
+ *
+ * @param run the run number
+ */
+ @Override
+ public void deleteScalerData(final int run) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = connection.prepareStatement(ScalerDataQuery.DELETE_RUN);
+ preparedStatement.setInt(1, run);
+ preparedStatement.executeUpdate();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ /**
+ * Get scaler data for a run.
+ *
+ * @param run the run number
+ * @return the scaler data for the run
+ */
+ @Override
+ public List<ScalerData> getScalerData(final int run) {
+ PreparedStatement preparedStatement = null;
+ final List<ScalerData> scalerDataList = new ArrayList<ScalerData>();
+ try {
+ preparedStatement = this.connection.prepareStatement(ScalerDataQuery.SELECT_RUN);
+ preparedStatement.setInt(1, run);
+ final ResultSet resultSet = preparedStatement.executeQuery();
+
+ int[] scalerArray = new int[ScalerData.ARRAY_SIZE];
+ int event = 0;
+
+ while (resultSet.next()) {
+
+ // Get record data.
+ event = resultSet.getInt("event");
+ final int idx = resultSet.getInt("idx");
+ final int value = resultSet.getInt("value");
+
+ // Is this the start of a new scaler data set and not the first one?
+ if (idx == 0 && resultSet.getRow() > 1) {
+ // Create new scaler data object and add to list.
+ final ScalerData scalerData = new ScalerData(scalerArray, event);
+ scalerDataList.add(scalerData);
+
+ // Reset the data array for next object.
+ scalerArray = new int[ScalerData.ARRAY_SIZE];
+ }
+
+ // Set value by index.
+ scalerArray[idx] = value;
+ }
+
+ // Add the last object which will not happen inside the loop.
+ if (scalerArray != null) {
+ scalerDataList.add(new ScalerData(scalerArray, event));
+ }
+
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return scalerDataList;
+ }
+
+ /**
+ * Insert scaler data for a run.
+ *
+ * @param scalerData the list of scaler data
+ * @param run the run number
+ */
+ @Override
+ public void insertScalerData(final List<ScalerData> scalerDataList, final int run) {
+ PreparedStatement preparedStatement = null;
+ try {
+ preparedStatement = this.connection.prepareStatement(ScalerDataQuery.INSERT);
+ for (final ScalerData scalerData : scalerDataList) {
+ final int size = scalerData.size();
+ final Integer event = scalerData.getEventId();
+ if (event == null) {
+ throw new IllegalStateException("The scaler data is missing the event ID.");
+ }
+ for (int i = 0; i < size; i++) {
+ preparedStatement.setInt(1, run);
+ preparedStatement.setInt(2, event);
+ preparedStatement.setInt(3, i);
+ preparedStatement.setInt(4, scalerData.getValue(i));
+ preparedStatement.executeUpdate();
+ }
+ }
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (preparedStatement != null) {
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
Added: java/trunk/record-util/src/main/java/org/hps/record/run/package-info.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/run/package-info.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/run/package-info.java Thu Aug 20 12:50:12 2015
@@ -0,0 +1,4 @@
+/**
+ * API for accessing the HPS run database with run summary information.
+ */
+package org.hps.record.run;
Modified: java/trunk/record-util/src/main/java/org/hps/record/scalers/ScalerData.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/scalers/ScalerData.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/scalers/ScalerData.java Thu Aug 20 12:50:12 2015
@@ -13,6 +13,11 @@
* @author <a href="mailto:[log in to unmask]">Jeremy McCormick</a>
*/
public final class ScalerData {
+
+ /**
+ * Fixed array size of scaler data in the EVIO bank.
+ */
+ public static final int ARRAY_SIZE = 72;
/**
* Default name of scaler data collection in LCSim events.
@@ -38,10 +43,11 @@
public static ScalerData read(final EventHeader event, final String collectionName) {
ScalerData data = null;
if (event.hasCollection(GenericObject.class, collectionName)) {
- //System.out.println("ScalerData - found collection");
+ // System.out.println("ScalerData - found collection");
final List<GenericObject> objects = event.get(GenericObject.class, collectionName);
data = new ScalerData();
data.fromGenericObject(objects.get(0));
+ data.setEventId(event.getEventNumber());
}
return data;
}
@@ -50,6 +56,11 @@
* The scaler data values.
*/
private int[] data;
+
+ /**
+ * The event ID of the data.
+ */
+ private Integer eventId;
/**
* This is the no argument constructor which is for package internal use only.
@@ -62,9 +73,10 @@
*
* @param data the scaler data
*/
- public ScalerData(final int[] data) {
+ public ScalerData(final int[] data, final int eventId) {
this.data = new int[data.length];
System.arraycopy(data, 0, this.data, 0, data.length);
+ this.eventId = eventId;
}
/**
@@ -80,6 +92,18 @@
}
/**
+ * Get the event ID of the scaler data.
+ * <p>
+ * This information is not persisted to the LCIO.
+ *
+ * @return the event ID of the scaler data
+ */
+ public Integer getEventId() {
+ // Null value will be returned here to indicate not set.
+ return this.eventId;
+ }
+
+ /**
* Get the scaler data value at the index.
*
* @param index the scaler data index
@@ -88,14 +112,23 @@
public Integer getValue(final int index) {
return this.data[index];
}
-
+
/**
* Get the value using a {@link ScalerDataIndex} enum.
- *
+ *
* @return the value at the index
*/
- public Integer getValue(ScalerDataIndex scalarDataIndex) {
+ public Integer getValue(final ScalerDataIndex scalarDataIndex) {
return this.data[scalarDataIndex.index()];
+ }
+
+ /**
+ * Set the event ID of the scaler data.
+ *
+ * @param eventId the event ID of the scaler data
+ */
+ void setEventId(final int eventId) {
+ this.eventId = eventId;
}
/**
@@ -154,5 +187,5 @@
final List<GenericObject> collection = new ArrayList<GenericObject>();
collection.add(this.toGenericObject());
event.put(collectionName, collection, GenericObject.class, 0);
- }
+ }
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/scalers/ScalersEvioProcessor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/scalers/ScalersEvioProcessor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/scalers/ScalersEvioProcessor.java Thu Aug 20 12:50:12 2015
@@ -1,10 +1,15 @@
package org.hps.record.scalers;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.hps.record.evio.EvioEventConstants;
import org.hps.record.evio.EvioEventProcessor;
+import org.hps.record.evio.EvioEventUtilities;
import org.jlab.coda.jevio.BaseStructure;
import org.jlab.coda.jevio.EvioEvent;
import org.lcsim.util.log.DefaultLogFormatter;
@@ -15,38 +20,36 @@
*
* @author <a href="mailto:[log in to unmask]">Jeremy McCormick</a>
*/
-public final class ScalersEvioProcessor extends EvioEventProcessor {
+public class ScalersEvioProcessor extends EvioEventProcessor {
- private static final Logger LOGGER = LogUtil.create(ScalersEvioProcessor.class, new DefaultLogFormatter(), Level.INFO);
+ private static final Logger LOGGER = LogUtil.create(ScalersEvioProcessor.class, new DefaultLogFormatter(),
+ Level.ALL);
/**
* Currently cached ScalerData object which was created by the process method.
*/
- private ScalerData data;
+ private ScalerData currentScalerData;
+ private boolean resetEveryEvent = true;
- boolean resetEveryEvent = true;
+ private Set<ScalerData> scalerDataSet = new LinkedHashSet<ScalerData>();
+
+ public ScalerData getCurrentScalerData() {
+ return this.currentScalerData;
+ }
/**
* Get the current scaler data or null if there was none in the last event processed.
*
* @return the current scaler data or <code>null</code> if none exists
*/
- public ScalerData getScalerData() {
- return this.data;
+ public List<ScalerData> getScalerData() {
+ return new ArrayList<ScalerData>(this.scalerDataSet);
}
- /**
- * This method will create a <code>ScalerData</code> object and cache it. The current object is first reset to <code>null</code> every time this
- * method is called.
- *
- * @param evio the EVIO event data
- */
- @Override
- public void process(final EvioEvent evio) {
- if (resetEveryEvent) {
- this.data = null;
- }
- for (final BaseStructure bank : evio.getChildrenList()) {
+ private ScalerData getScalerData(final EvioEvent evioEvent) {
+ ScalerData scalerData = null;
+ // Proceed if sync bit checking is not enabled or sync bit is on.
+ outerBankLoop: for (final BaseStructure bank : evioEvent.getChildrenList()) {
// Does the crate tag match?
if (bank.getHeader().getTag() == EvioEventConstants.SCALERS_CRATE_TAG) {
if (bank.getChildrenList() != null) {
@@ -54,19 +57,48 @@
// Does the bank tag match?
if (subBank.getHeader().getTag() == EvioEventConstants.SCALERS_BANK_TAG) {
- LOGGER.fine("found scaler data in bank " + subBank.getHeader().getTag() + " and EVIO event " + evio.getEventNumber());
+ LOGGER.fine("found scaler data in bank " + subBank.getHeader().getTag()
+ + " and EVIO event " + evioEvent.getEventNumber());
// Scaler data exists in event so create object and stop processing.
- this.data = new ScalerData(subBank.getIntData());
- break;
+ scalerData = new ScalerData(subBank.getIntData(),
+ EvioEventUtilities.getEventIdData(evioEvent)[0]);
+
+ break outerBankLoop;
}
}
}
}
+ }
+ return scalerData;
+ }
+
+ /**
+ * This method will create a <code>ScalerData</code> object and cache it. The current object is first reset to
+ * <code>null</code> every time this method is called.
+ *
+ * @param evioEvent the EVIO event data
+ */
+ @Override
+ public void process(final EvioEvent evioEvent) {
+ if (resetEveryEvent) {
+ // Reset the cached data object.
+ this.currentScalerData = null;
+ }
+
+ final ScalerData scalerData = this.getScalerData(evioEvent);
+ if (scalerData != null) {
+ this.currentScalerData = scalerData;
+ this.scalerDataSet.add(this.currentScalerData);
}
}
public void setResetEveryEvent(final boolean resetEveryEvent) {
this.resetEveryEvent = resetEveryEvent;
}
+
+ @Override
+ public void startJob() {
+ this.scalerDataSet = new LinkedHashSet<ScalerData>();
+ }
}
