Author: [log in to unmask]
Date: Wed Jan 14 12:21:37 2015
New Revision: 1929
Log:
Add reading of run numbers from head bank in EVIO converter. Also refactor code for updating conditions so it is less messy.
Modified:
java/trunk/evio/src/main/java/org/hps/evio/EvioToLcio.java
Modified: java/trunk/evio/src/main/java/org/hps/evio/EvioToLcio.java
=============================================================================
--- java/trunk/evio/src/main/java/org/hps/evio/EvioToLcio.java (original)
+++ java/trunk/evio/src/main/java/org/hps/evio/EvioToLcio.java Wed Jan 14 12:21:37 2015
@@ -22,11 +22,15 @@
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
+import org.freehep.record.source.NoSuchRecordException;
import org.hps.conditions.database.DatabaseConditionsManager;
import org.hps.job.JobManager;
import org.hps.record.LCSimEventBuilder;
+import org.hps.record.evio.EvioEventQueue;
import org.hps.record.evio.EvioEventUtilities;
+import org.jlab.coda.jevio.BaseStructure;
import org.jlab.coda.jevio.EvioEvent;
+import org.jlab.coda.jevio.EvioException;
import org.jlab.coda.jevio.EvioReader;
import org.lcsim.conditions.ConditionsManager;
import org.lcsim.conditions.ConditionsManager.ConditionsNotFoundException;
@@ -36,8 +40,8 @@
/**
* <p>
- * This class converts EVIO to LCIO, performing an LCSim job in the same session.
- * The processed events are then (optionally) written to disk using an LCIOWriter.
+ * This class converts EVIO to LCIO, performing an LCSim job in the same session. The processed
+ * events are then (optionally) written to disk using an LCIOWriter.
* <p>
* To run this class from the command line:<br>
* java -cp hps-distribution-bin.jar EvioToLcio [options] [evioFiles]
@@ -47,15 +51,27 @@
* <p>
* Extra arguments are treated as paths to EVIO files.
* <p>
- * This class attempts to automatically configure itself for Test Run or Engineering Run
- * based on the run numbers in the EVIO file. It will use an appropriate default detector
- * unless one is given on the command line, and it will also use the correct event builder.
- * It will not handle jobs correctly with files from both the Test and Engineering Run
- * so don't do this!
+ * This class attempts to automatically configure itself for Test Run or Engineering Run based on
+ * the run numbers in the EVIO file. It will use an appropriate default detector unless one is given
+ * on the command line, and it will also use the correct event builder. It will not handle jobs
+ * correctly with files from both the Test and Engineering Run, so don't do this!
+ * <p>
+ * The conditions system can be initialized in one of three ways.<br/>
+ * <ol>
+ * <li>user specified run number in which case the conditions system is frozen for the rest of the job</li>
+ * <li>run number from an EVIO pre start event</li>
+ * <li>run number from a header bank in an event</li>
+ * </ol>
+ * <p>
+ * In the case where a file has no pre start event and there are header banks present,
+ * the "-m" command line option can be used to buffer a number of EVIO events. If there is a
+ * head bank found while adding these events to queue, the conditions system will be initialized
+ * from it.
*
* @author Jeremy McCormick <[log in to unmask]>
* @author Sho Uemura <[log in to unmask]>
*/
+// TODO: Logger should print tracebacks. See Driver's setup for example of this.
public class EvioToLcio {
// The default steering resource, which basically does nothing except print event numbers.
@@ -67,9 +83,14 @@
// The class's logger.
Logger logger = LogUtil.create(EvioToLcio.class);
+ // The LCSim event builder used to convert from EVIO.
LCSimEventBuilder eventBuilder = null;
+ // The detector name for conditions.
String detectorName;
+
+ // The run number for conditions.
+ Integer runNumber = null;
/**
* The default constructor, which defines command line arguments and sets the log level.
@@ -84,14 +105,16 @@
options.addOption(new Option("D", true, "pass a variable to the steering file with format -Dname=value"));
options.addOption(new Option("l", true, "path of output LCIO file"));
options.addOption(new Option("d", true, "name of the detector to use for LCSim conditions"));
- options.addOption(new Option("R", true, "fixed run number which will override run numbers of input files"));
- options.addOption(new Option("n", true, "maximum number of events to process in the job"));
- options.addOption(new Option("b", false, "enable headless mode which will not show plots OR allow writing them to graphics files"));
+ options.addOption(new Option("R", true, "fixed run number which will override run numbers of input files"));
+ options.addOption(new Option("n", true, "maximum number of events to process in the job"));
+ options.addOption(new Option("b", false, "enable headless mode which will not show plots OR allow writing them to graphics files"));
+ options.addOption(new Option("v", false, "print EVIO XML for each event"));
+ options.addOption(new Option("m", true, "set the max event buffer size"));
logger.setLevel(Level.FINE);
}
/**
- * Run the EVIO to LCIO converter from the command line.
+ * Run the EVIO to LCIO converter from the command line.
* @param args The command line arguments.
*/
public static void main(String[] args) {
@@ -100,40 +123,40 @@
}
/**
- * This method will execute the EVIO to LCIO conversion and optionally process the events with LCSim Drivers
- * from a steering file. Then the resultant LCIO events will be written to disk if the <code>-l</code> option
- * is present.
+ * This method will execute the EVIO to LCIO conversion and optionally process the events with
+ * LCSim Drivers from a steering file. Then the resultant LCIO events will be written to disk if
+ * this option is enabled in the command line arguments.
* @param args The command line arguments.
*/
public void run(String[] args) {
int maxEvents = -1;
int nEvents = 0;
-
- // Set up command line parsing.
+ int maxBufferSize = 40;
+
+ // Parse the command line options.
if (args.length == 0) {
printUsage();
}
CommandLineParser parser = new PosixParser();
-
- // Parse command line arguments.
CommandLine cl = null;
try {
cl = parser.parse(options, args);
} catch (ParseException e) {
throw new RuntimeException("Problem parsing command line options.", e);
}
-
- // Set log level.
+
+ // Set the log level.
if (cl.hasOption("L")) {
Level level = Level.parse(cl.getOptionValue("L").toUpperCase());
logger.config("setting log level to " + level);
logger.setLevel(level);
}
-
+
// Add all extra arguments to the EVIO file list.
List<String> evioFileList = new ArrayList<String>(Arrays.asList(cl.getArgs()));
-
+
+ // Process text file containing list of EVIO file paths, one per line.
if (cl.hasOption("f")) {
// Add additional EVIO files to process from text file.
File file = new File(cl.getOptionValue("f"));
@@ -156,14 +179,14 @@
}
}
}
-
+
// Is the EVIO file list empty?
- if (evioFileList.isEmpty()) {
+ if (evioFileList.isEmpty()) {
// There weren't any EVIO files provided on the command line so exit.
logger.severe("No EVIO files were provided with command line arguments or -f option.");
printUsage();
}
-
+
String lcioFileName = null;
LCIOWriter writer = null;
InputStream steeringStream = null;
@@ -195,13 +218,14 @@
}
}
+ // Setup the default steering which just prints event numbers.
if (steeringStream == null) {
steeringStream = EvioToLcio.class.getResourceAsStream(DEFAULT_STEERING_RESOURCE);
logger.config("using default steering resource " + DEFAULT_STEERING_RESOURCE);
}
// Get the max number of events to process.
- if (cl.hasOption("n")) {
+ if (cl.hasOption("n")) {
maxEvents = Integer.valueOf(cl.getOptionValue("n"));
if (maxEvents <= 0) {
throw new IllegalArgumentException("Value of -n option is invalid: " + maxEvents);
@@ -236,62 +260,61 @@
logger.config("set steering variable: " + key + "=" + value);
}
}
+
+ // Enable headless mode so no plots are shown.
if (cl.hasOption("b")) {
- // Enable headless mode so no plots are shown.
logger.config("Headless mode is enabled. No plots will be shown.");
jobManager.enableHeadlessMode();
}
-
+
// Configure the LCSim job manager.
jobManager.setup(steeringStream);
jobManager.configure();
logger.config("LCSim job manager was successfully configured.");
-
+
+ // Get the user specified detector name.
if (cl.hasOption("d")) {
- // Get the user specified detector name.
detectorName = cl.getOptionValue("d");
logger.config("User set detector to " + detectorName + " with command option.");
}
- Integer runNumber = null;
- if (cl.hasOption("R")) {
- // Get the user specified run number.
+ // Get the user specified run number.
+ if (cl.hasOption("R")) {
runNumber = Integer.parseInt(cl.getOptionValue("R"));
logger.config("User set run number to " + runNumber + " with command option.");
}
- // Initialize the HPS database conditions system.
- DatabaseConditionsManager conditionsManager = DatabaseConditionsManager.getInstance();
-
- // Is there a user specified run number?
+ // Is there a run number from the command line options?
if (runNumber != null) {
- // Setup the event builder at the beginning of the job with the run number and detector name.
- this.setupEventBuilder(detectorName, runNumber);
- try {
- // Setup the conditions system, which will initialize the event builder via the ConditionsListener callback.
- conditionsManager.setDetector(detectorName, runNumber);
- } catch (ConditionsNotFoundException e) {
- throw new RuntimeException();
- }
- // Since there is a user specified run number override, the conditions system is frozen so that run numbers
- // from data are ignored.
- logger.config("Conditions system will be frozen to use specified run number and detector!");
- conditionsManager.freeze();
- }
-
+ // Initialize the conditions system before the job starts and freeze it.
+ checkConditions(runNumber, true);
+ }
+
// Print out the EVIO file list before the job starts.
StringBuffer buff = new StringBuffer();
buff.append("The job will include the following EVIO files ...");
buff.append('\n');
- for (String evioFileName : evioFileList) {
+ for (String evioFileName : evioFileList) {
buff.append(evioFileName);
buff.append('\n');
- }
+ }
logger.config(buff.toString());
-
+
+ // Get whether to debug print XML from the EVIO events.
+ boolean printXml = false;
+ if (cl.hasOption("v")) {
+ printXml = true;
+ }
+
+ // Get the maximum number of EVIO events to buffer.
+ if (cl.hasOption("m")) {
+ maxBufferSize = Integer.parseInt(cl.getOptionValue("m"));
+ }
+
// Loop over the input EVIO files.
+ EvioReader reader = null;
fileLoop: for (String evioFileName : evioFileList) {
-
+
// Get the next EVIO input file.
File evioFile = new File(evioFileName);
if (!evioFile.exists()) {
@@ -300,107 +323,138 @@
logger.info("Opening EVIO file " + evioFileName + " ...");
// Open the EVIO reader.
- EvioReader reader = null;
try {
reader = new EvioReader(evioFile);
} catch (Exception e) {
- throw new RuntimeException(e);
+ throw new RuntimeException("Error opening the EVIO file reader.", e);
}
boolean firstEvent = true;
- long time = 0; // in ms
-
- // Loop over EVIO events, build LCSim events, process them, and then write events to disk.
- readLoop: while (maxEvents == -1 || nEvents < maxEvents) {
- EvioEvent evioEvent = null;
- try {
- while (evioEvent == null) {
- evioEvent = reader.nextEvent();
- if (evioEvent == null) {
- break readLoop;
+ long eventTime = 0; // in ms
+
+ // Loop over events.
+ EvioEventQueue eventQueue = new EvioEventQueue(-1L, maxBufferSize);
+ eventLoop: for (;;) {
+
+ // Buffer the EVIO events into the queue.
+ bufferEvents(reader, eventQueue, maxBufferSize);
+
+ // Is the event queue empty?
+ if (eventQueue.size() == 0) {
+ // Break from the event processing loop.
+ break eventLoop;
+ }
+
+ // Loop over the EVIO events in the buffer until it is empty.
+ recordLoop: while (eventQueue.hasNext()) {
+
+ // Read and parse the next EVIO event.
+ EvioEvent evioEvent = null;
+ try {
+ eventQueue.next();
+ evioEvent = (EvioEvent) eventQueue.getCurrentRecord();
+ reader.parseEvent(evioEvent);
+ } catch (IOException | EvioException e) {
+ // This means the EVIO event has bad data.
+ logger.severe(e.getMessage());
+ e.printStackTrace();
+ continue recordLoop;
+ } catch (NoSuchRecordException e) {
+ // This means the queue does not have any more events.
+ // We checked hasNext() already so it should not happen.
+ logger.severe(e.getMessage());
+ e.printStackTrace();
+ break recordLoop;
+ }
+
+ // Print out event XML if enabled.
+ if (printXml) {
+ logger.info(evioEvent.toXML());
+ }
+
+ // Is this a pre start event?
+ if (EvioEventUtilities.isPreStartEvent(evioEvent)) {
+
+ // Get the pre start event's data bank.
+ int[] data = EvioEventUtilities.getControlEventData(evioEvent);
+
+ if (data == null) {
+ // This should never happen but just ignore it.
+ logger.severe("Pre start event is missing a data bank.");
+ } else {
+ // Check if conditions system needs to be updated from the pre start data.
+ checkConditions(data[1], false);
}
- reader.parseEvent(evioEvent);
- }
-
- // Handle a pre start event.
- if (EvioEventUtilities.isPreStartEvent(evioEvent)) {
-
- // Get the pre start event's int data bank.
- int[] data = EvioEventUtilities.getControlEventData(evioEvent);
-
- if (data == null) {
- logger.severe("Pre start data bank was not found.");
- throw new RuntimeException("Pre start data bank is null.");
- }
-
- // int seconds = data[0];
- int preStartRunNumber = data[1];
-
- logger.info("EVIO pre start event with run #" + preStartRunNumber);
-
- // Is the event builder uninitialized?
- if (eventBuilder == null) {
- // Initialize the event builder.
- setupEventBuilder(detectorName, preStartRunNumber);
- }
- }
-
- // Setup state in the LCSimEventBuilder based on the EVIO event.
- if (eventBuilder != null) {
- eventBuilder.readEvioEvent(evioEvent);
- } else {
- throw new RuntimeException("The event builder was never setup. Try manually setting a run number using the -R switch.");
- }
-
- // Handle an end event.
+ }
+
+ // Is this an end event?
if (EvioEventUtilities.isEndEvent(evioEvent)) {
int[] data = EvioEventUtilities.getControlEventData(evioEvent);
if (data == null) {
- logger.severe("The end event data bank was not found.");
- throw new RuntimeException("The end event data bank is null.");
+ // This should never happen but just ignore it.
+ logger.severe("The end event is missing a data bank.");
+ } else {
+ int seconds = data[0];
+ int totalEvents = data[2];
+ logger.info("EVIO end event with " + totalEvents + " events and " + seconds + " seconds");
}
- int seconds = data[0];
- int totalEvents = data[2];
- logger.info("EVIO end event with " + totalEvents + " events and " + seconds + " seconds");
- // Handle a physics event.
- } else if (EvioEventUtilities.isPhysicsEvent(evioEvent)) {
-
- logger.finest("got EVIO physics event #" + evioEvent.getEventNumber());
+ }
+
+ // Setup state in the LCSimEventBuilder based on the EVIO control event.
+ if (eventBuilder != null) {
+ eventBuilder.readEvioEvent(evioEvent);
+ }
+
+ // Is this a physics event?
+ if (EvioEventUtilities.isPhysicsEvent(evioEvent)) {
+
+ // Print physics event number, which is actually a sequence number from
+ // the reader, not the actual event number from the data.
+ logger.finest("physics event seq #" + evioEvent.getEventNumber());
// Is the event builder initialized?
if (eventBuilder == null) {
- // This can happen if there are no pre start events in the EVIO file and no run number was explicitly given on the command line.
- throw new RuntimeException("The LCSimEventBuilder was never initialized. You may need to manually specify a run number using the -R switch.");
+ // Die here, because the event builder should be setup by now.
+ throw new RuntimeException("The LCSimEventBuilder was never initialized.");
}
+ // Build the LCIO event.
EventHeader lcioEvent = eventBuilder.makeLCSimEvent(evioEvent);
- logger.finest("created LCIO event #" + lcioEvent.getEventNumber());
-
- time = (lcioEvent.getTimeStamp() / 1000000);
-
+ eventTime = (lcioEvent.getTimeStamp() / 1000000);
+ logger.finest("created LCIO event #" + lcioEvent.getEventNumber() + " with time " + new Date(eventTime));
if (firstEvent) {
- logger.info("first physics event time: " + time / 1000 + " - " + new Date(time));
+ logger.info("first physics event time: " + eventTime / 1000 + " - " + new Date(eventTime));
firstEvent = false;
}
- logger.finest("processing LCIO event in LCSim");
+ // Activate Driver process methods.
jobManager.processEvent(lcioEvent);
- logger.finest("done processing LCIO event in LCSim");
+
+ // Write out this LCIO event.
if (writer != null) {
- writer.write(lcioEvent);
- writer.flush();
- logger.finest("wrote LCIO event #" + lcioEvent.getEventNumber());
+ try {
+ writer.write(lcioEvent);
+ writer.flush();
+ } catch (IOException e) {
+ throw new RuntimeException("Error writing LCIO file.", e);
+ }
+ logger.finest("wrote event #" + lcioEvent.getEventNumber());
}
- nEvents++;
+ // Increment number of events processed.
+ nEvents++;
+
+ // Check if max events was reached and end job if this is true.
+ if (maxEvents != -1 && nEvents >= maxEvents) {
+ logger.info("maxEvents " + maxEvents + " was reached");
+ break fileLoop;
+ }
}
-
- } catch (Exception e) {
- logger.log(Level.SEVERE, "Error in LCIO event processing.", e);
- throw new RuntimeException(e);
- }
- }
- logger.info("Last physics event time: " + time / 1000 + " - " + new Date(time));
+ }
+ } // eventLoop
+ logger.info("Last physics event time: " + eventTime / 1000 + " - " + new Date(eventTime));
+
+ // Close the EVIO reader.
try {
reader.close();
logger.fine("EVIO reader closed.");
@@ -408,24 +462,99 @@
logger.warning(e.getMessage());
e.printStackTrace();
}
-
- if (maxEvents != -1 && nEvents < maxEvents) {
- logger.info("maxEvents " + maxEvents + " was reached");
- break fileLoop;
- }
- }
- jobManager.finish();
+ } // fileLoop
+
+ // If the processing stopped because of max events then need to cleanup the reader.
+ if (reader != null) {
+ if (!reader.isClosed()) {
+ // Close the EVIO reader.
+ try {
+ reader.close();
+ logger.fine("EVIO reader closed.");
+ } catch (IOException e) {
+ logger.warning(e.getMessage());
+ e.printStackTrace();
+ }
+ }
+ }
+
+ // Trigger endOfData on LCSim Drivers.
+ jobManager.finish();
+
+ // Close the LCIO writer.
if (writer != null) {
try {
writer.close();
logger.info("LCIO output writer closed okay.");
} catch (IOException e) {
+ e.printStackTrace();
logger.warning(e.getMessage());
+ }
+ }
+
+ logger.info("Job finished successfully!");
+ }
+
+ /**
+ * Buffer up to <code>maxBufferSize</code> events in the <code>eventQueue</code>.
+ * This method will also initialize the conditions system using a run number
+ * if a header bank is found.
+ * @param reader The EVIO reader.
+ * @param eventQueue The event queue.
+ * @param maxBufferSize The maximum number of records to buffer.
+ */
+ private void bufferEvents(EvioReader reader, EvioEventQueue eventQueue, int maxBufferSize) {
+ EvioEvent evioEvent = null;
+ while (eventQueue.size() < maxBufferSize) {
+ try {
+ // Break if no more events from reader.
+ if (reader.getNumEventsRemaining() == 0) {
+ break;
+ }
+
+ // Read the next event.
+ evioEvent = reader.nextEvent();
+
+ if (evioEvent == null) {
+ break;
+ }
+
+ // Add the event to the queue.
+ eventQueue.addRecord(evioEvent);
+
+ } catch (IOException | EvioException e) {
+ logger.severe(e.getMessage());
e.printStackTrace();
- }
- }
-
- logger.info("Job finished successfully!");
+ }
+
+ // Check here while buffering if the run number can be used to initialize the conditions system.
+ if (evioEvent != null) {
+ try {
+ reader.parseEvent(evioEvent);
+ } catch (EvioException e) {
+ logger.severe(e.getMessage());
+ e.printStackTrace();
+ continue;
+ }
+
+ // Head head bank from event.
+ BaseStructure headBank = EvioEventUtilities.getHeadBank(evioEvent);
+
+ // Is there a head bank available?
+ if (headBank != null) {
+
+ // Get the run number from the head bank.
+ int headBankRunNumber = headBank.getIntData()[1];
+ logger.finer("got head bank with run number " + headBankRunNumber);
+
+ // Check if the conditions system needs to be updated from the head bank.
+ checkConditions(headBankRunNumber, false);
+ } else {
+ logger.finest("event " + evioEvent.getEventNumber() + " does not have a head bank");
+ }
+ }
+ }
+ logger.finer("buffered " + eventQueue.size() + " events");
}
/**
@@ -439,20 +568,20 @@
}
/**
- * Setup the LCSimEventBuilder based on the given detector name and run number.
+ * Setup the LCSimEventBuilder based on the current detector name and run number.
* @param detectorName The detector name to be assigned to the event builder.
* @param runNumber The run number which determines which event builder to use.
* @return The LCSimEventBuilder for the Test Run or Engineering Run.
*/
- private void setupEventBuilder(String detectorName, int runNumber) {
+ private void setupEventBuilder(int runNumber) {
// Is this run number from the Test Run?
if (DatabaseConditionsManager.isTestRun(runNumber)) {
// Configure conditions system for Test Run.
logger.info("using LCSimTestRunEventBuilder");
eventBuilder = new LCSimTestRunEventBuilder();
- if (detectorName == null) {
+ if (detectorName == null) {
this.detectorName = DatabaseConditionsManager.getDefaultTestRunDetectorName();
- logger.info("using default Test Run detector name " + this.detectorName);
+ logger.info("using default Test Run detector name " + detectorName);
}
} else {
// Configure conditions system for Eng Run or default.
@@ -460,11 +589,43 @@
eventBuilder = new LCSimEngRunEventBuilder();
if (detectorName == null) {
this.detectorName = DatabaseConditionsManager.getDefaultEngRunDetectorName();
- logger.info("using default Eng Run detector name " + this.detectorName);
- }
- }
- eventBuilder.setDetectorName(this.detectorName);
- ConditionsManager.defaultInstance().addConditionsListener(eventBuilder);
- logger.config("initialized " + eventBuilder.getClass().getCanonicalName() + " with detector " + this.detectorName + " and run number " + runNumber);
+ logger.info("using default Eng Run detector name " + detectorName);
+ }
+ }
+ ConditionsManager conditions = ConditionsManager.defaultInstance();
+ conditions.addConditionsListener(eventBuilder);
+ }
+
+ /**
+ * Check if the conditions system and event builder need to be initialized
+ * or updated given a run number.
+ * @param runNumber The run number.
+ * @param freeze True to freeze conditions system after it is setup.
+ */
+ private void checkConditions(int runNumber, boolean freeze) {
+
+ // Is the event builder uninitialized?
+ if (eventBuilder == null) {
+ // Setup event builder.
+ setupEventBuilder(runNumber);
+ }
+
+ // Is run number not set or is this a new run number?
+ if (this.runNumber == null || runNumber != this.runNumber) {
+ // Make this the new run number.
+ this.runNumber = runNumber;
+
+ // Update the conditions system with the new run number.
+ try {
+ ConditionsManager.defaultInstance().setDetector(detectorName, this.runNumber);
+ } catch (ConditionsNotFoundException e) {
+ throw new RuntimeException("Error initializing conditions system.", e);
+ }
+ }
+
+ if (freeze) {
+ // Freeze the conditions system so subsequent run numbers are ignored.
+ DatabaseConditionsManager.getInstance().freeze();
+ }
}
}
|
