Author: [log in to unmask]
Date: Thu Nov 13 21:56:10 2014
New Revision: 1518
Log:
Rewrite EVIO to LCIO frontend so that it tries to configure itself properly from event numbers. Also remove a bunch of extraneous command line arguments.
Modified:
java/trunk/evio/src/main/java/org/hps/evio/EvioToLcio.java
Modified: java/trunk/evio/src/main/java/org/hps/evio/EvioToLcio.java
=============================================================================
--- java/trunk/evio/src/main/java/org/hps/evio/EvioToLcio.java (original)
+++ java/trunk/evio/src/main/java/org/hps/evio/EvioToLcio.java Thu Nov 13 21:56:10 2014
@@ -16,310 +16,385 @@
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
-import org.hps.conditions.DatabaseConditionsManager;
import org.jlab.coda.jevio.EvioEvent;
import org.jlab.coda.jevio.EvioReader;
import org.lcsim.conditions.ConditionsManager;
-import org.lcsim.conditions.ConditionsManager.ConditionsNotFoundException;
import org.lcsim.event.EventHeader;
import org.lcsim.job.JobControlManager;
import org.lcsim.lcio.LCIOWriter;
+import org.lcsim.util.log.LogUtil;
/**
- * This class is for converting Test Run EVIO to LCIO events and performing an
- * LCSim job in the same session. The processed events are then written to disk
- * using an LCIOWriter.
- *
- * To run this class from command line:
- *
- * TestRunEvioToLcio [evioFile] -l [lcioFile] -d [detectorName] -x
- * [lcsimXmlFile]
+ * <p>
+ * This class converts EVIO to LCIO, performing an LCSim job in the same
+ * session. The processed events are then (optionally) written to disk using an
+ * LCIOWriter.
+ * <p>
+ * To run this class from the command line:<br>
+ * java -cp hps-distribution-bin.jar EvioToLcio [options] [evioFiles]
+ * <p>
+ * The available arguments can be printed using:<br>
+ * java -cp hps-distribution-bin.jar EvioToLcio -h
+ * <p>
+ * Extra arguments are treated as paths to EVIO files.
+ * <p>
+ * This class attempts to automatically configure itself for Test Run or Engineering Run
+ * based on the run numbers in the EVIO file. It will use an appropriate default detector
+ * unless one is given on the command line, and it will also use the correct event builder.
+ * However, it will NOT correctly handle a mixed list of EVIO files from both runs, so don't
+ * do this!
*
* @author Jeremy McCormick <[log in to unmask]>
*/
public class EvioToLcio {
- private static final String defaultDetectorName = "";
- private static final String defaultSteeringFile = "/org/hps/steering/monitoring/DummyMonitoring.lcsim";
-
- /**
- * Defines command line options for this program.
- *
- * @return The command line options.
- */
- private static Options createCommandLineOptions() {
- Options options = new Options();
- options.addOption(new Option("l", true, "The name of the output LCIO file"));
- options.addOption(new Option("d", true, "The name of the detector to use for LCSim conditions"));
- options.addOption(new Option("x", true, "The LCSim XML file to process the LCIO events"));
- options.addOption(new Option("s", true, "Sleep duration between events (in ms)"));
- options.addOption(new Option("n", true, "Stop after N events"));
- options.addOption(new Option("w", false, "Wait after end of data"));
- options.addOption(new Option("D", true, "Pass a variable to the steering file"));
- options.addOption(new Option("r", false, "Interpret -x argument as a steering resource instead of a file path"));
- options.addOption(new Option("R", true, "The run number"));
- options.addOption(new Option("t", false, "Read test run data"));
- return options;
- }
-
- /**
- * This method will execute the EVIO to LCIO conversion and perform an
- * intermediate LCSim job. Then the resultant LCIO events will be written to
- * disk.
- * @param args The command line arguments.
- */
- public static void main(String[] args) {
- int maxEvents = 0;
- int nEvents = 0;
-
- // Set up command line parsing.
- Options options = createCommandLineOptions();
- if (args.length == 0) {
- System.out.println("TestRunEvioToLcio [options] [evioFiles]");
- HelpFormatter help = new HelpFormatter();
- help.printHelp(" ", options);
- System.exit(1);
- }
- CommandLineParser parser = new PosixParser();
-
- // Parse command line arguments.
- CommandLine cl = null;
- try {
- cl = parser.parse(options, args);
- } catch (ParseException e) {
- throw new RuntimeException("Problem parsing command line options.", e);
- }
-
- String lcioFileName = null;
- LCIOWriter writer = null;
- String detectorName = defaultDetectorName;
- InputStream steeringStream = null;
- int sleepTime = -1;
-
- // LCIO output file.
- if (cl.hasOption("l")) {
- lcioFileName = cl.getOptionValue("l");
- }
-
- // Name of detector.
- if (cl.hasOption("d")) {
- detectorName = cl.getOptionValue("d");
- }
-
- if ("".equals(detectorName)) {
- throw new RuntimeException("You need to specify a valid detector name as input, use the -d option");
- }
-
- // LCSim XML file to execute inline.
- if (cl.hasOption("x")) {
- String lcsimXmlName = cl.getOptionValue("x");
- if (cl.hasOption("r")) {
- steeringStream = TestRunEvioToLcio.class.getResourceAsStream(lcsimXmlName);
- if (steeringStream == null) {
- throw new RuntimeException("resource " + lcsimXmlName + " not found");
- }
- } else {
- try {
- steeringStream = new FileInputStream(lcsimXmlName);
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- if (steeringStream == null) {
- steeringStream = TestRunEvioToLcio.class.getResourceAsStream(defaultSteeringFile);
- }
-
- // Sleep time.
- if (cl.hasOption("s")) {
- sleepTime = Integer.valueOf(cl.getOptionValue("s"));
- }
-
- // Sleep time.
- if (cl.hasOption("n")) {
- maxEvents = Integer.valueOf(cl.getOptionValue("n"));
- }
-
- // LCIO writer.
- if (lcioFileName != null) {
- try {
- writer = new LCIOWriter(new File(lcioFileName));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- // LCSim job manager.
- JobControlManager jobManager = new JobControlManager();
-
- if (cl.hasOption("D")) {
- String[] steeringOptions = cl.getOptionValues("D");
- for (String def : steeringOptions) {
- String[] s = def.split("=");
- if (s.length != 2) {
- throw new RuntimeException("Bad variable format: " + def);
- }
- String key = s[0];
- String value = s[1];
- jobManager.addVariableDefinition(key, value);
- }
- }
-
- int runNumber = 0;
- if (cl.hasOption("R")) {
- runNumber = Integer.valueOf(cl.getOptionValue("R"));
- }
-
- jobManager.setup(steeringStream);
- jobManager.configure();
-
- // LCSim event builder.
- LCSimEventBuilder eventBuilder = null;
-
- DatabaseConditionsManager manager = DatabaseConditionsManager.getInstance();
-
- if (cl.hasOption("t")) {
- manager.configure("/org/hps/conditions/config/conditions_database_testrun_2012.xml");
- eventBuilder = new LCSimTestRunEventBuilder();
- } else {
- eventBuilder = new LCSimEngRunEventBuilder();
- }
-
- try {
- manager.setDetector(detectorName, runNumber);
- } catch (ConditionsNotFoundException e) {
- throw new RuntimeException(e);
- }
-
- eventBuilder.setDetectorName(detectorName);
- ConditionsManager.defaultInstance().addConditionsListener(eventBuilder);
-
- for (String evioFileName : cl.getArgs()) {
- // EVIO input file.
- File evioFile = new File(evioFileName);
- System.out.println("Opening file " + evioFileName);
- // EVIO reader.
- EvioReader reader = null;
- try {
- reader = new EvioReader(evioFile);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- boolean firstEvent = true;
- long time = 0; // in ms
-
- // Loop over EVIO events, build LCSim events, process them, and then
- // write events to disk.
- fileLoop: while (maxEvents == 0 || nEvents < maxEvents) {
- EvioEvent evioEvent = null;
- try {
- eventLoop: while (evioEvent == null) {
- evioEvent = reader.nextEvent();
- if (evioEvent == null) {
- break fileLoop;
- }
- try {
- reader.parseEvent(evioEvent);
- } catch (Exception e) {
- Logger.getLogger(TestRunEvioToLcio.class.getName()).log(Level.SEVERE, "Error reading EVIO event", e);
- continue eventLoop;
- }
- }
- // let event builder check for run information
- eventBuilder.readEvioEvent(evioEvent);
- // Handlers for different event types.
- if (EventConstants.isPreStartEvent(evioEvent)) {
- int[] data = evioEvent.getIntData();
- //int seconds = data[0];
- runNumber = data[1];
- // calibListener.prestart(seconds, runNumber);
- } else if (EventConstants.isEndEvent(evioEvent)) {
- int[] data = evioEvent.getIntData();
- //int seconds = data[0];
- //int nevents = data[2];
- // calibListener.endRun(seconds, nevents);
- } else if (eventBuilder.isPhysicsEvent(evioEvent)) {
- EventHeader lcioEvent = eventBuilder.makeLCSimEvent(evioEvent);
-
- time = (lcioEvent.getTimeStamp() / 1000000);
-
- if (firstEvent) {
- System.out.println("First physics event time: " + time / 1000 + " - " + new Date(time));
- firstEvent = false;
- }
- // if
- // (lcioEvent.hasCollection(BaseRawCalorimeterHit.class,
- // "EcalReadoutHits") &&
- // !lcioEvent.get(BaseRawCalorimeterHit.class,
- // "EcalReadoutHits").isEmpty()) {
- // continue;
- // }
- // if (lcioEvent.hasCollection(TriggerData.class,
- // "TriggerBank")) {
- // List<TriggerData> triggerList =
- // lcioEvent.get(TriggerData.class, "TriggerBank");
- // if (!triggerList.isEmpty()) {
- // TriggerData triggerData = triggerList.get(0);
- //
- // int orTrig = triggerData.getOrTrig();
- // int topTrig = triggerData.getTopTrig();
- // int botTrig = triggerData.getBotTrig();
- // int pairTrig = triggerData.getPairTrig();
- // if (topTrig != 0 && botTrig != 0) {
- // System.out.format("%x\t%x\t%x\t%x\n", orTrig,
- // topTrig, botTrig, pairTrig);
- // } else {
- // continue;
- // }
- // }
- // }
- jobManager.processEvent(lcioEvent);
- if (writer != null) {
- writer.write(lcioEvent);
- writer.flush();
- }
-
- }
- } catch (Exception e) {
- // Catch all event processing errors and continue.
- Logger.getLogger(TestRunEvioToLcio.class.getName()).log(Level.SEVERE, "Error in event processing", e);
- continue;
- } finally {
- if (sleepTime > 0) {
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
- nEvents++;
- }
- System.out.println("Last physics event time: " + time / 1000 + " - " + new Date(time));
- try {
- reader.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- System.out.println("No more data");
-
- if (!cl.hasOption("w")) {
- System.out.println("Exiting");
- jobManager.finish();
- System.out.println("jobManager finished");
- }
-
- if (writer != null) {
- System.out.println("close writer");
- try {
- writer.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- System.out.println("closed writer");
- }
- }
+ // Events with run numbers greater than this will use the new LCSimEngRunEventBuilder.
+ private static final int TEST_RUN_END_RUN = 1365;
+
+ // The default Test Run detector.
+ private static final String DEFAULT_TEST_RUN_DETECTOR = "HPS-TestRun-v8-5";
+
+ // The default Engineering Run detector.
+ private static final String DEFAULT_ENG_RUN_DETECTOR = "HPS-Proposal2014-v8-6pt6";
+
+ // The default steering resource, which basically does nothing except
+ // initialize the conditions system.
+ private static final String DEFAULT_STEERING_RESOURCE = "/org/hps/steering/monitoring/DummyMonitoring.lcsim";
+
+ // The command line options which will be defined in the constructor.
+ Options options = null;
+
+ // The class's logger.
+ Logger logger = LogUtil.create(EvioToLcio.class);
+
+ /**
+ * The default constructor, which defines command line arguments and sets the log level.
+ */
+ protected EvioToLcio() {
+ logger.config("initializing");
+ options = new Options();
+ options.addOption(new Option("l", true, "The name of the output LCIO file"));
+ options.addOption(new Option("d", true, "The name of the detector to use for LCSim conditions"));
+ options.addOption(new Option("x", true, "The XML steeering file to process the LCIO events"));
+ options.addOption(new Option("n", true, "Maximum number of events to process"));
+ options.addOption(new Option("D", true, "Pass a variable to the steering file with format -Dname=value"));
+ options.addOption(new Option("r", false, "Interpret steering from -x argument as a resource instead of a file"));
+ options.addOption(new Option("e", false, "Continue if errors occur during event processing"));
+ options.addOption(new Option("L", true, "Set the log level"));
+ logger.setLevel(Level.FINE);
+ }
+
+ /**
+ * Run the EVIO to LCIO converter from the command line.
+ * @param args The command line arguments.
+ */
+ public static void main(String[] args) {
+ EvioToLcio evioToLcio = new EvioToLcio();
+ evioToLcio.run(args);
+ }
+
+ /**
+ * This method will execute the EVIO to LCIO conversion and perform an
+ * intermediate LCSim job. Then the resultant LCIO events will be written to
+ * disk.
+ * @param args The command line arguments.
+ */
+ public void run(String[] args) {
+
+ int maxEvents = 0;
+ int nEvents = 0;
+
+ // Set up command line parsing.
+ if (args.length == 0) {
+ printUsage();
+ }
+ CommandLineParser parser = new PosixParser();
+
+ // Parse command line arguments.
+ CommandLine cl = null;
+ try {
+ cl = parser.parse(options, args);
+ } catch (ParseException e) {
+ throw new RuntimeException("Problem parsing command line options.",
+ e);
+ }
+
+ // Is the extra argument list empty?
+ if (cl.getArgs().length == 0) {
+ // User didn't supply any EVIO files so exit.
+ printUsage();
+ }
+
+ // Set log level.
+ if (cl.hasOption("L")) {
+ Level level = Level.parse(cl.getOptionValue("L").toUpperCase());
+ logger.config("setting log level to " + level);
+ logger.setLevel(level);
+ }
+
+ String lcioFileName = null;
+ LCIOWriter writer = null;
+ InputStream steeringStream = null;
+
+ // LCIO output file.
+ if (cl.hasOption("l")) {
+ logger.config("set LCIO file to " + lcioFileName);
+ lcioFileName = cl.getOptionValue("l");
+ }
+
+ // LCSim XML file to execute on each event created by the builder.
+ if (cl.hasOption("x")) {
+ String lcsimXmlName = cl.getOptionValue("x");
+ if (cl.hasOption("r")) {
+ steeringStream = EvioToLcio.class.getResourceAsStream(lcsimXmlName);
+ if (steeringStream == null) {
+ logger.severe("Resource " + lcsimXmlName + " was not found.");
+ throw new IllegalArgumentException("Resource was not found.");
+ }
+ logger.config("using steering resource " + lcsimXmlName);
+ } else {
+ try {
+ steeringStream = new FileInputStream(lcsimXmlName);
+ logger.config("using steering file " + lcsimXmlName);
+ } catch (FileNotFoundException e) {
+ logger.severe("The steering file " + lcsimXmlName + " does not exist.");
+ throw new IllegalArgumentException("Steering file does not exist.", e);
+ }
+ }
+ }
+
+ if (steeringStream == null) {
+ steeringStream = EvioToLcio.class.getResourceAsStream(DEFAULT_STEERING_RESOURCE);
+ logger.config("using default steering resource " + DEFAULT_STEERING_RESOURCE);
+ }
+
+ // Max number of events.
+ if (cl.hasOption("n")) {
+ maxEvents = Integer.valueOf(cl.getOptionValue("n"));
+ logger.config("set max events to " + maxEvents);
+ }
+
+ // Whether to continue on errors or stop.
+ boolean continueOnErrors = false;
+ if (cl.hasOption("e")) {
+ continueOnErrors = true;
+ logger.config("continue on errors enabled");
+ }
+
+ // LCIO writer.
+ if (lcioFileName != null) {
+ try {
+ writer = new LCIOWriter(new File(lcioFileName));
+ logger.config("initialized LCIO writer with file " + lcioFileName);
+ } catch (IOException e) {
+ logger.severe("Problem initializing the LCIO writer.");
+ throw new RuntimeException(e);
+ }
+ }
+
+ // LCSim job manager.
+ JobControlManager jobManager = new JobControlManager();
+ if (cl.hasOption("D")) {
+ String[] steeringOptions = cl.getOptionValues("D");
+ for (String def : steeringOptions) {
+ String[] s = def.split("=");
+ if (s.length != 2) {
+ logger.severe("bad variable format: " + def);
+ throw new IllegalArgumentException("Bad variable format: " + def);
+ }
+ String key = s[0];
+ String value = s[1];
+ jobManager.addVariableDefinition(key, value);
+ logger.config("set steering variable: " + key + "=" + value);
+ }
+ }
+ jobManager.setup(steeringStream);
+ jobManager.configure();
+ logger.config("LCSim job manager was successfully configured.");
+
+ // Name of detector.
+ String detectorName = null;
+ if (cl.hasOption("d")) {
+ detectorName = cl.getOptionValue("d");
+ logger.config("user set detector to " + detectorName);
+ }
+
+ // LCSim event builder which will be initialized within the loop based on the run number.
+ LCSimEventBuilder eventBuilder = null;
+
+ for (String evioFileName : cl.getArgs()) {
+ // EVIO input file.
+ File evioFile = new File(evioFileName);
+ if (!evioFile.exists()) {
+ throw new RuntimeException("EVIO file " + evioFile.getPath() + " does not exist.");
+ }
+ logger.info("Opening EVIO file " + evioFileName);
+
+ // EVIO reader.
+ EvioReader reader = null;
+ try {
+ reader = new EvioReader(evioFile);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ boolean firstEvent = true;
+ long time = 0; // in ms
+
+ // Loop over EVIO events, build LCSim events, process them, and then
+ // write events to disk.
+ fileLoop: while (maxEvents == 0 || nEvents < maxEvents) {
+ EvioEvent evioEvent = null;
+ try {
+ eventLoop: while (evioEvent == null) {
+ evioEvent = reader.nextEvent();
+ if (evioEvent == null) {
+ break fileLoop;
+ }
+ try {
+ reader.parseEvent(evioEvent);
+ } catch (Exception e) {
+ Logger.getLogger(EvioToLcio.class.getName()).log(Level.SEVERE, "Error reading EVIO event.", e);
+ if (!continueOnErrors) {
+ throw new RuntimeException(e);
+ } else {
+ continue eventLoop;
+ }
+ }
+ }
+
+ // Handlers for different event types.
+ if (EventConstants.isPreStartEvent(evioEvent)) {
+
+ int[] data = evioEvent.getIntData();
+ // int seconds = data[0];
+ int runNumber = data[1];
+
+ logger.info("got EVIO pre start event with run #" + runNumber);
+
+ // Has the event builder been initialized?
+ if (eventBuilder == null) {
+ // Initialize the event builder.
+ eventBuilder = setupEventBuilder(detectorName, runNumber);
+ }
+ }
+
+ // Setup state in the LCSimEventBuilder based on the EVIO event.
+ eventBuilder.readEvioEvent(evioEvent);
+
+ if (EventConstants.isEndEvent(evioEvent)) {
+ logger.info("got EVIO end event");
+ // int[] data = evioEvent.getIntData();
+ // int seconds = data[0];
+ // int nevents = data[2];
+ // calibListener.endRun(seconds, nevents);
+ } else if (EventConstants.isPhysicsEvent(evioEvent)) {
+
+ logger.finest("got EVIO physics event #" + evioEvent.getEventNumber());
+
+ // Is the event builder initialized?
+ if (eventBuilder == null) {
+ // This can happen if there are no pre-start events in the EVIO file.
+ throw new RuntimeException( "The LCSimEventBuilder was never setup.");
+ }
+
+ EventHeader lcioEvent = eventBuilder.makeLCSimEvent(evioEvent);
+ logger.finest("created LCIO event #" + lcioEvent.getEventNumber());
+
+ time = (lcioEvent.getTimeStamp() / 1000000);
+
+ if (firstEvent) {
+ logger.info("First physics event time: " + time / 1000 + " - " + new Date(time));
+ firstEvent = false;
+ }
+
+ logger.finest("processing LCIO event in LCSim");
+ jobManager.processEvent(lcioEvent);
+ logger.finest("done processing LCIO event in LCSim");
+ if (writer != null) {
+ writer.write(lcioEvent);
+ writer.flush();
+ logger.finest("wrote LCIO event #" + lcioEvent.getEventNumber());
+ }
+ }
+
+ } catch (Exception e) {
+ logger.log(Level.SEVERE, "Error in LCIO event processing.", e);
+ if (!continueOnErrors) {
+ throw new RuntimeException(e);
+ } else {
+ continue;
+ }
+ }
+ nEvents++;
+ }
+ logger.info("Last physics event time: " + time / 1000 + " - " + new Date(time));
+ try {
+ logger.fine("closing reader");
+ reader.close();
+ logger.fine("reader closed");
+ } catch (IOException e) {
+ logger.log(Level.SEVERE, "An IO error occurred.", e);
+ if (!continueOnErrors) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ logger.info("no more data");
+
+ logger.info("executing jobManager.finish");
+ jobManager.finish();
+ logger.info("jobManager is done");
+
+ if (writer != null) {
+ logger.info("closing LCIO writer");
+ try {
+ writer.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ logger.info("closed LCIO writer");
+ }
+
+ logger.info("job finished successfully");
+ }
+
+ /**
+ * Print the CLI usage and exit.
+ */
+ private void printUsage() {
+ System.out.println("EvioToLcio [options] [evioFiles]");
+ HelpFormatter help = new HelpFormatter();
+ help.printHelp(" ", options);
+ System.exit(1);
+ }
+
+ /**
+ * Setup and return the LCSimEventBuilder.
+ * @param detectorName The detector name to be assigned to the event builder.
+ * @param runNumber The run number which determines which event builder to use.
+ * @return The LCSimEventBuilder for the Test Run or Engineering Run.
+ */
+ private LCSimEventBuilder setupEventBuilder(String detectorName, int runNumber) {
+ LCSimEventBuilder eventBuilder = null;
+ // Is this run number not part of the Test Run?
+ if (runNumber > EvioToLcio.TEST_RUN_END_RUN) {
+ logger.info("using LCSimEngRunEventBuilder");
+ eventBuilder = new LCSimEngRunEventBuilder();
+ if (detectorName == null) {
+ detectorName = DEFAULT_ENG_RUN_DETECTOR;
+ }
+ } else {
+ // This looks like a Test Run file.
+ logger.info("using LCSimTestRunEventBuilder");
+ eventBuilder = new LCSimTestRunEventBuilder();
+ if (detectorName == null) {
+ detectorName = DEFAULT_TEST_RUN_DETECTOR;
+ }
+ }
+ eventBuilder.setDetectorName(detectorName);
+ ConditionsManager.defaultInstance().addConditionsListener(eventBuilder);
+ logger.config("initialized " + eventBuilder.getClass().getCanonicalName() + " with detector " + detectorName + " and run number " + runNumber);
+ return eventBuilder;
+ }
}
|
