projects/lcsim/trunk/job-manager/src/main/java/org/lcsim/job
--- projects/lcsim/trunk/job-manager/src/main/java/org/lcsim/job/JobControlManager.java 2014-04-03 22:00:16 UTC (rev 3080)
+++ projects/lcsim/trunk/job-manager/src/main/java/org/lcsim/job/JobControlManager.java 2014-04-03 22:00:58 UTC (rev 3081)
@@ -1,5 +1,7 @@
package org.lcsim.job;
+import hep.aida.ref.BatchAnalysisFactory;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -55,28 +57,37 @@
import org.lcsim.util.xml.JDOMExpressionFactory;
/**
+ * <p>
* This class provides a frontend for running and managing LCSim jobs using an XML steering format.
- *
+ * </p>
+ * <p>
* The command line syntax is:
- *
+ * </p>
+ * <p>
* <code>java -jar ./lib/lcsim-bin.jar steeringFile.xml [options]</code>
- *
+ * </p>
+ * <p>
* To see the available command line options with descriptions, run with "-h" as the only option.
+ * </p>
+ * <p>
+ * In the XML file, variables have the format <code>${variableName}</code>. No nested variable
+ * definitions are allowed (e.g. no variables within variables).
+ * </p>
+ * <p>
+ * Command-line parameters that can be defined using switches are overridden by the corresponding
+ * settings in the job XML file, if they are present. This means that if these parameters are to be
+ * taken from the CL, the matching settings should be left out of the XML job file. This is not the
+ * case, however, for input files specified by the "-i" option, which are appended to the ones
+ * listed in the steering file.
+ * </p>
*
- * In the XML file, variables have the format <code>${variableName}</code>. No nested variable definitions are
- * allowed (e.g. no variables within variables).
- *
- * Command-line parameters that can be defined using switches are overridden by the corresponding settings in
- * the job XML file, if they are present. This means that if these parameters are to be taken from the CL, the
- * matching settings should be left out of the XML job file. This is not the case, however, for input files
- * specified by the "-i" option, which are appended to the ones listed in the steering file.
- *
* @version $Id: JobControlManager.java,v 1.65 2013/02/14 22:30:07 jeremy Exp $
* @author Jeremy McCormick
*/
// TODO: Do proper logging using Java Logger class rather than just a PrintStream.
-public class JobControlManager
-{
+@SuppressWarnings({"unchecked", "rawtypes"})
+public class JobControlManager {
+
// The LCIO record loop.
private LCSimLoop loop;
@@ -99,6 +110,7 @@
private boolean performDryRun;
private boolean rewrite;
private boolean useSteeringResource;
+ private boolean headless;
// Settings effecting logging verbosity.
private boolean printInputFiles;
@@ -106,7 +118,7 @@
private boolean printSystemProperties;
private boolean printUserClassPath;
private boolean printDriversDetailed;
- //private boolean printVersion;
+ // private boolean printVersion;
private boolean verbose;
// File caching.
@@ -140,8 +152,7 @@
/**
* The default constructor.
*/
- public JobControlManager()
- {
+ public JobControlManager() {
try {
fileCache = new FileCache();
} catch (IOException x) {
@@ -155,8 +166,7 @@
*
* @param args The command line arguments.
*/
- public static void main(String args[])
- {
+ public static void main(String args[]) {
JobControlManager mgr = new JobControlManager();
if (args.length == 0) {
mgr.logStream.println("java -jar lcsim-bin.jar [options] steeringFile.xml");
@@ -173,8 +183,7 @@
*
* @return The command line options for the manager.
*/
- private static Options createCommandLineOptions()
- {
+ private static Options createCommandLineOptions() {
Options options = new Options();
options.addOption(new Option("p", true, "Load a properties file containing variable definitions"));
options.addOption(new Option("D", true, "Define a variable with form [name]=[value]"));
@@ -186,18 +195,19 @@
options.addOption(new Option("q", false, "Turn on quiet mode"));
options.addOption(new Option("i", true, "Add an LCIO input file to process"));
options.addOption(new Option("r", false, "Use a steering resource rather than a file"));
+ //options.addOption(new Option("b", false, "Run in headless mode in which plots will not be shown."));
return options;
}
/**
- * Parse command-line options and setup job state from them. This method calls {@link #setup(File)} to
- * load the steering paramters from an XML file, after processing other command line options. This method
- * is private so that callers must all use the {@link #main(String[])} routine as the primary entry point.
+ * Parse command-line options and setup job state from them. This method calls
+ * {@link #setup(File)} to load the steering paramters from an XML file, after processing other
+ * command line options. This method is private so that callers must all use the
+ * {@link #main(String[])} routine as the primary entry point.
*
* @param args The command line arguments.
*/
- private void parseCommandLineOptions(String args[])
- {
+ private void parseCommandLineOptions(String args[]) {
// Setup parser.
CommandLineParser parser = new PosixParser();
@@ -282,7 +292,7 @@
if (cl.hasOption("x")) {
this.performDryRun = true;
}
-
+
if (cl.hasOption("r")) {
this.useSteeringResource = true;
}
@@ -293,7 +303,7 @@
} else if (cl.getArgList().size() > 1) {
throw new RuntimeException("Too many extra arguments.");
}
-
+
// Local LCIO files to process.
if (cl.hasOption("i")) {
String[] files = cl.getOptionValues("i");
@@ -305,14 +315,19 @@
inputFiles.add(new File(fileName));
}
}
-
- // Steering argument points to either a file or embedded resource.
- String steering = (String) cl.getArgList().get(0);
+ // Run in headless mode in which plots will not show even if <code>IPlotter.show()</code> is called.
+ //if (cl.hasOption("b")) {
+ // System.setProperty("hep.aida.IAnalysisFactory", BatchAnalysisFactory.class.getName());
+ //}
+
+ // Steering argument points to either a file or embedded resource.
+ String steering = (String) cl.getArgList().get(0);
+
// Using an embedded resource in the jar for steering.
if (this.useSteeringResource) {
- setup(steering);
- // Steering from a local file.
+ setup(steering);
+ // Steering from a local file.
} else {
File xmlRunControlFile = new File(steering);
if (!xmlRunControlFile.exists()) {
@@ -323,14 +338,14 @@
}
/**
- * Add a variable definition to be substituted into the job's XML file. This method is public so that
- * caller's not using the CL can still define necessary variables for the steering file.
+ * Add a variable definition to be substituted into the job's XML file. This method is public
+ * so that caller's not using the CL can still define necessary variables for the steering
+ * file.
*
* @param key The variable name.
* @param value The variable's value.
*/
- public void addVariableDefinition(String key, String value)
- {
+ public void addVariableDefinition(String key, String value) {
if (verbose) {
logStream.println(key + " = " + value);
}
@@ -344,28 +359,24 @@
/**
* Execute a job using the current parameters.
*/
- public boolean run()
- {
+ public boolean run() {
// If setup was not called first, then abort the job.
if (!wasSetup) {
logStream.println("Aborting job! Setup was never called.");
return false;
}
- //if (printVersion) {
- // (new LCSimVersion()).printOut(logStream);
- // logStream.println();
- //}
+ // if (printVersion) {
+ // (new LCSimVersion()).printOut(logStream);
+ // logStream.println();
+ // }
// Dry run selected. No events will be processed here.
if (performDryRun) {
logStream.println("Executed dry run. No events processed!");
return false;
}
-
- // Setup lcsim loop before running.
- getLCSimLoop();
-
+
try {
// Add the LCIO files to the loop.
loop.setLCIORecordSource(new LCIOEventSource(this.getClass().getSimpleName(), inputFiles));
@@ -415,26 +426,17 @@
*
* @return The LCSimLoop.
*/
- public LCSimLoop getLCSimLoop()
- {
- if (loop == null) {
- loop = new LCSimLoop();
- // Add the drivers to the loop.
- for (Driver driver : driverExec) {
- loop.add(driver);
- }
- }
+ public LCSimLoop getLCSimLoop() {
return loop;
}
/**
- * Return a list of Drivers to be executed. This can be used from an external framework like JAS3. The
- * list will be empty unless the <code>setup()</code> method has been called.
+ * Return a list of Drivers to be executed. This can be used from an external framework like
+ * JAS3. The list will be empty unless the <code>setup()</code> method has been called.
*
* @return A <code>List</code> of <code>Drivers</code>.
*/
- public List<Driver> getDriverExecList()
- {
+ public List<Driver> getDriverExecList() {
return this.driverExec;
}
@@ -443,8 +445,7 @@
*
* @param file
*/
- public void setup(File file)
- {
+ public void setup(File file) {
try {
setup((new FileInputStream(file)));
} catch (FileNotFoundException x) {
@@ -453,10 +454,10 @@
}
/**
- * Setup job parameters from an embedded resource. This method calls {@link #setup(InputStream)}.
+ * Setup job parameters from an embedded resource. This method calls
+ * {@link #setup(InputStream)}.
*/
- public void setup(String resourceURL)
- {
+ public void setup(String resourceURL) {
setup(this.getClass().getResourceAsStream(resourceURL));
}
@@ -465,8 +466,7 @@
*
* @param in The XML input stream.
*/
- public void setup(InputStream in)
- {
+ public void setup(InputStream in) {
// Make the Document builder.
SAXBuilder builder = new SAXBuilder();
@@ -492,20 +492,19 @@
}
/**
- * Setup the job parameters from an XML Document. Though it isn't publically accessible, this method
- * contains the primary logic for setting up job parameters from an XML file. The other setup methods,
- * namely, {@link #setup(InputStream)}, {@link #setup(String)} and {@link #setup(File)}, all eventually
- * call this method.
+ * Setup the job parameters from an XML Document. Though it isn't publically accessible, this
+ * method contains the primary logic for setting up job parameters from an XML file. The other
+ * setup methods, namely, {@link #setup(InputStream)}, {@link #setup(String)} and
+ * {@link #setup(File)}, all eventually call this method.
*
* @param doc The lcsim recon XML document describing the job.
*/
- private void setup(Document doc)
- {
+ private void setup(Document doc) {
if (wasSetup) {
logStream.println("Ignoring call to setup as it was already called.");
return;
}
-
+
// Set the root element.
root = doc.getRootElement();
@@ -562,20 +561,18 @@
*
* @param doc The XML document.
*/
- private void substituteVariables(Document doc)
- {
+ private void substituteVariables(Document doc) {
substituteVariables(doc.getRootElement());
}
/**
- * Substitute values from the <code>variableMap</code> into an XML element and all its children,
- * recursively.
+ * Substitute values from the <code>variableMap</code> into an XML element and all its
+ * children, recursively.
*
* @param element The XML element.
* @throw RuntimeException If a variable does not exist in the <code>variableMap</code>.
*/
- private void substituteVariables(Element element)
- {
+ private void substituteVariables(Element element) {
String text = element.getTextNormalize();
if (text.length() != 0) {
// Create a new matcher.
@@ -623,13 +620,12 @@
}
/**
- * Rewrite the XML steering file after resolving variables (done externally). The output path is set by
- * command line argument to "-w".
+ * Rewrite the XML steering file after resolving variables (done externally). The output path
+ * is set by command line argument to "-w".
*
* @param doc The XML steering doc with variables substituted.
*/
- private void rewriteXMLSteering(Document doc)
- {
+ private void rewriteXMLSteering(Document doc) {
if (verbose) {
logStream.println("Rewriting XML to " + this.rewriteFile + " ...");
}
@@ -645,14 +641,18 @@
}
/**
- * Setup the drivers from the XML job file, adding them to the JobManager's <code>driverMap</code> .
+ * Setup the drivers from the XML job file, adding them to the JobManager's
+ * <code>driverMap</code> .
*/
- private void setupDrivers()
- {
+ private void setupDrivers() {
if (printDriversDetailed) {
logStream.println("--- Drivers ---");
}
+ // Instantiate the LCSimLoop here in case subsequently created Drivers want to override
+ // its default conditions implementation.
+ loop = new LCSimLoop();
+
// Loop over the list of driver elements.
List<Element> drivers = root.getChild("drivers").getChildren("driver");
for (Element driver : drivers) {
@@ -701,8 +701,8 @@
// The parameter's setter method that we will find.
Method setter = null;
- // The parameter's type that will be inferred from the method or is given by a type string in
- // the case of an overloaded method.
+ // The parameter's type that will be inferred from the method or is given by a type
+ // string in the case of an overloaded method.
Class propertyType = null;
// Get the parameter name.
@@ -713,30 +713,27 @@
List<Method> methodCandidates = new ArrayList<Method>();
for (Method method : methods) {
String propHack = method.getName().replaceFirst("set", "");
- // FIXME: Converting to lower case need not happen + users might not find desireable.
propHack = propHack.substring(0, 1).toLowerCase() + propHack.substring(1);
if (propHack.equals(pname)) {
methodCandidates.add(method);
}
- }
-
- // Found the single setter method.
- if (methodCandidates.size() == 1) {
+ } if (methodCandidates.size() == 1) {
+ // Found the single setter method.
setter = methodCandidates.get(0);
if (setter.getParameterTypes().length > 1) {
throw new RuntimeException("The set method has too many arguments for parameter: " + pname);
}
propertyType = setter.getParameterTypes()[0];
- }
- // Found several, overloaded methods. Try to disambiguate them.
- else if (methodCandidates.size() > 1) {
+ } else if (methodCandidates.size() > 1) {
+ // Found several, overloaded methods. Try to disambiguate them.
if (parameterElement.getAttribute("type") == null)
throw new RuntimeException("Parameter " + pname + " in Driver " + driverClass.getCanonicalName() + " is overloaded, but a type field is missing from the parameter's XML element.");
try {
// Try a primitive type first.
propertyType = getPrimitiveType(parameterElement.getAttribute("type").getValue());
- // If type is null, then parameter is an Object and not a primitive, or it is not a
+ // If type is null, then parameter is an Object and not a primitive, or it
+ // is not a
// valid type.
if (propertyType == null)
propertyType = Class.forName(parameterElement.getAttribute("type").getValue());
@@ -750,9 +747,8 @@
break;
}
}
- }
- // No method found. The parameter name is probably invalid.
- else if (methodCandidates.size() == 0) {
+ } else if (methodCandidates.size() == 0) {
+ // No method found. The parameter name is probably invalid.
throw new RuntimeException("Set method was not found: " + driverClass.getSimpleName() + "." + pname);
}
@@ -802,6 +798,11 @@
throw new RuntimeException("A Driver called " + driverName + " was not found.");
}
}
+
+ // Add the drivers to the LCSimLoop.
+ for (Driver driver : driverExec) {
+ loop.add(driver);
+ }
}
/**
@@ -810,16 +811,14 @@
* @param name The unique name of the Driver.
* @param driver The instance of the Driver.
*/
- private void addDriver(String name, Driver driver)
- {
+ private void addDriver(String name, Driver driver) {
if (driverMap.containsKey(name)) {
throw new RuntimeException("Duplicate driver name: " + name);
}
driverMap.put(name, driver);
}
- private void printSystemProperties(PrintStream ps)
- {
+ private void printSystemProperties(PrintStream ps) {
logStream.println("--- System Properties ---");
for (Entry<Object, Object> entry : System.getProperties().entrySet()) {
logStream.println(entry.getKey() + " = " + entry.getValue());
@@ -835,8 +834,7 @@
* @param fileList The list files to which new file will be appended.
* @return The file that was created from the text.
*/
- private File processFileText(String fileText, List<File> fileList)
- {
+ private File processFileText(String fileText, List<File> fileList) {
Element fileElement = new Element("file");
fileElement.setText(fileText.trim());
return processFileElement(fileElement, fileList);
@@ -849,8 +847,7 @@
* @param fileList List to append new <code>File</code>.
* @return The <code>File</code> object.
*/
- private File processFileElement(Element fileElement, List<File> fileList)
- {
+ private File processFileElement(Element fileElement, List<File> fileList) {
String fileLoc = processPath(fileElement.getText().trim());
File file = null;
@@ -888,8 +885,7 @@
* Setup the list of input files to be processed from the XML job file.
*/
@SuppressWarnings("unchecked")
- private void setupInputFiles()
- {
+ private void setupInputFiles() {
if (root.getChild("inputFiles") == null) {
logStream.println("No input files in XML file.");
// this.performDryRun = true;
@@ -914,12 +910,21 @@
}
String line = null;
try {
- // Read the next file, turn the text into an XML element, and process it using common method.
+ // Read the next file, turn the text into an XML element, and process it using
+ // common method.
while ((line = input.readLine()) != null) {
processFileText(line.trim(), inputFiles);
}
} catch (IOException x) {
throw new RuntimeException(x);
+ } finally {
+ if (input != null) {
+ try {
+ input.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
}
}
@@ -1001,22 +1006,20 @@
*
* @param verbose True to turn on verbose mode; false to turn on quiet mode.
*/
- private void setVerbose(boolean verbose)
- {
+ private void setVerbose(boolean verbose) {
this.verbose = verbose;
this.printInputFiles = verbose;
this.printDriversDetailed = verbose;
this.printDriverStatistics = verbose;
this.printSystemProperties = verbose;
this.printUserClassPath = verbose;
- //this.printVersion = verbose;
+ // this.printVersion = verbose;
}
/**
* Setup the job control parameters using the XML steering document.
*/
- private void setupJobControlParameters()
- {
+ private void setupJobControlParameters() {
Element control = root.getChild("control");
if (control == null)
@@ -1126,10 +1129,10 @@
if (verbose)
logStream.println("printUserClassPath = " + printUserClassPath);
- //Element printVersionElement = control.getChild("printVersion");
- //if (printVersionElement != null) {
- // printVersion = Boolean.valueOf(printVersionElement.getText());
- //}
+ // Element printVersionElement = control.getChild("printVersion");
+ // if (printVersionElement != null) {
+ // printVersion = Boolean.valueOf(printVersionElement.getText());
+ // }
Element printDriversDetailedElement = control.getChild("printDriversDetailed");
if (printDriversDetailedElement != null)
@@ -1145,8 +1148,7 @@
/**
* Setup the manager's class loader.
*/
- private void setupClassLoader()
- {
+ private void setupClassLoader() {
Element classpath = root.getChild("classpath");
List<URL> urlList = new ArrayList<URL>();
if (classpath != null) {
@@ -1205,8 +1207,7 @@
/**
* Setup the file cache.
*/
- private void setupFileCache()
- {
+ private void setupFileCache() {
if (cacheDirectory == null)
return;
try {
@@ -1221,8 +1222,7 @@
/**
* Create the constants from the XML file.
*/
- private void processConstants()
- {
+ private void processConstants() {
Element define = root.getChild("define");
if (define != null) {
for (Object o : define.getChildren()) {
@@ -1238,8 +1238,7 @@
/**
* Setup the system of units.
*/
- private void setupUnits()
- {
+ private void setupUnits() {
Constants constants = Constants.getInstance();
for (Entry<String, Double> unit : constants.entrySet()) {
factory.addConstant(unit.getKey(), unit.getValue());
@@ -1252,8 +1251,7 @@
* @param name The name of the type.
* @return The primitive type class.
*/
- private static Class getPrimitiveType(String name)
- {
+ private static Class getPrimitiveType(String name) {
if (name.equals("byte"))
return byte.class;
if (name.equals("short"))
@@ -1281,8 +1279,7 @@
* @param klass The class.
* @return A list of setter methods.
*/
- private List<Method> getSetterMethods(Class klass)
- {
+ private List<Method> getSetterMethods(Class klass) {
List<Method> methods = new ArrayList<Method>();
Class currentClass = klass;
while (currentClass != null) {
@@ -1297,14 +1294,13 @@
}
/**
- * Process the file path string to substitute in the user's home directory for the "~" character. This is
- * needed if running on Windows.
+ * Process the file path string to substitute in the user's home directory for the "~"
+ * character. This is needed if running on Windows.
*
* @param path The original path.
* @return The path with home dir substitution.
*/
- private String processPath(String path)
- {
+ private String processPath(String path) {
if (path.startsWith("~")) {
return path.replaceFirst("~", System.getProperty("user.home"));
} else {
@@ -1313,8 +1309,7 @@
}
// Wrap Drivers with a DriverAdapter. Only one will be created per job.
- public DriverAdapter getDriverAdapter()
- {
+ public DriverAdapter getDriverAdapter() {
if (driverAdapter == null) {
Driver topDriver = new Driver();
for (Driver driver : getDriverExecList()) {
@@ -1326,16 +1321,14 @@
}
// Process a single LCSim event using the DriverAdapter.
- public void processEvent(EventHeader event)
- {
+ public void processEvent(EventHeader event) {
getDriverAdapter().recordSupplied(new RecordEvent(loop, event));
}
/**
* Reset the class's state so another job can be run.
*/
- public synchronized void reset()
- {
+ public synchronized void reset() {
driverAdapter = null;
loop = null;
@@ -1365,7 +1358,7 @@
printSystemProperties = false;
printUserClassPath = false;
printDriversDetailed = false;
- //printVersion = false;
+ // printVersion = false;
verbose = false;
// File caching.
@@ -1388,33 +1381,27 @@
wasSetup = false;
}
- public void configure()
- {
+ public void configure() {
getDriverAdapter().start(null);
}
- public void reconfigure()
- {
+ public void reconfigure() {
getDriverAdapter().start(null);
}
- public void suspend()
- {
+ public void suspend() {
getDriverAdapter().suspend(null);
}
- public void finish()
- {
+ public void finish() {
getDriverAdapter().finish(null);
}
- public void setPerformDryRun(boolean d)
- {
+ public void setPerformDryRun(boolean d) {
this.performDryRun = d;
}
- private void checkConditions()
- {
+ private void checkConditions() {
// Get conditions element if it is there; otherwise return.
Element conditionsElement = root.getChild("conditions");
@@ -1462,11 +1449,13 @@
ConditionsCheckDriver d = createConditionsCheckDriver();
if (d != null) {
if (detectorsElement == null) {
- // No detector names provided, so individual detectors will have all conditions checked.
+ // No detector names provided, so individual detectors will have all conditions
+ // checked.
d.setCheckDetector(false);
d.setCheckConditions(true);
} else {
- // Detectors already checked by name so just check that detector names in the job are okay.
+ // Detectors already checked by name so just check that detector names in the job
+ // are okay.
d.setCheckConditions(false);
d.setCheckDetector(true);
}
@@ -1474,8 +1463,7 @@
}
}
- private ConditionsCheckDriver createConditionsCheckDriver()
- {
+ private ConditionsCheckDriver createConditionsCheckDriver() {
ConditionsCheckDriver driver = new ConditionsCheckDriver();
Element conditionsElement = root.getChild("conditions");
if (conditionsElement == null)