lcsim/src/org/lcsim/job
diff -u -r1.7 -r1.8
--- JobControlManager.java 23 Sep 2008 22:37:45 -0000 1.7
+++ JobControlManager.java 25 Sep 2008 22:16:16 -0000 1.8
@@ -8,18 +8,22 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.PrintStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.StringTokenizer;
+import java.util.Map.Entry;
import org.jdom.Document;
import org.jdom.Element;
@@ -29,139 +33,313 @@
import org.lcsim.util.loop.LCIOEventSource;
import org.lcsim.util.loop.LCSimLoop;
-//TODO: handle additional parameter types such as SymmetricMatrix
-
-//TODO: handle 2d array parameters
-
-//TODO: turn on/off driverStatistics (does this require Driver changes?)
-
-//TODO: expressions as in compact description for simple arithmetic with units
-
-//TODO: handle constructor arguments (no idea how to do this)
-
-public class JobControlManager
+/**
+ * The <code>JobControlManager</code> provides an XML frontend for running LCSim jobs.
+ * The command line syntax is <code>./bin/lcsim steeringFile.xml</code>.
+ * @author jeremym
+ * @verbose $id: $
+ */
+//TODO: additional parameter types (SymmetricMatrix, etc.)
+//TODO: 2d parameter arrays
+//TODO: simple expressions as in compact description
+//TODO: output of example XML steering file (e.g. like Marlin)
+//TODO: basic command line arg support (no args then expect steering file loc):
+// -s steeringFile
+// -e exampleSteeringOutput.xml
+// -d # debug only; don't actually run job.
+// etc.
+public class JobControlManager
{
- Map<String,Driver> driverMap;
+ Map<String, Driver> driverMap;
List<Driver> driverExec;
List<File> inputFiles;
- Map<String,String> availableDrivers = new HashMap<String,String>();
+ Map<String, String> availableDrivers = new HashMap<String, String>();
int maxEvents = -1;
- File cacheDirectory = null;
- FileCache fileCache = null;
+ File cacheDirectory;
+ FileCache fileCache;
+ LCSimClassLoader loader;
+ boolean printDriverStatistics;
+ boolean printSystemProperties;
+ boolean printUserClassPath;
+ boolean printDriversDetailed;
+ boolean verbose;
+ boolean wasSetup;
+ PrintStream logStream = System.out;
- JobControlManager()
+ public JobControlManager()
{
findAvailableDrivers();
}
+ /**
+ * Simple frontend method taking the name of the XML steering file.
+ * @param args
+ */
public static void main(String args[])
{
if (args.length == 0)
- throw new RuntimeException("Missing XML input file!");
+ throw new RuntimeException("Missing XML input file argument!");
File xmlRunControlFile = new File(args[0]);
- if (!xmlRunControlFile.exists())
- throw new RuntimeException("File " + args[0] + " does not exist!");
- JobControlManager mgr = new JobControlManager();
+ if (!xmlRunControlFile.exists())
+ throw new RuntimeException("The file " + args[0] + " does not exist!");
+ JobControlManager mgr = new JobControlManager();
mgr.setup(xmlRunControlFile);
- mgr.run();
- }
+ mgr.run();
+ }
- public void run()
+ /**
+ * User method for executing the job if JobControlManager is being explicitly instantiated rather than called through main method.
+ */
+ public void run()
{
+ if (!wasSetup)
+ throw new RuntimeException("Bailing on job! The setup() method was never called!");
+ if (printSystemProperties)
+ printSystemProperties(logStream);
LCSimLoop loop = new LCSimLoop();
- for (Driver driver : driverExec) {
+ for (Driver driver : driverExec)
+ {
loop.add(driver);
}
- try
+ try
{
- loop.setLCIORecordSource(new LCIOEventSource("test",inputFiles));
- loop.loop(maxEvents,null);
- }
- catch (Exception x)
+ loop.setLCIORecordSource(new LCIOEventSource("test", inputFiles));
+ PrintStream statsStream = null;
+ if (printDriverStatistics)
+ statsStream = logStream;
+ if (verbose)
+ {
+ logStream.println("Start time: " + (new Date()));
+ logStream.println();
+ }
+ loop.loop(maxEvents, statsStream);
+ if (verbose)
+ {
+ logStream.println();
+ logStream.println("End time: " + (new Date()));
+ }
+ } catch (Exception x)
{
throw new RuntimeException(x);
- }
+ }
loop.dispose();
- }
+ }
- // setup from embedded resource
- public void setup(String resource)
+ /**
+ * User setup method which uses an embedded resource as the steering file.
+ * @param resource
+ */
+ public void setup(String resource)
{
setup(JobControlManager.class.getResourceAsStream(resource));
}
- // setup from file
- public void setup(File file)
+ /**
+ * User setup method which uses a File.
+ * @param file
+ */
+ public void setup(File file)
{
- try {
+ try
+ {
setup((new FileInputStream(file)));
- }
- catch (FileNotFoundException x)
+ } catch (FileNotFoundException x)
{
throw new RuntimeException(x);
}
- }
+ }
- // setup from InputStream (primary setup routine)
- public void setup(InputStream in)
+ /**
+ * User setup method which uses a generic <code>InputStream</code> for steering.
+ * This stream should be a text stream of XML.
+ * @param in The XML input.
+ */
+ public void setup(InputStream in)
{
- // Clear data structures for next job.
- clear();
-
- // Build the XML steering document.
SAXBuilder builder = new SAXBuilder();
Document doc = null;
- try {
+ try
+ {
doc = builder.build(in);
- } catch (Exception x) {
+ }
+ catch (Exception x)
+ {
throw new RuntimeException(x);
}
+ setup(doc);
+ }
+
+ /**
+ * The primary setup method. Public setup methods all end up here.
+ * @param doc
+ */
+ private void setup(Document doc)
+ {
+ // Clear data structures if a previous job was run.
+ clear();
// Get the root element.
- Element root = doc.getRootElement();
+ Element root = doc.getRootElement();
// Setup job control parameters.
Element control = root.getChild("control");
if (control != null)
{
+ // Verbose mode.
+ Element verboseElement = control.getChild("verbose");
+ if (verboseElement != null)
+ verbose = Boolean.valueOf(verboseElement.getText());
+
+ // Log file setup needs to come first.
+ Element logFileElement = control.getChild("logFile");
+ String logFilePath = null;
+ if (logFileElement != null)
+ {
+ logFilePath = logFileElement.getText();
+ File logFile = new File(logFilePath);
+ try {
+ logFile.createNewFile();
+ logStream = new PrintStream(new FileOutputStream(logFile));
+
+ // Redirect standard out and err to log file.
+ System.setOut(logStream);
+ System.setErr(logStream);
+ }
+ catch (IOException x)
+ {
+ throw new RuntimeException(x);
+ }
+ }
+
+ // Print hello world message to appear at top of log.
+ if (verbose)
+ {
+ logStream.println(this.getClass().getCanonicalName() + " is initialized.");
+ logStream.println();
+ logStream.println("--- Job Control Parameters ---");
+ }
+
+ // Print log file path now that PrintStream is set.
+ if (verbose)
+ logStream.println("logFile = " + logFilePath);
+
+ // Number of events to run.
Element controlElement = control.getChild("numberOfEvents");
if (controlElement != null)
{
maxEvents = Integer.valueOf(controlElement.getText());
- System.out.println("set maxEvents <" + maxEvents + ">");
+ if (verbose)
+ logStream.println("numberOfEvents = " + maxEvents);
}
+ // The cache directory. Defaults to the current directory.
Element cacheDirElement = control.getChild("cacheDirectory");
if (cacheDirElement != null)
{
cacheDirectory = new File(cacheDirElement.getText());
if (!cacheDirectory.exists())
- throw new RuntimeException("cacheDirectory does not exist at location: " + cacheDirElement.getText());
- }
+ throw new RuntimeException("cacheDirectory does not exist at location: " + cacheDirElement.getText());
+ }
else
{
- // Default to the current working dir.
+ // Default to the current working dir if cacheDirectory was not set explicitly.
cacheDirectory = new File(System.getProperties().get("user.dir").toString());
- }
- }
+ }
- // Setup the file cache.
- try {
- fileCache = new FileCache();
- fileCache.setCacheDirectory(cacheDirectory);
- System.out.println("cache directory set to: " + fileCache.getCacheDirectory().getAbsolutePath());
+ if (verbose)
+ logStream.println("cacheDirectory = " + cacheDirectory);
+
+ Element printStatisticsElement = control.getChild("printDriverStatistics");
+ if (printStatisticsElement != null)
+ {
+ printDriverStatistics = Boolean.valueOf(printStatisticsElement.getText());
+ }
+
+ if (verbose)
+ logStream.println("printDriverStatistics = " + printDriverStatistics);
+
+ Element printSystemPropertiesElement = control.getChild("printSystemProperties");
+ if (printSystemPropertiesElement != null)
+ {
+ printSystemProperties = Boolean.valueOf(printSystemPropertiesElement.getText());
+ }
+
+ if (verbose)
+ logStream.println("printSystemProperties = " + printSystemProperties);
+
+ Element printUserClassPathElement = control.getChild("printUserClassPath");
+ if (printUserClassPathElement != null)
+ printUserClassPath = Boolean.valueOf(printUserClassPathElement.getText());
+
+ if (verbose)
+ logStream.println("printUserClassPath = " + printUserClassPath);
+
+ Element printDriversDetailedElement = control.getChild("printDriversDetailed");
+ if (printDriversDetailedElement != null)
+ printDriversDetailed = Boolean.valueOf(printDriversDetailedElement.getText());
+
+ if (verbose)
+ {
+ logStream.println("printDriversDetailed = " + printDriversDetailed);
+ logStream.println("--- End Job Control Parameters ---");
+ }
}
- catch (IOException x)
+
+ // Setup the class loader.
+ Element classpath = root.getChild("classpath");
+ List<URL> urlList = new ArrayList<URL>();
+ if (classpath != null)
{
- throw new RuntimeException(x);
- }
+ for (Object jarObject : classpath.getChildren("jar"))
+ {
+ Element jarElement = (Element) jarObject;
+ try
+ {
+ urlList.add((new File(jarElement.getText()).toURL()));
+ } catch (Exception x)
+ {
+ throw new RuntimeException("Bad jar location: " + jarElement.getText());
+ }
+ }
+ for (Object jarUrlObject : classpath.getChildren("jarUrl"))
+ {
+ Element jarUrlElement = (Element) jarUrlObject;
+ try
+ {
+ urlList.add(new URL(jarUrlElement.getText()));
+ } catch (Exception x)
+ {
+ throw new RuntimeException("Bad jar URL: " + jarUrlElement.getText());
+ }
+ }
+ }
+ URL[] urls = urlList.toArray(new URL[]
+ {});
+ loader = new LCSimClassLoader(urls);
+
+ // Print user's classpath entries.
+ if (printUserClassPath)
+ {
+ logStream.println();
+ logStream.println("-- Extra Classpath URLs --");
+ for (URL url : loader.getURLs())
+ {
+ logStream.println(url);
+ }
+ logStream.println("-- End Extra Classpath URLs --");
+ logStream.println();
+ }
// Get list of driver declarations from steering file.
List<Element> drivers = root.getChild("drivers").getChildren("driver");
// Loop over the driver declarations.
- for (Element driver : drivers)
- {
+ if (printDriversDetailed)
+ {
+ logStream.println("--- Drivers ---");
+ }
+ for (Element driver : drivers)
+ {
// Get the name of the Driver.
String name = driver.getAttributeValue("name");
@@ -187,11 +365,15 @@
Driver newDriver;
// Get Class, BeanInfo, and Constructor, and setup new Driver instance.
- try {
- driverClass = Class.forName(type);
+ try
+ {
+ driverClass = loader.loadClass(type);
+ if (printDriversDetailed)
+ System.out.println(driverClass.getCanonicalName());
beaninfo = java.beans.Introspector.getBeanInfo(driverClass);
- newDriver = (Driver)driverClass.newInstance();
- } catch (Exception x) {
+ newDriver = (Driver) driverClass.newInstance();
+ } catch (Exception x)
+ {
throw new RuntimeException(x);
}
@@ -199,16 +381,16 @@
List<Element> parameters = driver.getChildren();
// Process the parameters.
- for (Element parameter : parameters)
- {
+ for (Element parameter : parameters)
+ {
// Get the parameter name.
String pname = parameter.getName();
// Find the property descriptor for this parameter.
PropertyDescriptor propFind = null;
- for (PropertyDescriptor prop : beaninfo.getPropertyDescriptors())
+ for (PropertyDescriptor prop : beaninfo.getPropertyDescriptors())
{
- if (prop.getName().equals(pname))
+ if (prop.getName().equals(pname))
{
propFind = prop;
break;
@@ -216,42 +398,54 @@
}
// Found a valid property?
- if (propFind != null)
- {
+ if (propFind != null)
+ {
// Get the class of the property.
Class propertyType = propFind.getPropertyType();
// Create an Object for next Driver parameter using conversion method.
- Object nextParameter = convertParameter(parameter.getText(), propertyType);
+ Object nextParameter = convertParameter(parameter.getText(), propertyType);
// If got a valid parameter value, then invoke the setter with its value.
- if (nextParameter != null) {
+ if (nextParameter != null)
+ {
+
+ if (printDriversDetailed)
+ {
+ logStream.println(" " + pname + " = " + parameter.getText());
+ }
+
Method propWrite = propFind.getWriteMethod();
Object pargs[] = new Object[1];
pargs[0] = nextParameter;
- try {
+ try
+ {
propWrite.invoke(newDriver, pargs);
- } catch (Exception x) {
+ } catch (Exception x)
+ {
throw new RuntimeException(x);
}
+ } else
+ {
+ throw new RuntimeException("Error! Failed to create parameter " + pname + " with value: " + parameter.getText());
}
- else {
- throw new RuntimeException("Error! Failed to create parameter: " + parameter.getText());
- }
- }
- else {
+ } else
+ {
throw new RuntimeException("Error! No property descriptor found: " + pname);
}
}
-
- // Add this Driver to the map.
- System.out.println("adding to Driver map: " + name + " - " + newDriver.getClass().getCanonicalName());
- addDriver(name,newDriver);
- }
+ addDriver(name, newDriver);
+ }
+ if (printDriversDetailed)
+ {
+ logStream.println("--- End Drivers ---");
+ logStream.println();
+ }
// Make a list of Drivers to be executed from the execute list.
List<Element> exec = root.getChild("execute").getChildren("driver");
- for (Element execDriver : exec) {
+ for (Element execDriver : exec)
+ {
String driverName = execDriver.getAttributeValue("name");
Driver driverFind = driverMap.get(driverName);
if (driverFind != null)
@@ -260,139 +454,200 @@
throw new RuntimeException("driver not found: " + driverName);
}
+ // Setup the file cache.
+ try
+ {
+ fileCache = new FileCache();
+ fileCache.setCacheDirectory(cacheDirectory);
+ fileCache.setPrintStream(null);
+ } catch (IOException x)
+ {
+ throw new RuntimeException(x);
+ }
+
// Make list of input LCIO files.
+ if (verbose)
+ logStream.println("--- Input Files ---");
List<Element> files = root.getChild("inputFiles").getChildren("file");
- for (Element file : files) {
- String fileLoc = file.getText();
+ for (Element file : files)
+ {
+ String fileLoc = file.getText();
File nextFile = new File(fileLoc);
if (!nextFile.exists())
{
throw new RuntimeException("File not found on local filesystem: " + fileLoc);
}
inputFiles.add(nextFile);
+ if (verbose)
+ {
+ try
+ {
+ logStream.println(nextFile.getCanonicalPath());
+ } catch (IOException x)
+ {
+ throw new RuntimeException(x);
+ }
+ }
}
// Make list of input LCIO files with URLs. Uses the file cache.
- List<Element> fileURLs = root.getChild("inputFiles").getChildren("fileURL");
+ List<Element> fileURLs = root.getChild("inputFiles").getChildren("fileUrl");
for (Element fileURL : fileURLs)
{
- try {
+ try
+ {
URL url = new URL(fileURL.getText());
+ if (verbose)
+ {
+ logStream.println(url);
+ }
File nextFile = fileCache.getCachedFile(url);
inputFiles.add(nextFile);
- }
- catch (Exception x)
+ } catch (Exception x)
{
throw new RuntimeException("Bad file URL: " + fileURL.getText());
- }
- }
- }
+ }
+ }
+
+ if (verbose)
+ {
+ logStream.println("--- End Input Files ---");
+ logStream.println();
+ }
+
+ wasSetup = true;
+ }
/**
- * Reset internal data structures for next job.
+ * Reset all private variables.
*/
- public void clear()
+ private void clear()
{
inputFiles = new ArrayList<File>();
- driverExec = new ArrayList<Driver>();
- driverMap = new LinkedHashMap<String,Driver>();
+ driverExec = new ArrayList<Driver>();
+ driverMap = new LinkedHashMap<String, Driver>();
+ maxEvents = -1;
+ cacheDirectory = null;
+ fileCache = null;
+ loader = null;
+ printDriverStatistics = false;
+ printSystemProperties = false;
+ printUserClassPath = false;
+ printDriversDetailed = false;
+ verbose = false;
+ wasSetup = false;
+ logStream = System.out;
}
/**
- * Converts String <code>value</code> to Java class <code>propertyType</code>.
- * @param value The string value from the XML steering file.
+ * Converts a String <code>value</code> to a Java class specified by <code>propertyType</code>.
+ * @param value The string value from the XML parameter.
* @param propertyType The type of the parameter.
* @return The value of <code>value</code> as Class <code>propertyType</code>.
*/
- private Object convertParameter(String value, Class propertyType) {
-
- //System.out.println("converting type: <" + propertyType.getName() +">");
-
+ private Object convertParameter(String value, Class propertyType)
+ {
Object o = null;
// Single int.
- if (propertyType.equals(int.class)) {
+ if (propertyType.equals(int.class))
+ {
o = Integer.valueOf(value);
}
// Single String.
- else if (propertyType.equals(String.class)) {
+ else if (propertyType.equals(String.class))
+ {
o = value;
}
// Single double.
- else if (propertyType.equals(double.class)) {
+ else if (propertyType.equals(double.class))
+ {
o = Double.valueOf(value);
}
// Single float.
- else if (propertyType.equals(float.class)) {
+ else if (propertyType.equals(float.class))
+ {
o = Float.valueOf(value);
}
// Single boolean.
- else if (propertyType.equals(boolean.class)) {
+ else if (propertyType.equals(boolean.class))
+ {
o = Boolean.valueOf(value);
}
// Single Hep3Vector type.
- else if (propertyType.equals(Hep3Vector.class)) {
+ else if (propertyType.equals(Hep3Vector.class))
+ {
StringTokenizer tokenize = new StringTokenizer(value);
double x = Double.valueOf(tokenize.nextToken());
double y = Double.valueOf(tokenize.nextToken());
double z = Double.valueOf(tokenize.nextToken());
- o = new BasicHep3Vector(x,y,z);
+ o = new BasicHep3Vector(x, y, z);
}
// 1d array of doubles.
- else if (propertyType.getName().equals("[D")) {
+ else if (propertyType.getName().equals("[D"))
+ {
StringTokenizer tokenize = new StringTokenizer(value);
int size = tokenize.countTokens();
double da[] = new double[size];
int i = 0;
- while (tokenize.hasMoreTokens()) {
+ while (tokenize.hasMoreTokens())
+ {
da[i] = Double.valueOf(tokenize.nextToken());
++i;
}
o = da;
}
// 1d array of ints.
- else if (propertyType.getName().equals("[I")) {
+ else if (propertyType.getName().equals("[I"))
+ {
StringTokenizer tokenize = new StringTokenizer(value);
int size = tokenize.countTokens();
int ia[] = new int[size];
int i = 0;
- while (tokenize.hasMoreTokens()) {
+ while (tokenize.hasMoreTokens())
+ {
ia[i] = Integer.valueOf(tokenize.nextToken());
++i;
}
o = ia;
}
// 1d array of floats.
- else if (propertyType.getName().equals("[F")) {
+ else if (propertyType.getName().equals("[F"))
+ {
StringTokenizer tokenize = new StringTokenizer(value);
int size = tokenize.countTokens();
float fa[] = new float[size];
int i = 0;
- while (tokenize.hasMoreTokens()) {
+ while (tokenize.hasMoreTokens())
+ {
fa[i] = Float.valueOf(tokenize.nextToken());
++i;
}
o = fa;
}
// 1d array of strings.
- else if (propertyType.getName().equals("[Ljava.lang.String;")) {
+ else if (propertyType.getName().equals("[Ljava.lang.String;"))
+ {
StringTokenizer tokenize = new StringTokenizer(value);
int size = tokenize.countTokens();
String sa[] = new String[size];
int i = 0;
- while (tokenize.hasMoreTokens()) {
+ while (tokenize.hasMoreTokens())
+ {
sa[i] = tokenize.nextToken();
++i;
}
o = sa;
}
// 1d array of booleans.
- else if (propertyType.getName().equals("[Z")) {
+ else if (propertyType.getName().equals("[Z"))
+ {
StringTokenizer tokenize = new StringTokenizer(value);
int size = tokenize.countTokens();
boolean ba[] = new boolean[size];
int i = 0;
- while (tokenize.hasMoreTokens()) {
+ while (tokenize.hasMoreTokens())
+ {
ba[i] = Boolean.valueOf(tokenize.nextToken());
++i;
}
@@ -406,10 +661,10 @@
// Single URL type.
else if (propertyType.getName().equals(URL.class))
{
- try {
+ try
+ {
o = new URL(value);
- }
- catch (Exception x)
+ } catch (Exception x)
{
throw new RuntimeException(x);
}
@@ -423,9 +678,9 @@
* @param name The unique name of the Driver.
* @param driver The instance of the Driver.
*/
- private void addDriver(String name, Driver driver)
+ private void addDriver(String name, Driver driver)
{
- if (driverMap.containsKey(name))
+ if (driverMap.containsKey(name))
{
throw new RuntimeException("ERROR: duplicate driver name: " + name);
}
@@ -433,22 +688,33 @@
}
/**
- * Sets up an internal list of Drivers if a services file exists.
+ * Setup a list of available Drivers if a services file exists.
*/
- void findAvailableDrivers()
+ private void findAvailableDrivers()
{
InputStream in = JobControlManager.class.getResourceAsStream("/META-INF/services/org.lcsim.util.Driver");
if (in == null)
{
System.err.println("Warning! Driver services file not found.");
return;
- }
+ }
Scanner scan = new Scanner(in);
- while(scan.hasNext())
+ while (scan.hasNext())
{
String fullName = scan.nextLine().trim();
- String shortName = fullName.substring(fullName.lastIndexOf(".")+1);
- availableDrivers.put(shortName,fullName);
- }
+ String shortName = fullName.substring(fullName.lastIndexOf(".") + 1);
+ availableDrivers.put(shortName, fullName);
+ }
}
-}
+
+ private void printSystemProperties(PrintStream ps)
+ {
+ logStream.println("--- System Properties ---");
+ for (Entry<Object, Object> entry : System.getProperties().entrySet())
+ {
+ logStream.println(entry.getKey() + " = " + entry.getValue());
+ }
+ logStream.println("-- End System Properties");
+ logStream.println();
+ }
+}
\ No newline at end of file