lcsim/src/org/lcsim/job
diff -u -r1.6 -r1.7
--- JobControlManager.java 11 Sep 2008 21:26:10 -0000 1.6
+++ JobControlManager.java 23 Sep 2008 22:37:45 -0000 1.7
@@ -8,9 +8,11 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
+import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -23,355 +25,430 @@
import org.jdom.Element;
import org.jdom.input.SAXBuilder;
import org.lcsim.util.Driver;
+import org.lcsim.util.cache.FileCache;
import org.lcsim.util.loop.LCIOEventSource;
import org.lcsim.util.loop.LCSimLoop;
-// TODO: implement global parameters such as numberToRun as
-// T getParameter(name,Class<T>)
-//
-// TODO: handle additional parameter types such as SymmetricMatrix
-//
-// TODO: handle 2d array parameters
-//
-// TODO: turn on/off driverStatistics (does this require Driver changes?)
-//
-// TODO: expressions as in compact description for simple arithmetic with units
-//
-// TODO: handle constructor arguments (no idea how to do this)
-//
-public class JobControlManager {
-
- Map<String,Driver> driverMap;
- List<Driver> driverExec;
- List<File> inputFiles;
- Map<String,String> availableDrivers = new HashMap<String,String>();
- int maxEvents = -1;
-
- JobControlManager()
- {
- findAvailableDrivers();
- }
-
- public static void main(String args[])
- {
- if (args.length == 0)
- throw new RuntimeException("Missing XML input file!");
- File xmlRunControlFile = new File(args[0]);
- if (!xmlRunControlFile.exists())
- throw new RuntimeException("File " + args[0] + " does not exist!");
- JobControlManager mgr = new JobControlManager();
- mgr.setup(xmlRunControlFile);
- mgr.run();
- }
-
- public void run()
- {
- LCSimLoop loop = new LCSimLoop();
- for (Driver driver : driverExec) {
- loop.add(driver);
- }
- try {
- loop.setLCIORecordSource(new LCIOEventSource("test",inputFiles));
- loop.loop(maxEvents,null);
- } catch (Exception x) {
- throw new RuntimeException(x);
- }
- loop.dispose();
- }
-
- // setup from embedded resource
- public void setup(String resource)
- {
- setup(JobControlManager.class.getResourceAsStream(resource));
- }
-
- // setup from file
- public void setup(File file)
- {
- try {
- setup((new FileInputStream(file)));
- } catch (FileNotFoundException x) {
- throw new RuntimeException(x);
- }
- }
-
- // setup from InputStream (primary setup routine)
- public void setup(InputStream in)
- {
- // clear data structures for next job
- clear();
-
- // build the steering xml document
- SAXBuilder builder = new SAXBuilder();
- Document doc = null;
- try {
- doc = builder.build(in);
- } catch (Exception x) {
- throw new RuntimeException(x);
- }
-
- // get the root element
- Element root = doc.getRootElement();
-
- // setup job control parameters (manually process xml!)
- Element control = root.getChild("control");
- if (control != null)
- {
- Element controlElement = control.getChild("numberOfEvents");
- if (controlElement != null)
- {
- maxEvents = Integer.valueOf(controlElement.getText());
- System.out.println("set maxEvents <" + maxEvents + ">");
- }
- }
-
- // get list of driver elements from the file
- List<Element> drivers = root.getChild("drivers").getChildren("driver");
-
- // process list of drivers
- for (Element driver : drivers)
- {
- // get the name of driver
- String name = driver.getAttributeValue("name");
-
- String type = driver.getAttributeValue("type");
-
- // translate from short name if applicable
- if (availableDrivers.get(type) != null)
- {
- type = availableDrivers.get(type);
- }
-
- // get the class of driver
- Class driverClass;
-
- // the BeanInfo for this class
- BeanInfo beaninfo;
-
- // the constructor for this class
- Constructor constructor;
-
- // instantiated object
- Driver newDriver;
-
- // a lot of exceptions to catch for introspection so just wrap
- // the next 4 gets in a try block
- try {
- driverClass = Class.forName(type);
-
- // get bean info for driver
- beaninfo = java.beans.Introspector.getBeanInfo(driverClass);
-
- // create a new instance of this driver
- newDriver = (Driver)driverClass.newInstance();
- } catch (Exception x) {
- throw new RuntimeException(x);
- }
-
- // get list of parameter elements
- List<Element> parameters = driver.getChildren();
-
- // process the parameters
- for (Element parameter : parameters)
- {
- // get parameter name
- String pname = parameter.getName();
-
- // find the correct property descriptor for this parameter
- PropertyDescriptor propFind = null;
- for (PropertyDescriptor prop : beaninfo.getPropertyDescriptors())
- {
- if (prop.getName().equals(pname))
- {
- propFind = prop;
- break;
- }
- }
-
- // found a valid property?
- if (propFind != null)
- {
- // get the class of the property
- Class propertyType = propFind.getPropertyType();
-
- // create Object for next Driver parameter
- //System.out.println("converting: " + propFind.getName());
- Object nextParameter = convertParameter(parameter.getText(), propertyType);
-
- // if got a valid parameter value, then invoke the setter with its value
- if (nextParameter != null) {
- Method propWrite = propFind.getWriteMethod();
- Object pargs[] = new Object[1];
- pargs[0] = nextParameter;
- try {
- propWrite.invoke(newDriver, pargs);
- } catch (Exception x) {
- throw new RuntimeException(x);
- }
- }
- else {
- throw new RuntimeException("Error! Failed to create parameter: " + parameter.getText());
- }
- }
- else {
- throw new RuntimeException("Error! No property descriptor found: " + pname);
- }
- }
-
- // add this driver to Driver map
- addDriver(name,newDriver);
- }
-
- // add driver references to list for execution order
- List<Element> exec = root.getChild("execute").getChildren("driver");
- for (Element execDriver : exec) {
- String driverName = execDriver.getAttributeValue("name");
- Driver driverFind = driverMap.get(driverName);
- if (driverFind != null)
- driverExec.add(driverFind);
- else
- throw new RuntimeException("driver not found: " + driverName);
- }
-
- // process list of input LCIO files
- List<Element> files = root.getChild("inputFiles").getChildren("file");
- for (Element file : files) {
- String fileLoc = file.getText();
- File nextFile = new File(fileLoc);
- inputFiles.add(nextFile);
- }
- }
-
- // reset internal data structures
- public void clear() {
- inputFiles = new ArrayList<File>();
- driverExec = new ArrayList<Driver>();
- driverMap = new LinkedHashMap<String,Driver>();
- }
-
- // generic parameter conversion to int, String, double, float
- private Object convertParameter(String value, Class propertyType) {
-
- //System.out.println("converting type: <" + propertyType.getName() +">");
-
- Object o = null;
-
- // int
- if (propertyType.equals(int.class)) {
- o = Integer.valueOf(value);
- }
- // String
- else if (propertyType.equals(String.class)) {
- o = value;
- }
- // double
- else if (propertyType.equals(double.class)) {
- o = Double.valueOf(value);
- }
- // float
- else if (propertyType.equals(float.class)) {
- o = Float.valueOf(value);
- }
- // boolean
- else if (propertyType.equals(boolean.class)) {
- o = Boolean.valueOf(value);
- }
- // Hep3Vector
- else if (propertyType.equals(Hep3Vector.class)) {
- StringTokenizer tokenize = new StringTokenizer(value);
- double x = Double.valueOf(tokenize.nextToken());
- double y = Double.valueOf(tokenize.nextToken());
- double z = Double.valueOf(tokenize.nextToken());
- o = new BasicHep3Vector(x,y,z);
- }
- // 1d array of doubles
- else if (propertyType.getName().equals("[D")) {
- StringTokenizer tokenize = new StringTokenizer(value);
- int size = tokenize.countTokens();
- double da[] = new double[size];
- int i = 0;
- while (tokenize.hasMoreTokens()) {
- da[i] = Double.valueOf(tokenize.nextToken());
- ++i;
- }
- o = da;
- }
- // 1d array of ints
- else if (propertyType.getName().equals("[I")) {
- StringTokenizer tokenize = new StringTokenizer(value);
- int size = tokenize.countTokens();
- int ia[] = new int[size];
- int i = 0;
- while (tokenize.hasMoreTokens()) {
- ia[i] = Integer.valueOf(tokenize.nextToken());
- ++i;
- }
- o = ia;
- }
- // 1d array of floats
- else if (propertyType.getName().equals("[F")) {
- StringTokenizer tokenize = new StringTokenizer(value);
- int size = tokenize.countTokens();
- float fa[] = new float[size];
- int i = 0;
- while (tokenize.hasMoreTokens()) {
- fa[i] = Float.valueOf(tokenize.nextToken());
- ++i;
- }
- o = fa;
- }
- // 1d array of strings
- else if (propertyType.getName().equals("[Ljava.lang.String;")) {
- StringTokenizer tokenize = new StringTokenizer(value);
- int size = tokenize.countTokens();
- String sa[] = new String[size];
- int i = 0;
- while (tokenize.hasMoreTokens()) {
- sa[i] = tokenize.nextToken();
- ++i;
- }
- o = sa;
- }
- // 1d array of booleans
- else if (propertyType.getName().equals("[Z")) {
- StringTokenizer tokenize = new StringTokenizer(value);
- int size = tokenize.countTokens();
- boolean ba[] = new boolean[size];
- int i = 0;
- while (tokenize.hasMoreTokens()) {
- ba[i] = Boolean.valueOf(tokenize.nextToken());
- ++i;
- }
- o = ba;
- }
-
- return o;
- }
-
- // add a driver with associated name
- private void addDriver(String name, Driver driver)
- {
- if (driverMap.containsKey(name))
- {
- throw new RuntimeException("ERROR: duplicate driver name: " + name);
- }
- driverMap.put(name, driver);
- }
-
- void findAvailableDrivers()
- {
- InputStream in = JobControlManager.class.getResourceAsStream("/META-INF/services/org.lcsim.util.Driver");
- if (in == null)
- {
- System.err.println("Warning! Driver services file not found.");
- return;
- }
- Scanner scan = new Scanner(in);
- while(scan.hasNext())
- {
- String fullName = scan.nextLine().trim();
- String shortName = fullName.substring(fullName.lastIndexOf(".")+1);
- availableDrivers.put(shortName,fullName);
- }
- }
+//TODO: handle additional parameter types such as SymmetricMatrix
+
+//TODO: handle 2d array parameters
+
+//TODO: turn on/off driverStatistics (does this require Driver changes?)
+
+//TODO: expressions as in compact description for simple arithmetic with units
+
+//TODO: handle constructor arguments (no idea how to do this)
+
+public class JobControlManager
+{
+ Map<String,Driver> driverMap;
+ List<Driver> driverExec;
+ List<File> inputFiles;
+ Map<String,String> availableDrivers = new HashMap<String,String>();
+ int maxEvents = -1;
+ File cacheDirectory = null;
+ FileCache fileCache = null;
+
+ JobControlManager()
+ {
+ findAvailableDrivers();
+ }
+
+ public static void main(String args[])
+ {
+ if (args.length == 0)
+ throw new RuntimeException("Missing XML input file!");
+ File xmlRunControlFile = new File(args[0]);
+ if (!xmlRunControlFile.exists())
+ throw new RuntimeException("File " + args[0] + " does not exist!");
+ JobControlManager mgr = new JobControlManager();
+ mgr.setup(xmlRunControlFile);
+ mgr.run();
+ }
+
+ public void run()
+ {
+ LCSimLoop loop = new LCSimLoop();
+ for (Driver driver : driverExec) {
+ loop.add(driver);
+ }
+ try
+ {
+ loop.setLCIORecordSource(new LCIOEventSource("test",inputFiles));
+ loop.loop(maxEvents,null);
+ }
+ catch (Exception x)
+ {
+ throw new RuntimeException(x);
+ }
+ loop.dispose();
+ }
+
+ // setup from embedded resource
+ public void setup(String resource)
+ {
+ setup(JobControlManager.class.getResourceAsStream(resource));
+ }
+
+ // setup from file
+ public void setup(File file)
+ {
+ try {
+ setup((new FileInputStream(file)));
+ }
+ catch (FileNotFoundException x)
+ {
+ throw new RuntimeException(x);
+ }
+ }
+
+ // setup from InputStream (primary setup routine)
+ public void setup(InputStream in)
+ {
+ // Clear data structures for next job.
+ clear();
+
+ // Build the XML steering document.
+ SAXBuilder builder = new SAXBuilder();
+ Document doc = null;
+ try {
+ doc = builder.build(in);
+ } catch (Exception x) {
+ throw new RuntimeException(x);
+ }
+
+ // Get the root element.
+ Element root = doc.getRootElement();
+
+ // Setup job control parameters.
+ Element control = root.getChild("control");
+ if (control != null)
+ {
+ Element controlElement = control.getChild("numberOfEvents");
+ if (controlElement != null)
+ {
+ maxEvents = Integer.valueOf(controlElement.getText());
+ System.out.println("set maxEvents <" + maxEvents + ">");
+ }
+
+ Element cacheDirElement = control.getChild("cacheDirectory");
+ if (cacheDirElement != null)
+ {
+ cacheDirectory = new File(cacheDirElement.getText());
+ if (!cacheDirectory.exists())
+ throw new RuntimeException("cacheDirectory does not exist at location: " + cacheDirElement.getText());
+ }
+ else
+ {
+ // Default to the current working dir.
+ cacheDirectory = new File(System.getProperties().get("user.dir").toString());
+ }
+ }
+
+ // Setup the file cache.
+ try {
+ fileCache = new FileCache();
+ fileCache.setCacheDirectory(cacheDirectory);
+ System.out.println("cache directory set to: " + fileCache.getCacheDirectory().getAbsolutePath());
+ }
+ catch (IOException x)
+ {
+ throw new RuntimeException(x);
+ }
+
+ // Get list of driver declarations from steering file.
+ List<Element> drivers = root.getChild("drivers").getChildren("driver");
+
+ // Loop over the driver declarations.
+ for (Element driver : drivers)
+ {
+ // Get the name of the Driver.
+ String name = driver.getAttributeValue("name");
+
+ // Get the fully qualified type of the Driver. ([packageName].[className])
+ String type = driver.getAttributeValue("type");
+
+ // Translate from a short name (optional).
+ if (availableDrivers.get(type) != null)
+ {
+ type = availableDrivers.get(type);
+ }
+
+ // Class of the Driver.
+ Class driverClass;
+
+ // BeanInfo for the Driver class.
+ BeanInfo beaninfo;
+
+ // Constructor for the class.
+ Constructor constructor;
+
+ // The actual Driver instance.
+ Driver newDriver;
+
+ // Get Class, BeanInfo, and Constructor, and setup new Driver instance.
+ try {
+ driverClass = Class.forName(type);
+ beaninfo = java.beans.Introspector.getBeanInfo(driverClass);
+ newDriver = (Driver)driverClass.newInstance();
+ } catch (Exception x) {
+ throw new RuntimeException(x);
+ }
+
+ // Get a list of Driver parameters.
+ List<Element> parameters = driver.getChildren();
+
+ // Process the parameters.
+ for (Element parameter : parameters)
+ {
+ // Get the parameter name.
+ String pname = parameter.getName();
+
+ // Find the property descriptor for this parameter.
+ PropertyDescriptor propFind = null;
+ for (PropertyDescriptor prop : beaninfo.getPropertyDescriptors())
+ {
+ if (prop.getName().equals(pname))
+ {
+ propFind = prop;
+ break;
+ }
+ }
+
+ // Found a valid property?
+ if (propFind != null)
+ {
+ // Get the class of the property.
+ Class propertyType = propFind.getPropertyType();
+
+ // Create an Object for next Driver parameter using conversion method.
+ Object nextParameter = convertParameter(parameter.getText(), propertyType);
+
+ // If got a valid parameter value, then invoke the setter with its value.
+ if (nextParameter != null) {
+ Method propWrite = propFind.getWriteMethod();
+ Object pargs[] = new Object[1];
+ pargs[0] = nextParameter;
+ try {
+ propWrite.invoke(newDriver, pargs);
+ } catch (Exception x) {
+ throw new RuntimeException(x);
+ }
+ }
+ else {
+ throw new RuntimeException("Error! Failed to create parameter: " + parameter.getText());
+ }
+ }
+ else {
+ throw new RuntimeException("Error! No property descriptor found: " + pname);
+ }
+ }
+
+ // Add this Driver to the map.
+ System.out.println("adding to Driver map: " + name + " - " + newDriver.getClass().getCanonicalName());
+ addDriver(name,newDriver);
+ }
+
+ // Make a list of Drivers to be executed from the execute list.
+ List<Element> exec = root.getChild("execute").getChildren("driver");
+ for (Element execDriver : exec) {
+ String driverName = execDriver.getAttributeValue("name");
+ Driver driverFind = driverMap.get(driverName);
+ if (driverFind != null)
+ driverExec.add(driverFind);
+ else
+ throw new RuntimeException("driver not found: " + driverName);
+ }
+
+ // Make list of input LCIO files.
+ List<Element> files = root.getChild("inputFiles").getChildren("file");
+ for (Element file : files) {
+ String fileLoc = file.getText();
+ File nextFile = new File(fileLoc);
+ if (!nextFile.exists())
+ {
+ throw new RuntimeException("File not found on local filesystem: " + fileLoc);
+ }
+ inputFiles.add(nextFile);
+ }
+
+ // Make list of input LCIO files with URLs. Uses the file cache.
+ List<Element> fileURLs = root.getChild("inputFiles").getChildren("fileURL");
+ for (Element fileURL : fileURLs)
+ {
+ try {
+ URL url = new URL(fileURL.getText());
+ File nextFile = fileCache.getCachedFile(url);
+ inputFiles.add(nextFile);
+ }
+ catch (Exception x)
+ {
+ throw new RuntimeException("Bad file URL: " + fileURL.getText());
+ }
+ }
+ }
+
+ /**
+ * Reset internal data structures for next job.
+ */
+ public void clear()
+ {
+ inputFiles = new ArrayList<File>();
+ driverExec = new ArrayList<Driver>();
+ driverMap = new LinkedHashMap<String,Driver>();
+ }
+
+ /**
+ * Converts String <code>value</code> to Java class <code>propertyType</code>.
+ * @param value The string value from the XML steering file.
+ * @param propertyType The type of the parameter.
+ * @return The value of <code>value</code> as Class <code>propertyType</code>.
+ */
+ private Object convertParameter(String value, Class propertyType) {
+
+ //System.out.println("converting type: <" + propertyType.getName() +">");
+
+ Object o = null;
+
+ // Single int.
+ if (propertyType.equals(int.class)) {
+ o = Integer.valueOf(value);
+ }
+ // Single String.
+ else if (propertyType.equals(String.class)) {
+ o = value;
+ }
+ // Single double.
+ else if (propertyType.equals(double.class)) {
+ o = Double.valueOf(value);
+ }
+ // Single float.
+ else if (propertyType.equals(float.class)) {
+ o = Float.valueOf(value);
+ }
+ // Single boolean.
+ else if (propertyType.equals(boolean.class)) {
+ o = Boolean.valueOf(value);
+ }
+ // Single Hep3Vector type.
+ else if (propertyType.equals(Hep3Vector.class)) {
+ StringTokenizer tokenize = new StringTokenizer(value);
+ double x = Double.valueOf(tokenize.nextToken());
+ double y = Double.valueOf(tokenize.nextToken());
+ double z = Double.valueOf(tokenize.nextToken());
+ o = new BasicHep3Vector(x,y,z);
+ }
+ // 1d array of doubles.
+ else if (propertyType.getName().equals("[D")) {
+ StringTokenizer tokenize = new StringTokenizer(value);
+ int size = tokenize.countTokens();
+ double da[] = new double[size];
+ int i = 0;
+ while (tokenize.hasMoreTokens()) {
+ da[i] = Double.valueOf(tokenize.nextToken());
+ ++i;
+ }
+ o = da;
+ }
+ // 1d array of ints.
+ else if (propertyType.getName().equals("[I")) {
+ StringTokenizer tokenize = new StringTokenizer(value);
+ int size = tokenize.countTokens();
+ int ia[] = new int[size];
+ int i = 0;
+ while (tokenize.hasMoreTokens()) {
+ ia[i] = Integer.valueOf(tokenize.nextToken());
+ ++i;
+ }
+ o = ia;
+ }
+ // 1d array of floats.
+ else if (propertyType.getName().equals("[F")) {
+ StringTokenizer tokenize = new StringTokenizer(value);
+ int size = tokenize.countTokens();
+ float fa[] = new float[size];
+ int i = 0;
+ while (tokenize.hasMoreTokens()) {
+ fa[i] = Float.valueOf(tokenize.nextToken());
+ ++i;
+ }
+ o = fa;
+ }
+ // 1d array of strings.
+ else if (propertyType.getName().equals("[Ljava.lang.String;")) {
+ StringTokenizer tokenize = new StringTokenizer(value);
+ int size = tokenize.countTokens();
+ String sa[] = new String[size];
+ int i = 0;
+ while (tokenize.hasMoreTokens()) {
+ sa[i] = tokenize.nextToken();
+ ++i;
+ }
+ o = sa;
+ }
+ // 1d array of booleans.
+ else if (propertyType.getName().equals("[Z")) {
+ StringTokenizer tokenize = new StringTokenizer(value);
+ int size = tokenize.countTokens();
+ boolean ba[] = new boolean[size];
+ int i = 0;
+ while (tokenize.hasMoreTokens()) {
+ ba[i] = Boolean.valueOf(tokenize.nextToken());
+ ++i;
+ }
+ o = ba;
+ }
+ // Single File type.
+ else if (propertyType.getName().equals(File.class))
+ {
+ o = new File(value);
+ }
+ // Single URL type.
+ else if (propertyType.getName().equals(URL.class))
+ {
+ try {
+ o = new URL(value);
+ }
+ catch (Exception x)
+ {
+ throw new RuntimeException(x);
+ }
+ }
+
+ return o;
+ }
+
+ /**
+ * Add a Driver to the Driver map.
+ * @param name The unique name of the Driver.
+ * @param driver The instance of the Driver.
+ */
+ private void addDriver(String name, Driver driver)
+ {
+ if (driverMap.containsKey(name))
+ {
+ throw new RuntimeException("ERROR: duplicate driver name: " + name);
+ }
+ driverMap.put(name, driver);
+ }
+
+ /**
+ * Sets up an internal list of Drivers if a services file exists.
+ */
+ void findAvailableDrivers()
+ {
+ InputStream in = JobControlManager.class.getResourceAsStream("/META-INF/services/org.lcsim.util.Driver");
+ if (in == null)
+ {
+ System.err.println("Warning! Driver services file not found.");
+ return;
+ }
+ Scanner scan = new Scanner(in);
+ while(scan.hasNext())
+ {
+ String fullName = scan.nextLine().trim();
+ String shortName = fullName.substring(fullName.lastIndexOf(".")+1);
+ availableDrivers.put(shortName,fullName);
+ }
+ }
}