Commit in hps-java/src/test/java/org/lcsim/hps/evio on MAIN | |||
Evio2LcioTest.java | +848 | added 1.1 |
add generic EVIO v3 to LCIO conversion test
diff -N Evio2LcioTest.java --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ Evio2LcioTest.java 11 Feb 2012 00:33:18 -0000 1.1 @@ -0,0 +1,848 @@
+package org.lcsim.hps.evio; + +import static java.lang.System.arraycopy; + +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Vector; + +import junit.framework.TestCase; + +import org.freehep.record.loop.LoopException; +import org.jlab.coda.jevio.BaseStructure; +import org.jlab.coda.jevio.BaseStructureHeader; +import org.jlab.coda.jevio.EvioEvent; +import org.jlab.coda.jevio.EvioException; +import org.jlab.coda.jevio.EvioFile; +import org.lcsim.conditions.ConditionsManager; +import org.lcsim.conditions.ConditionsManagerImplementation; +import org.lcsim.conditions.ConditionsReader; +import org.lcsim.event.EventHeader; +import org.lcsim.event.GenericObject; +import org.lcsim.event.LCRelation; +import org.lcsim.event.base.BaseLCRelation; +import org.lcsim.event.base.BaseLCSimEvent; +import org.lcsim.util.Driver; +import org.lcsim.util.cache.FileCache; +import org.lcsim.util.lcio.LCIOWriter; +import org.lcsim.util.loop.DummyConditionsConverter; +import org.lcsim.util.loop.DummyDetector; +import org.lcsim.util.loop.LCSimLoop; + +/** + * Test that generically converts an EVIO v3 file into a collection of LCIO GenericObjects. + * Each data bank is copied into a corresponding GenericObject which has an associated object with its bank metadata. + * No type conversion to LCIO classes is performed. This can be done in a subsequent step within the LCSim event loop. + * + * @author Jeremy McCormick + * @version $Id: Evio2LcioTest.java,v 1.1 2012/02/11 00:33:18 jeremy Exp $ + */ +public class Evio2LcioTest extends TestCase +{ + // Control flags for debugging. Change these by hand and recompile to use. + static private final boolean DEBUG = false; // Set to true for a lot of debug prints. + static private final boolean BREAK_AFTER_SINGLE_EVENT = false; // Set to true to stop after one event. + static private final boolean WRITE_XML = false; // Set to true to write the EVIO file as XML before the conversion begins. + + // Converted LCIO file name. + static final private String outputFilePath = "./EvioCnvTest.slcio"; + + // URL pointing to test data from Maurik, which was generated by the GEMC simulation. + static final private String testFileUrl = "http://www.lcsim.org/test/hps/hps_test_data_2011_nov_18_1.evio"; + + /** + * Convert an EVIO file to LCIO and then read them back in parallel to check the output. + */ + public void testEvio2Lcio() + { + // Setup the file cache. + FileCache cache = null; + File testFile = null; + try + { + cache = new FileCache(new File(".")); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + + // Get the test file to the local machine. + try + { + testFile = cache.getCachedFile(new URL(testFileUrl)); + } + catch (Exception x) + { + throw new RuntimeException(x); + } + + // Convert a test EVIO file to LCIO. + String[] args = new String[2]; + try + { + args[0] = testFile.getCanonicalPath(); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + args[1] = outputFilePath; + (new Evio2LcioConverter()).main(args); + + if (DEBUG) + { + System.out.println("-----------------------"); + } + + // Test that the LCIO data matches the EVIO. + readBackLcio(new File(outputFilePath), testFile); + } + + /** + * Example class for generically converting EVIO data to LCIO. + */ + private static class Evio2LcioConverter + { + String bankInfoCollName = "BankInfo"; + String rawDataCollName = "RawDataBank"; + String relCollName = "BankInfoRelations"; + + public Evio2LcioConverter() + {} + + /** + * Would use this constructor to change the names of the LCIO collections that are created. (Not currently used.) + * @param bankInfoCollName + * @param rawDataCollName + * @param relCollName + */ + public Evio2LcioConverter(String bankInfoCollName, String rawDataCollName, String relCollName) + { + this.bankInfoCollName = bankInfoCollName; + this.rawDataCollName = rawDataCollName; + this.relCollName = relCollName; + } + + public void main(String[] args) + { + // Use dummy detector for LCSim conditions to avoid errors. + setDummyDetector("DUMMY"); + + // Check and set user arguments. + if (args.length < 2) + throw new RuntimeException("Not enough arguments."); + String inFilePath = args[0]; + String outFilePath = args[1]; + + // Open the EVIO file for reading. + EvioFile evioFile = null; + try + { + evioFile = new EvioFile(inFilePath); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + + // Write out test XML file with EVIO structure and data. + if (WRITE_XML) + { + System.out.println("Writing EVIO data to XML file ..."); + evioFile.toXMLFile("EvioTest.xml"); + } + + // Read the first EVIO event outside the processing loop. + EvioEvent evioEvent = null; + int nread = 0; + try + { + evioEvent = evioFile.parseNextEvent(); + ++nread; + } + catch (EvioException x) + { + throw new RuntimeException(x); + } + + if (DEBUG) + { + System.out.println("Read first EVIO event OKAY."); + } + + // Open new LCIOWriter for converted output. + LCIOWriter writer = null; + try + { + writer = new LCIOWriter(outputFilePath); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + + if (DEBUG) + System.out.println(); + + // Loop over EVIO events. + while (evioEvent != null) + { + // Make a new LCIO event. + // FIXME EVIO event numbers all seem to be zero in the test file. + // FIXME Set run number. + BaseLCSimEvent lcioEvent = new BaseLCSimEvent(0, evioEvent.getEventNumber(), "DUMMY"); + + // GenericObject collection to fill with the raw data. + List<GenericObject> objectColl = new ArrayList<GenericObject>(); + + // LCRelation list to connect bank info with data. + List<LCRelation> bankRelColl = new ArrayList<LCRelation>(); + + // Map to represent bank tag and number info. + Map<String, GenericObject> bankIds = new HashMap<String, GenericObject>(); + + // Loop over the EVIO data banks. + Vector<BaseStructure> children = evioEvent.getChildren(); + for (BaseStructure topBank : children) + { + BaseStructureHeader topHeader = topBank.getHeader(); + if (topBank.getChildCount() > 0) + { + for (BaseStructure subBank : topBank.getChildren()) + { + BaseStructureHeader subHeader = subBank.getHeader(); + if (subBank.getChildCount() > 0) + { + for (BaseStructure dataBank : subBank.getChildren()) + { + BaseStructureHeader dataBankHeader = dataBank.getHeader(); + + // Make a concatenated ID for this bank. + String bankId = makeHeaderId(topHeader, subHeader, dataBankHeader); + + // Make a new GenericObject for this bank's info. + if (bankIds.get(bankId) == null) + { + int[] bankInfo = new int[6]; + bankInfo[0] = topHeader.getTag(); + bankInfo[1] = topHeader.getNumber(); + bankInfo[2] = subHeader.getTag(); + bankInfo[3] = subHeader.getNumber(); + bankInfo[4] = dataBankHeader.getTag(); + bankInfo[5] = dataBankHeader.getNumber(); + bankIds.put(bankId, new LcioDataBankInfo(bankInfo)); + } + + // Get the bank info. + GenericObject bankInfo = bankIds.get(bankId); + + //System.out.println("BankInfo: tag = " + bankInfo.getIntVal(0) + "; number = " + bankInfo.getIntVal(1)); + + if (DEBUG) + { + printOut(dataBank, System.out); + } + + // LCIO object to represent one bank of data. + LcioDataBank lcioData = null; + + // Float data. + if (dataBank.getFloatData() != null) + { + lcioData = new LcioDataBank(dataBank.getFloatData()); + } + // Int data. + if (dataBank.getIntData() != null) + { + lcioData = new LcioDataBank(dataBank.getIntData()); + } + // Double data. + if (dataBank.getDoubleData() != null) + { + lcioData = new LcioDataBank(dataBank.getDoubleData()); + } + + // Setup relation pointing from bank data to its info. + if (lcioData != null) + { + objectColl.add(lcioData); + LCRelation relation = new BaseLCRelation(lcioData, bankInfo); + bankRelColl.add(relation); + } + + if (DEBUG) + { + System.out.println(); + } + } + } + } + } + } + + // Put the data bank info into the LCIO event. + List<GenericObject> bankInfoList = new ArrayList<GenericObject>(bankIds.values()); + lcioEvent.put(bankInfoCollName, bankInfoList, GenericObject.class, 0); + + // Put the GenericObject object collection containing the block data into the event. + lcioEvent.put(rawDataCollName, objectColl, GenericObject.class, 0); + + // Put the relations into the event. + lcioEvent.put(relCollName, bankRelColl, LCRelation.class, 0); + + // Write out the LCIO event. + try + { + writer.write(lcioEvent); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + + // Check if need to break on one event. + if (BREAK_AFTER_SINGLE_EVENT) + break; + + // Read next EVIO event. + try + { + evioEvent = evioFile.parseNextEvent(); + ++nread; + } + catch (EvioException x) + { + throw new RuntimeException(x); + } + } + + try + { + writer.close(); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + + if (DEBUG) + { + System.out.println("Read " + nread + " EVIO events OKAY."); + } + } + } + + private static void readBackLcio(File lcioFile, File evioFile) + { + LCSimLoop loop = new LCSimLoop(); + loop.setDummyDetector("DUMMY"); + try + { + loop.setLCIORecordSource(lcioFile); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + LcioCheckDriver checkDriver = new LcioCheckDriver(); + try + { + checkDriver.setEvioFilePath(evioFile.getCanonicalPath()); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + loop.add(checkDriver); + try + { + loop.loop(-1, null); + } + catch (LoopException x) + { + throw new RuntimeException(x); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + } + + /** + * Read back the LCIO event and check that the data matches the corresponding EVIO file. + */ + static private class LcioCheckDriver extends Driver + { + // Number of events read. + int nread = 0; + + // Settable path to EVIO file. + String evioFilePath = null; + + // The EVIO file for comparison. + EvioFile evioFile = null; + + // The current EVIO event. + EvioEvent evioEvent = null; + + public LcioCheckDriver() + {} + + /** + * Set the path of the EVIO file to be checked against. + * @param evioFilePath The path to the EVIO file. + */ + public void setEvioFilePath(String evioFilePath) + { + this.evioFilePath = evioFilePath; + } + + public void startOfData() + { + // Open the EVIO file for reading. + if (evioFilePath == null) + { + throw new RuntimeException("The EVIO file path was not set!"); + } + try + { + evioFile = new EvioFile(evioFilePath); + } + catch (IOException x) + { + throw new RuntimeException(x); + } + nread = 0; + } + + // FIXME The names of the LCIO collections are hard-coded and correspond to the defaults from the converter class. + public void process(EventHeader event) + { + if (DEBUG) + { + System.out.println("Processing event #" + event.getEventNumber()); + } + + // Read in next EVIO event which should match up with this LCIO event. + try + { + evioEvent = evioFile.parseNextEvent(); + } + catch (EvioException x) + { + throw new RuntimeException(x); + } + + // Make a list of EVIO banks to check. Only the leaf banks with data are added to the list. + Vector<BaseStructure> children = evioEvent.getChildren(); + List<BaseStructure> evioBanks = new ArrayList<BaseStructure>(); + for (BaseStructure topBank : children) + { + if (topBank.getChildCount() > 0) + { + for (BaseStructure subBank : topBank.getChildren()) + { + if (subBank.getChildCount() > 0) + { + for (BaseStructure dataBank : subBank.getChildren()) + { + // Only add leaf banks containing some data. + if (dataBank.getFloatData() != null || dataBank.getDoubleData() != null || dataBank.getIntData() != null) + { + evioBanks.add(dataBank); + } + } + } + } + } + } + + if (DEBUG) + { + System.out.println("EVIO event has " + evioBanks.size() + " data banks."); + } + + // Get the GenericObject collection containing the bank data. + List<GenericObject> lcioDataColl = event.get(GenericObject.class, "RawDataBank"); + + // Get the list of relations to bank info. + List<LCRelation> infoRel = event.get(LCRelation.class, "BankInfoRelations"); + + if (DEBUG) + { + System.out.println("LCIO collection RawDataBank has " + lcioDataColl.size() + " generic objects."); + } + + // Check that number of banks and GenericObjects is the same. + assertEquals("Number of EVIO data banks and LCIO GenericObjects does not match.", evioBanks.size(), lcioDataColl.size()); + + // Loop over the LCIO GenericObject collection. + int ibank = 0; + for (GenericObject lcioObj : lcioDataColl) + { + // Get the next bank from the EVIO file. + BaseStructure evioBank = evioBanks.get(ibank); + + if (DEBUG) + { + System.out.println("GenericObject ..."); + System.out.println(" ndoubles = " + lcioObj.getNDouble()); + System.out.println(" nfloats = " + lcioObj.getNFloat()); + System.out.println(" nints = " + lcioObj.getNInt()); + printOut(lcioObj, System.out); + } + + // Find the info relation for this data. + LCRelation rel = findRelation(infoRel, lcioObj); + + // Check that the relation exists. + TestCase.assertTrue("Could not find corresponding bank info for GenericObject.", rel != null); + + // Get the bank info. + GenericObject bankInfo = (GenericObject)rel.getTo(); + + if (DEBUG) + { + System.out.println("LCIO BankInfo ..."); + printOut(bankInfo, System.out); + System.out.println("EVIO Header; tag = " + evioBank.getHeader().getTag() + "; number = " + evioBank.getHeader().getNumber()); + } + + // Check that data bank info matches the EVIO file. This just checks the last tag and number. + assertEquals("Bank tag does not match EVIO.", evioBank.getHeader().getTag(), bankInfo.getIntVal(4)); + assertEquals("Bank number does not match EVIO.", evioBank.getHeader().getNumber(), bankInfo.getIntVal(5)); + + // Check double values. + if (lcioObj.getNDouble() != 0) + { + double[] evioDoubleData = evioBank.getDoubleData(); + int ndouble = evioDoubleData.length; + assertEquals("Number of doubles in GenericObject and EVIO bank don't match.", ndouble, lcioObj.getNDouble()); + for (int i=0; i<ndouble; i++) + { + // Seems that we need a big tolerance for this to pass! + assertEquals("EVIO double value does not match LCIO.", evioDoubleData[i], lcioObj.getDoubleVal(i), 1e-2); + } + } + + // Check int values. + if (lcioObj.getNInt() != 0) + { + int[] evioIntData = evioBank.getIntData(); + int nint = evioIntData.length; + assertEquals("Number of ints in GenericObject and EVIO bank don't match.", nint, lcioObj.getNInt()); + for (int i=0; i<nint; i++) + { + assertEquals("EVIO int value does not match LCIO.", evioIntData[i], lcioObj.getIntVal(i), 1e-2); + } + } + + // Check float values. + if (lcioObj.getNFloat() != 0) + { + float[] evioFloatData = evioBank.getFloatData(); + int nfloat = evioFloatData.length; + assertEquals("Number of floats in GenericObject and EVIO bank don't match.", nfloat, lcioObj.getNInt()); + for (int i=0; i<nfloat; i++) + { + assertEquals("EVIO float value does not match LCIO.", evioFloatData[i], lcioObj.getDoubleVal(i)); + } + } + + // Increment EVIO bank number. + ++ibank; + } + if (DEBUG) + { + System.out.println("Read " + event.getEventNumber() + " LCIO events OKAY."); + } + ++nread; + } + + public void endOfData() + { + if (DEBUG) + { + System.out.println(this.getClass().getSimpleName() + " read " + nread + " events OKAY."); + } + evioFile.close(); + } + } + + /** + * Represents the tag and number values from a set of EVIO banks. + */ + private static class LcioDataBankInfo implements GenericObject + { + private static final int BANK_INFO_SIZE = 6; + int[] bankInfo = new int[BANK_INFO_SIZE]; + + public LcioDataBankInfo(int[] bankInfo) + { + this.bankInfo = bankInfo; + } + + public int getNInt() + { + return BANK_INFO_SIZE; + } + + public int getNFloat() + { + return 0; + } + + public int getNDouble() + { + return 0; + } + + public int getIntVal(int index) + { + return bankInfo[index]; + } + + public float getFloatVal(int index) + { + return 0; + } + + public double getDoubleVal(int index) + { + return 0; + } + + public boolean isFixedSize() + { + return true; + } + } + + /** + * Implementation of GenericObject to represent an EVIO data bank in LCIO. + */ + private static class LcioDataBank implements GenericObject + { + float[] floatVals = null; + double[] doubleVals = null; + int[] intVals = null; + + int nint = 0; + int nfloat = 0; + int ndouble = 0; + + public LcioDataBank(float[] floatVals) + { + this.floatVals = new float[floatVals.length]; + arraycopy(floatVals, 0, this.floatVals, 0, floatVals.length); + nfloat = this.floatVals.length; + if (DEBUG) + { + System.out.println("Made LcioDataBank with " + nfloat + " floats."); + } + } + + public LcioDataBank(int[] intVals) + { + this.intVals = new int[intVals.length]; + arraycopy(intVals, 0, this.intVals, 0, intVals.length); + nint = this.intVals.length; + if (DEBUG) + { + System.out.println("Made LcioDataBank with " + nint + " ints."); + } + } + + public LcioDataBank(double[] doubleVals) + { + this.doubleVals = new double[doubleVals.length]; + arraycopy(doubleVals, 0, this.doubleVals, 0, doubleVals.length); + ndouble = this.doubleVals.length; + if (DEBUG) + { + System.out.println("Made LcioDataBank with " + ndouble + " doubles."); + } + } + + public int getNInt() + { + return nint; + } + + public int getNFloat() + { + return nfloat; + } + + public int getNDouble() + { + return ndouble; + } + + public int getIntVal(int index) + { + return intVals[index]; + } + + public float getFloatVal(int index) + { + return floatVals[index]; + } + + public double getDoubleVal(int index) + { + return doubleVals[index]; + } + + public boolean isFixedSize() + { + return false; + } + } + + /** + * Print out GenericObject values. + * @param lcioObj The GenericObject to be printed. + * @param ps The PrintStream to be used. + */ + private static void printOut(GenericObject lcioObj, PrintStream ps) + { + if (lcioObj.getNDouble() != 0) + { + int ndouble = lcioObj.getNDouble(); + ps.print("[ "); + for (int i=0; i<ndouble; i++) + { + ps.print(lcioObj.getDoubleVal(i) + " "); + } + ps.println("]"); + } + if (lcioObj.getNInt() != 0) + { + int nint = lcioObj.getNInt(); + ps.print("[ "); + for (int i=0; i<nint; i++) + { + ps.print(lcioObj.getIntVal(i) + " "); + } + ps.println("]"); + } + if (lcioObj.getNFloat() != 0) + { + int nfloat = lcioObj.getNFloat(); + ps.print("[ "); + for (int i=0; i<nfloat; i++) + { + ps.print(lcioObj.getFloatVal(i) + " "); + } + ps.println("]"); + } + } + + /** + * Makes an ID hash from three EVIO headers. + * @param h1 The first header. + * @param h2 The second header. + * @param h3 The third header. + * @return An ID that sequentially concatenates the tag and number values of the headers. + */ + private static String makeHeaderId(BaseStructureHeader h1, BaseStructureHeader h2, BaseStructureHeader h3) + { + return Integer.toString(h1.getTag()) + Integer.toString(h1.getNumber()) + + Integer.toString(h2.getTag()) + Integer.toString(h2.getNumber()) + + Integer.toString(h3.getTag()) + Integer.toString(h3.getNumber()); + } + + /** + * Print out the values from an EVIO data bank. + * @param dataBank The EVIO bank. + * @param ps The PrintStream to use. + */ + private static void printOut(BaseStructure dataBank, PrintStream ps) + { + if (dataBank.getFloatData() != null) + { + ps.println("Data bank has " + dataBank.getFloatData().length + " floats."); + float[] floatData = dataBank.getFloatData(); + ps.print("[ "); + for (int i=0; i<floatData.length; i++) + { + ps.print(floatData[i] + " "); + } + ps.println("]"); + } + if (dataBank.getIntData() != null) + { + ps.println("Data bank has " + dataBank.getIntData().length + " ints."); + int[] intData = dataBank.getIntData(); + ps.print("[ "); + for (int i=0; i<intData.length; i++) + { + ps.print(intData[i] + " "); + } + ps.println("]"); + } + if (dataBank.getDoubleData() != null) + { + ps.println("Data bank has " + dataBank.getDoubleData().length + " doubles."); + double[] doubleData = dataBank.getDoubleData(); + ps.print("[ "); + for (int i=0; i<doubleData.length; i++) + { + ps.print(doubleData[i] + " "); + } + ps.println("]"); + } + if (dataBank.getByteData() != null) + { + ps.println("Data bank has " + dataBank.getByteData().length + " bytes."); + byte[] byteData = dataBank.getByteData(); + ps.print("[ "); + for (int i = 0; i < byteData.length; i++) + { + ps.print(byteData[i] + " "); + } + ps.println("]"); + } + } + + /** + * An (inefficient) method for finding a relation by its 'from' object. + * @param relations The list of all relations to check. + * @param o1 The from object to find. + * @return The matching LCRelation or null if does not exist. + */ + private static LCRelation findRelation(List<LCRelation> relations, Object o1) + { + LCRelation fnd = null; + for (LCRelation rel : relations) + { + if (rel.getFrom() == o1) + { + fnd = rel; + break; + } + } + return fnd; + } + + /** + * Copied from conditions system to setup dummy detector. + * @param detectorName The name of the detector (shouldn't actually matter). + */ + private static void setDummyDetector(String detectorName) + { + ConditionsManager cond = ConditionsManager.defaultInstance(); + ConditionsReader dummyReader = ConditionsReader.createDummy(); + ((ConditionsManagerImplementation)cond).setConditionsReader(dummyReader, detectorName); + DummyDetector detector = new DummyDetector(detectorName); + cond.registerConditionsConverter(new DummyConditionsConverter(detector)); + } +}
\ No newline at end of file
Use REPLY-ALL to reply to list
To unsubscribe from the LCD-CVS list, click the following link:
https://listserv.slac.stanford.edu/cgi-bin/wa?SUBED1=LCD-CVS&A=1