Author: [log in to unmask] Date: Fri Nov 6 16:07:23 2015 New Revision: 3939 Log: Development work on crawler and related EVIO classes; remove some unused classes; merges from trunk. Added: java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/TridentMCFilter.java - copied unchanged from r3938, java/trunk/users/src/main/java/org/hps/users/meeg/TridentMCFilter.java Removed: java/branches/jeremy-dev/record-util/src/main/java/org/hps/job/ java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileMetadata.java java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileMetadataAdapter.java java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileMetadataProcessor.java Modified: java/branches/jeremy-dev/ (props changed) java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerConfig.java java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatUtilities.java java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileFormatFilter.java java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileMetadataReader.java java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/JSONUtilities.java java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EventTagBitMask.java java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioEventUtilities.java java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/KinkAnalysisDriver.java Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerConfig.java ============================================================================= --- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerConfig.java (original) +++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerConfig.java Fri Nov 6 16:07:23 2015 @@ -80,6 +80,11 @@ * A file to use for getting the timestamp date. */ private File timestampFile = null; + + /** + * Dry run for not actually executing updates. + */ + private boolean dryRun = false; /** * Get the set of runs that will be accepted for the job. @@ -209,9 +214,22 @@ * * @return this object */ - void setDatasetSite(final DatasetSite site) { + CrawlerConfig setDatasetSite(final DatasetSite site) { this.site = site; - } + return this; + } + + /** + * Enable dry run. + * + * @param dryRun set to <code>true</code> to enable dry run + * @return this object + */ + CrawlerConfig setDryRun(boolean dryRun) { + this.dryRun = dryRun; + return this; + } + /** * Set whether metadata extraction is enabled. @@ -302,4 +320,13 @@ File timestampFile() { return timestampFile; } + + /** + * Returns <code>true</code> if dry run which means no updates will occur. + * + * @return <code>true</code> if dry run + */ + boolean dryRun() { + return this.dryRun; + } } Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java ============================================================================= --- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java (original) +++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java Fri Nov 6 16:07:23 2015 @@ -28,6 +28,7 @@ import org.hps.datacat.client.DatacatClient; import org.hps.datacat.client.DatacatClientFactory; import org.hps.datacat.client.DatasetFileFormat; +import org.hps.datacat.client.DatasetSite; /** * Command line file crawler for populating the data catalog. @@ -146,6 +147,7 @@ OPTIONS.addOption("s", "site", true, "datacat site"); OPTIONS.addOption("t", "timestamp-file", true, "existing or new timestamp file name"); OPTIONS.addOption("x", "max-depth", true, "max depth to crawl"); + OPTIONS.addOption("D", "dry-run", false, "dry run which will not update the datacat"); } /** @@ -204,10 +206,10 @@ this.printUsage(); } - // Log level. + // Log level (only used for this class's logger). if (cl.hasOption("L")) { final Level level = Level.parse(cl.getOptionValue("L")); - LOGGER.config("setting log level to " + level); + LOGGER.config("log level " + level); LOGGER.setLevel(level); } @@ -221,7 +223,7 @@ throw new IllegalArgumentException("The specified path is not a directory."); } config.setRootDir(rootDir); - LOGGER.config("root dir set to " + config.rootDir()); + LOGGER.config("root dir " + config.rootDir()); } // Timestamp file for date filtering. @@ -313,6 +315,14 @@ } config.setAcceptRuns(acceptRuns); } + + // Dataset site (defaults to JLAB). + DatasetSite site = DatasetSite.JLAB; + if (cl.hasOption("s")) { + site = DatasetSite.valueOf(cl.getOptionValue("s")); + } + LOGGER.config("dataset site " + site); + config.setDatasetSite(site); } catch (final ParseException e) { throw new RuntimeException("Error parsing options.", e); @@ -379,25 +389,38 @@ private void updateDatacat(final FileSet fileSet) { final DatacatClient datacatClient = new DatacatClientFactory().createClient(); for (final DatasetFileFormat fileFormat : config.getFileFormats()) { - LOGGER.info("adding files to datacat with format " + fileFormat.name()); - for (final File file : fileSet.get(fileFormat)) { - - LOGGER.info("adding file " + file.getAbsolutePath() + " to datacat"); - - // Create metadata if this is enabled (takes awhile). + List<File> formatFiles = fileSet.get(fileFormat); + LOGGER.info("adding " + formatFiles.size() + " files with format " + fileFormat.name()); + for (final File file : formatFiles) { + + LOGGER.info("adding file " + file.getAbsolutePath()); + + // Create metadata if this is enabled (will take awhile). Map<String, Object> metadata = new HashMap<String, Object>(); if (config.enableMetaData()) { + LOGGER.info("creating metadata for " + file.getPath()); metadata = DatacatUtilities.createMetadata(file); } // Register file in the catalog. - DatacatUtilities.addFile(datacatClient, config.datacatFolder(), file, metadata); - } - } - } - - /** - * Walk the directory tree to find EVIO files for the runs that are being processed in the job. + if (!config.dryRun()) { + int response = DatacatUtilities.addFile(datacatClient, config.datacatFolder(), file, config.datasetSite(), metadata); + LOGGER.info("HTTP response " + response); + if (response >= 400) { + // Throw exception if response from server indicates an error occurred. + throw new RuntimeException("HTTP error code " + response + " received from server."); + } + } else { + LOGGER.info("update on " + file.getPath() + " skipped from dry run"); + } + } + LOGGER.info("successfully added " + formatFiles.size() + " " + fileFormat + " files"); + } + LOGGER.info("done updating datacat"); + } + + /** + * Walk the directory tree to find files for the runs that are being processed in the job. * * @param visitor the file visitor */ Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatUtilities.java ============================================================================= --- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatUtilities.java (original) +++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatUtilities.java Fri Nov 6 16:07:23 2015 @@ -9,6 +9,7 @@ import org.hps.datacat.client.DatasetDataType; import org.hps.datacat.client.DatasetFileFormat; import org.hps.datacat.client.DatasetSite; +import org.hps.record.evio.EvioFileUtilities; /** * Datacat utilities for the crawler. @@ -35,11 +36,11 @@ * @param file the file with the full path * @param metadata the file's meta data */ - static void addFile(final DatacatClient datacatClient, final String folder, final File file, - final Map<String, Object> metadata) { + static int addFile(final DatacatClient datacatClient, final String folder, final File file, + DatasetSite site, final Map<String, Object> metadata) { final DatasetFileFormat fileFormat = DatacatUtilities.getFileFormat(file); final DatasetDataType dataType = DatacatUtilities.getDataType(file); - DatacatUtilities.addFile(datacatClient, folder, file, metadata, fileFormat, dataType, DatasetSite.SLAC); + return DatacatUtilities.addFile(datacatClient, folder, file, metadata, fileFormat, dataType, site); } /** @@ -56,14 +57,14 @@ final Map<String, Object> metadata, final DatasetFileFormat fileFormat, final DatasetDataType dataType, final DatasetSite site) { - // Strip out cache dir prefix. - String filePath = file.getAbsolutePath(); - if (filePath.startsWith("/cache")) { - filePath = filePath.replace("/cache", ""); + // Get the cache file if this file is on JLAB MSS. + File actualFile = file; + if (EvioFileUtilities.isMssFile(file)) { + actualFile = EvioFileUtilities.getCachedFile(file); } // Add the dataset to the data catalog using the REST API. - final int response = client.addDataset(folder, dataType, filePath, file.length(), site, fileFormat, + final int response = client.addDataset(folder, dataType, file.getAbsolutePath(), actualFile.length(), site, fileFormat, file.getName(), metadata); return response; Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java ============================================================================= --- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java (original) +++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java Fri Nov 6 16:07:23 2015 @@ -2,14 +2,19 @@ import java.io.File; import java.io.IOException; -import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; +import org.hps.record.evio.EventTagBitMask; import org.hps.record.evio.EventTagConstant; import org.hps.record.evio.EvioEventUtilities; import org.hps.record.evio.EvioFileUtilities; +import org.hps.record.scalers.ScalersEvioProcessor; import org.jlab.coda.jevio.EvioEvent; import org.jlab.coda.jevio.EvioException; import org.jlab.coda.jevio.EvioReader; @@ -35,114 +40,209 @@ @Override public Map<String, Object> getMetadata(final File file) throws IOException { - Date startDate = null; - Date endDate = null; - int badEventCount = 0; + Integer firstTimestamp = null; + Integer lastTimestamp = null; + int badEvents = 0; int eventCount = 0; - int byteCount = 0; - boolean hasPrestart = false; - boolean hasEnd = false; - int[] eventIdData = null; + int epicsEvents = 0; + int scalerBanks = 0; + boolean prestart = false; + boolean end = false; + boolean go = false; + boolean blinded = true; Integer run = null; - Integer endEvent = null; - Integer startEvent = null; - Long lastTimestamp = null; - + Integer lastPhysicsEvent = null; + Integer firstPhysicsEvent = null; + + // Create map for counting event masks. + Map<EventTagBitMask, Integer> triggerCounts = new HashMap<EventTagBitMask, Integer>(); + for (EventTagBitMask mask : EventTagBitMask.values()) { + triggerCounts.put(mask, 0); + } + + // Scaler processor to check for scaler bank. + ScalersEvioProcessor scalersProcessor = new ScalersEvioProcessor(); + + // Get the file number from the name. + final int fileNumber = EvioFileUtilities.getSequenceFromName(file); + + // Only files divisible by 10 are unblinded (Eng Run 2015 scheme). + if (fileNumber % 10 == 0) { + blinded = false; + } + EvioReader evioReader = null; try { - evioReader = EvioFileUtilities.open(file, false); + + evioReader = EvioFileUtilities.open(file, true); + + EvioEvent evioEvent = null; + + // Event read loop. + while (true) { + + // Read in an EVIO event, trapping exceptions in case a parse error occurs. + boolean badEvent = false; + try { + evioEvent = evioReader.parseNextEvent(); + } catch (IOException | EvioException e) { + badEvent = true; + LOGGER.warning("bad EVIO event " + evioEvent.getEventNumber() + " could not be parsed"); + } + + // Increment bad event count and continue. + if (badEvent) { + badEvents++; + continue; + } + + // End of file. + if (evioEvent == null) { + LOGGER.info("end of file reached after " + eventCount + " events"); + break; + } + + // Process different event types. + if (EventTagConstant.PRESTART.equals(evioEvent)) { + + // File has PRESTART event. + LOGGER.info("found PRESTART event " + evioEvent.getEventNumber()); + prestart = true; + + // Set the run number from the PRESTART event. + final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent); + if (run == null) { + run = controlEventData[1]; + LOGGER.info("set run to " + run + " from PRESTART"); + } + + } else if (EventTagConstant.GO.equals(evioEvent)) { + + // File has GO event. + go = true; + + // Set the first timestamp from the GO event. + final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent); + firstTimestamp = controlEventData[0]; + LOGGER.info("set first timestamp to " + firstTimestamp + " from GO event " + evioEvent.getEventNumber()); + + } else if (EventTagConstant.END.equals(evioEvent)) { + + // File has END event. + LOGGER.info("got END event"); + end = true; + + // Set the last timestamp from the END event. + final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent); + lastTimestamp = controlEventData[0]; + LOGGER.info("set last timestamp " + lastTimestamp + " from END event " + evioEvent.getEventNumber()); + if (run == null) { + run = controlEventData[1]; + LOGGER.info("set run to " + run); + } + + } else if (EvioEventUtilities.isPhysicsEvent(evioEvent)) { + + // Event count on the file only includes physics events. + eventCount++; + + // Get head bank. + final int[] headBankData = EvioEventUtilities.getHeadBankData(evioEvent); + + // Set first timestamp from head bank. + if (firstTimestamp == null) { + if (headBankData[3] != 0) { + firstTimestamp = headBankData[3]; + LOGGER.info("set first timestamp to " + firstTimestamp + " from physics event " + evioEvent.getEventNumber()); + } + } + + // Set run number from head bank if not set already. + if (run == null) { + run = headBankData[1]; + LOGGER.info("set run to " + run + " from physics event " + evioEvent.getEventNumber()); + } + + // Get the event ID data. + final int[] eventIdData = EvioEventUtilities.getEventIdData(evioEvent); + if (eventIdData == null) { + throw new RuntimeException("The event ID data bank for event " + evioEvent.getEventNumber() + " is null."); + } + + // Set the first physics event from event ID data. + if (firstPhysicsEvent == null) { + firstPhysicsEvent = eventIdData[0]; + LOGGER.info("set start event " + firstPhysicsEvent + " from physics event " + evioEvent.getEventNumber()); + } + + // Set the last physics event from the event ID data. + lastPhysicsEvent = eventIdData[0]; + + // Set the last timestamp from head bank. + if (headBankData[3] != 0) { + lastTimestamp = headBankData[3]; + } + + // Increment scaler bank count if exists in event. + scalersProcessor.process(evioEvent); + if (scalersProcessor.getCurrentScalerData() != null) { + scalerBanks++; + } + + // Increment event mask counts for each type. + Set<EventTagBitMask> masks = EventTagBitMask.getEventTagBitMasks(evioEvent); + for (EventTagBitMask mask : masks) { + int count = triggerCounts.get(mask) + 1; + triggerCounts.put(mask, count); + } + + } else if (EventTagConstant.EPICS.equals(evioEvent)) { + // Count EPICS events. + ++epicsEvents; + } + } + } catch (final EvioException e) { + // Error reading the EVIO file. throw new IOException(e); - } - - final int fileNumber = EvioFileUtilities.getSequenceFromName(file); - - EvioEvent evioEvent = null; - - while (true) { - try { - evioEvent = evioReader.parseNextEvent(); - } catch (IOException | EvioException e) { - ++badEventCount; - continue; + } finally { + // Close the reader. + if (evioReader != null) { + try { + evioReader.close(); + } catch (IOException e) { + LOGGER.log(Level.WARNING, "error closing EVIO reader", e); + } } - if (evioEvent == null) { - break; - } - byteCount += evioEvent.getTotalBytes(); - if (EventTagConstant.PRESTART.equals(evioEvent)) { - LOGGER.info("found PRESTART"); - hasPrestart = true; - final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent); - final long timestamp = controlEventData[0] * 1000L; - startDate = new Date(timestamp); - LOGGER.info("set start date to " + startDate + " from PRESTART"); - if (run == null) { - run = controlEventData[1]; - LOGGER.info("set run to " + run); - } - } else if (EventTagConstant.END.equals(evioEvent)) { - LOGGER.info("found END event"); - hasEnd = true; - final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent); - final long timestamp = controlEventData[0] * 1000L; - endDate = new Date(timestamp); - LOGGER.info("set end date to " + endDate); - if (run == null) { - run = controlEventData[1]; - LOGGER.info("set run to " + run); - } - } else if (EvioEventUtilities.isPhysicsEvent(evioEvent)) { - final int[] headBankData = EvioEventUtilities.getHeadBankData(evioEvent); - if (startDate == null) { - if (headBankData[3] != 0) { - startDate = new Date(headBankData[3] * 1000L); - LOGGER.info("set start date to " + startDate + " from physics event"); - } - } - if (run == null) { - run = headBankData[1]; - LOGGER.info("set run to " + run + " from physics event"); - } - eventIdData = EvioEventUtilities.getEventIdData(evioEvent); - if (startEvent == null) { - startEvent = eventIdData[0]; - LOGGER.info("set start event " + startEvent); - } - if (headBankData[3] != 0) { - lastTimestamp = headBankData[3] * 1000L; - } - ++eventCount; - } - } - - // Set end date from last valid timestamp. - if (endDate == null) { - endDate = new Date(lastTimestamp); - LOGGER.info("set end date to " + endDate + " from last timestamp " + lastTimestamp); - } - - // Set end event number. - if (eventIdData != null) { - endEvent = eventIdData[0]; - LOGGER.info("set end event " + endEvent); - } - - final Map<String, Object> metaDataMap = new HashMap<String, Object>(); - + } + + // Create and fill the metadata map. + final Map<String, Object> metaDataMap = new LinkedHashMap<String, Object>(); metaDataMap.put("runMin", run); metaDataMap.put("runMax", run); metaDataMap.put("eventCount", eventCount); - metaDataMap.put("size", byteCount); - metaDataMap.put("fileNumber", fileNumber); - metaDataMap.put("badEventCount", badEventCount); - metaDataMap.put("endTimestamp", endDate.getTime()); - metaDataMap.put("startTimestamp", startDate.getTime()); - metaDataMap.put("startEvent", startEvent); - metaDataMap.put("endEvent", endEvent); - metaDataMap.put("hasEnd", hasEnd ? 1 : 0); - metaDataMap.put("hasPrestart", hasPrestart ? 1 : 0); - + metaDataMap.put("FILE", fileNumber); + metaDataMap.put("FIRST_TIMESTAMP", firstTimestamp); + metaDataMap.put("LAST_TIMESTAMP", lastTimestamp); + metaDataMap.put("FIRST_PHYSICS_EVENT", firstPhysicsEvent); + metaDataMap.put("LAST_PHYSICS_EVENT", lastPhysicsEvent); + metaDataMap.put("BAD_EVENTS", badEvents); + metaDataMap.put("EPICS_EVENTS", epicsEvents); + metaDataMap.put("SCALER_BANKS", scalerBanks); + metaDataMap.put("END", end); + metaDataMap.put("PRESTART", prestart); + metaDataMap.put("GO", go); + metaDataMap.put("BLINDED", blinded); + + // Add the event mask counts. + for (Entry<EventTagBitMask, Integer> entry : triggerCounts.entrySet()) { + // These two keys aren't working right so don't include them. + if (EventTagBitMask.PHYSICS != entry.getKey() && EventTagBitMask.SYNC != entry.getKey()) { + metaDataMap.put(entry.getKey().name(), entry.getValue()); + } + } + + // Return the completed metadata map. return metaDataMap; } } Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileFormatFilter.java ============================================================================= --- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileFormatFilter.java (original) +++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileFormatFilter.java Fri Nov 6 16:07:23 2015 @@ -3,7 +3,6 @@ import java.io.File; import java.io.FileFilter; import java.util.Set; -import java.util.logging.Logger; import org.hps.datacat.client.DatasetFileFormat; @@ -15,11 +14,6 @@ * @author Jeremy McCormick, SLAC */ public class FileFormatFilter implements FileFilter { - - /** - * Initialize the logger. - */ - private static final Logger LOGGER = Logger.getLogger(FileFormatFilter.class.getPackage().getName()); /** * The file format. @@ -48,13 +42,10 @@ */ @Override public boolean accept(final File pathname) { - LOGGER.info(pathname.getPath()); final DatasetFileFormat fileFormat = DatacatUtilities.getFileFormat(pathname); if (fileFormat != null) { - LOGGER.info("file " + pathname.getPath() + " has format " + fileFormat.name()); return formats.contains(fileFormat); } else { - LOGGER.info("rejected file " + pathname.getPath() + " with unknown format"); return false; } } Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileMetadataReader.java ============================================================================= --- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileMetadataReader.java (original) +++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileMetadataReader.java Fri Nov 6 16:07:23 2015 @@ -4,8 +4,19 @@ import java.io.IOException; import java.util.Map; - +/** + * Interface for reading metadata for the datacat from files. + * + * @author Jeremy McCormick, SLAC + */ public interface FileMetadataReader { + /** + * Create a metadata map with keys and values from the contents of a file. + * + * @param the input file for extracting metadata + * @return the metadata map + * @throws IOException if there is an error reading the file + */ public Map<String, Object> getMetadata(File file) throws IOException; } Modified: java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java ============================================================================= --- java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java (original) +++ java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java Fri Nov 6 16:07:23 2015 @@ -28,7 +28,7 @@ private static Logger LOGGER = Logger.getLogger(DatacatClientImpl.class.getPackage().getName()); /** - * The root directory (e.g. should be 'HPS'). + * The root directory (should be 'HPS'). */ private final String rootDir; @@ -46,7 +46,7 @@ * Create client with default parameters. */ DatacatClientImpl() { - this(DatacatConstants.BASE_URL, DatasetSite.SLAC, DatacatConstants.ROOT_DIR); + this(DatacatConstants.BASE_URL, DatasetSite.JLAB, DatacatConstants.ROOT_DIR); } /** @@ -99,14 +99,14 @@ final Map<String, Object> parameters = new HashMap<String, Object>(); parameters.put("dataType", dataType.toString()); parameters.put("resource", resource); - parameters.put("site", DatasetSite.SLAC.name()); + parameters.put("site", site); parameters.put("fileFormat", fileFormat.toString()); parameters.put("name", name); parameters.put("size", size); final JSONObject jsonDataset = JSONUtilities.createJSONDataset(parameters, metadata); final String urlLocation = url + "/datasets.json/" + this.rootDir + "/" + folder; - LOGGER.info("addDataset: " + urlLocation); - LOGGER.info("dataset JSON: " + jsonDataset.toString()); + LOGGER.info("add dataset " + urlLocation); + LOGGER.info("dataset JSON " + jsonDataset.toString()); return HttpUtilities.doPost(urlLocation, jsonDataset.toString()); } Modified: java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/JSONUtilities.java ============================================================================= --- java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/JSONUtilities.java (original) +++ java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/JSONUtilities.java Fri Nov 6 16:07:23 2015 @@ -84,16 +84,22 @@ JSONObject metadataObject = new JSONObject(); metadataObject.put("key", entry.getKey()); Object rawValue = entry.getValue(); + if (rawValue == null) { + throw new IllegalArgumentException("The metadata key " + entry.getKey() + " has a null value."); + } if (rawValue instanceof String) { metadataObject.put("type", "string"); } else if (rawValue instanceof Integer | rawValue instanceof Long) { metadataObject.put("type", "integer"); } else if (rawValue instanceof Float | rawValue instanceof Double) { metadataObject.put("type", "decimal"); + } else if (rawValue instanceof Boolean) { + metadataObject.put("type", "integer"); + rawValue = (Boolean)rawValue ? 1 : 0; } else { - throw new IllegalArgumentException("Do not know how to handle type: " + rawValue.getClass().getName()); - } - metadataObject.put("value", entry.getValue()); + throw new IllegalArgumentException("Metadata value " + rawValue + " with key " + entry.getKey() + " has unknown type."); + } + metadataObject.put("value", rawValue); array.put(metadataObject); } return array; Modified: java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EventTagBitMask.java ============================================================================= --- java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EventTagBitMask.java (original) +++ java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EventTagBitMask.java Fri Nov 6 16:07:23 2015 @@ -20,7 +20,7 @@ /** Pair 1 trigger. */ PAIRS1(3), /** Physics event. */ - PHYSICS(7), + PHYSICS(7), // FIXME: Doesn't work! /** Pulser triggered event. */ PULSER(5), /** Single 0 trigger. */ Modified: java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioEventUtilities.java ============================================================================= --- java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioEventUtilities.java (original) +++ java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioEventUtilities.java Fri Nov 6 16:07:23 2015 @@ -13,9 +13,6 @@ import org.hps.conditions.database.DatabaseConditionsManager; import org.hps.record.daqconfig.EvioDAQParser; -import org.hps.record.epics.EpicsData; -import org.hps.record.epics.EpicsHeader; -import org.hps.record.scalers.ScalerData; import org.jlab.coda.jevio.BaseStructure; import org.jlab.coda.jevio.EvioEvent; import org.lcsim.conditions.ConditionsManager.ConditionsNotFoundException; Modified: java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java ============================================================================= --- java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java (original) +++ java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java Fri Nov 6 16:07:23 2015 @@ -28,19 +28,23 @@ /** * Get a cached file path, assuming that the input file path is on the JLAB MSS e.g. it starts with "/mss". + * If the file is not on the JLAB MSS an error will be thrown. + * <p> + * If the file is already on the cache disk just return the same file. * - * @param file the MSS file path + * @param mssFile the MSS file path * @return the cached file path (prepends "/cache" to the path) * @throws IllegalArgumentException if the file is not on the MSS (e.g. path does not start with "/mss") */ - public static File getCachedFile(final File file) { - if (!isMssFile(file)) { - throw new IllegalArgumentException("File " + file.getPath() + " is not on the JLab MSS."); + public static File getCachedFile(final File mssFile) { + if (!isMssFile(mssFile)) { + throw new IllegalArgumentException("File " + mssFile.getPath() + " is not on the JLab MSS."); } - if (isCachedFile(file)) { - throw new IllegalArgumentException("File " + file.getPath() + " is already on the cache disk."); - } - return new File("/cache" + file.getPath()); + File cacheFile = mssFile; + if (!isCachedFile(mssFile)) { + cacheFile = new File("/cache" + mssFile.getAbsolutePath()); + } + return cacheFile; } /** @@ -98,7 +102,7 @@ * @throws EvioException if there is an error reading the EVIO data */ public static EvioReader open(final File file) throws IOException, EvioException { - return open(file, false); + return open(file, true); } /** @@ -111,6 +115,7 @@ * @throws EvioException if there is an error reading the EVIO data */ public static EvioReader open(final File file, final boolean sequential) throws IOException, EvioException { + LOGGER.info("opening " + file.getPath() + " in " + (sequential ? "sequential" : "mmap" + " mode")); File openFile = file; if (isMssFile(file)) { openFile = getCachedFile(file); Modified: java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/KinkAnalysisDriver.java ============================================================================= --- java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/KinkAnalysisDriver.java (original) +++ java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/KinkAnalysisDriver.java Fri Nov 6 16:07:23 2015 @@ -78,7 +78,6 @@ // } // } // } - if (event.hasCollection(ReconstructedParticle.class, "AprimeBeamspotConstrained")) { List<ReconstructedParticle> particles = event.get(ReconstructedParticle.class, "AprimeBeamspotConstrained"); int nvertices = 0; @@ -101,47 +100,14 @@ for (MCParticle particle : MCParticles) { if (particle.getOrigin().magnitude() > 10.0) { - hardScatters.add(VecOp.neg(particle.getOrigin())); - } - } - + hardScatters.add(particle.getOrigin()); + } + } List<SimTrackerHit> trackerHits = event.get(SimTrackerHit.class, "TrackerHits"); // Map<MCParticle, List<SimTrackerHit>> hitMap = new HashMap<MCParticle, List<SimTrackerHit>>(); - Map<MCParticle, Map<Integer, SimTrackerHit>> trackMap = new HashMap<MCParticle, Map<Integer, SimTrackerHit>>(); - - for (SimTrackerHit hit : trackerHits) { -// List hitList = hitMap.get(hit.getMCParticle()); -// if (hitList == null) { -// hitList = new ArrayList<SimTrackerHit>(); -// hitMap.put(hit.getMCParticle(), hitList); -// } -// hitList.add(hit); - - Map<Integer, SimTrackerHit> layerMap = trackMap.get(hit.getMCParticle()); - if (layerMap == null) { - layerMap = new HashMap<Integer, SimTrackerHit>(); - trackMap.put(hit.getMCParticle(), layerMap); - } - int layer = hit.getIdentifierFieldValue("layer"); - if (layerMap.containsKey(layer)) { - boolean nearHardScatter = false; - for (Hep3Vector scatter : hardScatters) { - if (VecOp.add(hit.getPositionVec(), scatter).magnitude() < 5.0) { - nearHardScatter = true; - } - } - if (!nearHardScatter) { - hardScatters.add(VecOp.neg(hit.getPositionVec())); - } -// System.out.format("Double hit in layer %d, %s\n", layer, nearHardScatter ? "near hard scatter" : "not near hard scatter"); - if (layerMap.get(layer).getPathLength() < hit.getPathLength()) { - continue; - } - } - layerMap.put(layer, hit); - } + Map<MCParticle, Map<Integer, SimTrackerHit>> trackMap = makeTrackHitMap(trackerHits, hardScatters); List<MCParticle> particlesWithoutTracks = new ArrayList<MCParticle>(); for (MCParticle particle : trackMap.keySet()) { @@ -208,13 +174,12 @@ frontDP.fill(p1.magnitude() - p2.magnitude()); frontDT.fill(deflection12); - for (int i = 0; i < layers.size() - 1; i++) { SimTrackerHit hit = layerMap.get(layers.get(i)); boolean nearHardScatter = false; for (Hep3Vector scatter : hardScatters) { - if (VecOp.add(hit.getPositionVec(), scatter).magnitude() < 5.0) { + if (VecOp.sub(hit.getPositionVec(), scatter).magnitude() < 5.0) { nearHardScatter = true; } } @@ -268,7 +233,7 @@ } } - private double angle(SimTrackerHit hit1, SimTrackerHit hit2) { + private static double angle(SimTrackerHit hit1, SimTrackerHit hit2) { double y1 = hit2.getMCParticle().getOriginY(); // double z1 = hit2.getMCParticle().getOriginZ(); double s1 = hit2.getMCParticle().getProductionTime() * PhysicalConstants.c_light; @@ -286,7 +251,7 @@ return Math.asin((y2 - y1) / (s2 - s1)); } - private double angle(List<Integer> layers, Map<Integer, SimTrackerHit> layerMap, int layer1, int layer2) { + private static double angle(List<Integer> layers, Map<Integer, SimTrackerHit> layerMap, int layer1, int layer2) { SimTrackerHit hit1 = null; if (layer1 > 0) { hit1 = layerMap.get(layers.get(layer1 - 1)); @@ -296,7 +261,7 @@ return angle(hit1, hit2); } - private double deflection(Map<Integer, SimTrackerHit> layerMap, int layer1, int layer2) { + static double deflection(Map<Integer, SimTrackerHit> layerMap, int layer1, int layer2) { List<Integer> layers = new ArrayList<Integer>(layerMap.keySet()); Collections.sort(layers); @@ -304,6 +269,38 @@ double angle2 = angle(layers, layerMap, layer2, layer2 + 1); return (angle2 - angle1) * Math.signum(angle1); + } + + static Map<MCParticle, Map<Integer, SimTrackerHit>> makeTrackHitMap(List<SimTrackerHit> trackerHits, List<Hep3Vector> hardScatters) { + Map<MCParticle, Map<Integer, SimTrackerHit>> trackMap = new HashMap<MCParticle, Map<Integer, SimTrackerHit>>(); + + for (SimTrackerHit hit : trackerHits) { + Map<Integer, SimTrackerHit> layerMap = trackMap.get(hit.getMCParticle()); + if (layerMap == null) { + layerMap = new HashMap<Integer, SimTrackerHit>(); + trackMap.put(hit.getMCParticle(), layerMap); + } + int layer = hit.getIdentifierFieldValue("layer"); + if (layerMap.containsKey(layer)) { + if (hardScatters != null) { + boolean nearHardScatter = false; + for (Hep3Vector scatter : hardScatters) { + if (VecOp.sub(hit.getPositionVec(), scatter).magnitude() < 5.0) { + nearHardScatter = true; + } + } + if (!nearHardScatter) { + hardScatters.add(hit.getPositionVec()); + } + } +// System.out.format("Double hit in layer %d, %s\n", layer, nearHardScatter ? "near hard scatter" : "not near hard scatter"); + if (layerMap.get(layer).getPathLength() < hit.getPathLength()) { + continue; + } + } + layerMap.put(layer, hit); + } + return trackMap; } @Override