Author: [log in to unmask]
Date: Fri Nov 6 16:07:23 2015
New Revision: 3939
Log:
Development work on crawler and related EVIO classes; remove some unused classes; merges from trunk.
Added:
java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/TridentMCFilter.java
- copied unchanged from r3938, java/trunk/users/src/main/java/org/hps/users/meeg/TridentMCFilter.java
Removed:
java/branches/jeremy-dev/record-util/src/main/java/org/hps/job/
java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileMetadata.java
java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileMetadataAdapter.java
java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileMetadataProcessor.java
Modified:
java/branches/jeremy-dev/ (props changed)
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerConfig.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatUtilities.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileFormatFilter.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileMetadataReader.java
java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java
java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/JSONUtilities.java
java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EventTagBitMask.java
java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioEventUtilities.java
java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java
java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/KinkAnalysisDriver.java
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerConfig.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerConfig.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerConfig.java Fri Nov 6 16:07:23 2015
@@ -80,6 +80,11 @@
* A file to use for getting the timestamp date.
*/
private File timestampFile = null;
+
+ /**
+ * Dry run for not actually executing updates.
+ */
+ private boolean dryRun = false;
/**
* Get the set of runs that will be accepted for the job.
@@ -209,9 +214,22 @@
*
* @return this object
*/
- void setDatasetSite(final DatasetSite site) {
+ CrawlerConfig setDatasetSite(final DatasetSite site) {
this.site = site;
- }
+ return this;
+ }
+
+ /**
+ * Enable dry run.
+ *
+ * @param dryRun set to <code>true</code> to enable dry run
+ * @return this object
+ */
+ CrawlerConfig setDryRun(boolean dryRun) {
+ this.dryRun = dryRun;
+ return this;
+ }
+
/**
* Set whether metadata extraction is enabled.
@@ -302,4 +320,13 @@
File timestampFile() {
return timestampFile;
}
+
+ /**
+ * Returns <code>true</code> if dry run which means no updates will occur.
+ *
+ * @return <code>true</code> if dry run
+ */
+ boolean dryRun() {
+ return this.dryRun;
+ }
}
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java Fri Nov 6 16:07:23 2015
@@ -28,6 +28,7 @@
import org.hps.datacat.client.DatacatClient;
import org.hps.datacat.client.DatacatClientFactory;
import org.hps.datacat.client.DatasetFileFormat;
+import org.hps.datacat.client.DatasetSite;
/**
* Command line file crawler for populating the data catalog.
@@ -146,6 +147,7 @@
OPTIONS.addOption("s", "site", true, "datacat site");
OPTIONS.addOption("t", "timestamp-file", true, "existing or new timestamp file name");
OPTIONS.addOption("x", "max-depth", true, "max depth to crawl");
+ OPTIONS.addOption("D", "dry-run", false, "dry run which will not update the datacat");
}
/**
@@ -204,10 +206,10 @@
this.printUsage();
}
- // Log level.
+ // Log level (only used for this class's logger).
if (cl.hasOption("L")) {
final Level level = Level.parse(cl.getOptionValue("L"));
- LOGGER.config("setting log level to " + level);
+ LOGGER.config("log level " + level);
LOGGER.setLevel(level);
}
@@ -221,7 +223,7 @@
throw new IllegalArgumentException("The specified path is not a directory.");
}
config.setRootDir(rootDir);
- LOGGER.config("root dir set to " + config.rootDir());
+ LOGGER.config("root dir " + config.rootDir());
}
// Timestamp file for date filtering.
@@ -313,6 +315,14 @@
}
config.setAcceptRuns(acceptRuns);
}
+
+ // Dataset site (defaults to JLAB).
+ DatasetSite site = DatasetSite.JLAB;
+ if (cl.hasOption("s")) {
+ site = DatasetSite.valueOf(cl.getOptionValue("s"));
+ }
+ LOGGER.config("dataset site " + site);
+ config.setDatasetSite(site);
} catch (final ParseException e) {
throw new RuntimeException("Error parsing options.", e);
@@ -379,25 +389,38 @@
private void updateDatacat(final FileSet fileSet) {
final DatacatClient datacatClient = new DatacatClientFactory().createClient();
for (final DatasetFileFormat fileFormat : config.getFileFormats()) {
- LOGGER.info("adding files to datacat with format " + fileFormat.name());
- for (final File file : fileSet.get(fileFormat)) {
-
- LOGGER.info("adding file " + file.getAbsolutePath() + " to datacat");
-
- // Create metadata if this is enabled (takes awhile).
+ List<File> formatFiles = fileSet.get(fileFormat);
+ LOGGER.info("adding " + formatFiles.size() + " files with format " + fileFormat.name());
+ for (final File file : formatFiles) {
+
+ LOGGER.info("adding file " + file.getAbsolutePath());
+
+ // Create metadata if this is enabled (will take awhile).
Map<String, Object> metadata = new HashMap<String, Object>();
if (config.enableMetaData()) {
+ LOGGER.info("creating metadata for " + file.getPath());
metadata = DatacatUtilities.createMetadata(file);
}
// Register file in the catalog.
- DatacatUtilities.addFile(datacatClient, config.datacatFolder(), file, metadata);
- }
- }
- }
-
- /**
- * Walk the directory tree to find EVIO files for the runs that are being processed in the job.
+ if (!config.dryRun()) {
+ int response = DatacatUtilities.addFile(datacatClient, config.datacatFolder(), file, config.datasetSite(), metadata);
+ LOGGER.info("HTTP response " + response);
+ if (response >= 400) {
+ // Throw exception if response from server indicates an error occurred.
+ throw new RuntimeException("HTTP error code " + response + " received from server.");
+ }
+ } else {
+ LOGGER.info("update on " + file.getPath() + " skipped from dry run");
+ }
+ }
+ LOGGER.info("successfully added " + formatFiles.size() + " " + fileFormat + " files");
+ }
+ LOGGER.info("done updating datacat");
+ }
+
+ /**
+ * Walk the directory tree to find files for the runs that are being processed in the job.
*
* @param visitor the file visitor
*/
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatUtilities.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatUtilities.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatUtilities.java Fri Nov 6 16:07:23 2015
@@ -9,6 +9,7 @@
import org.hps.datacat.client.DatasetDataType;
import org.hps.datacat.client.DatasetFileFormat;
import org.hps.datacat.client.DatasetSite;
+import org.hps.record.evio.EvioFileUtilities;
/**
* Datacat utilities for the crawler.
@@ -35,11 +36,11 @@
* @param file the file with the full path
* @param metadata the file's meta data
*/
- static void addFile(final DatacatClient datacatClient, final String folder, final File file,
- final Map<String, Object> metadata) {
+ static int addFile(final DatacatClient datacatClient, final String folder, final File file,
+ DatasetSite site, final Map<String, Object> metadata) {
final DatasetFileFormat fileFormat = DatacatUtilities.getFileFormat(file);
final DatasetDataType dataType = DatacatUtilities.getDataType(file);
- DatacatUtilities.addFile(datacatClient, folder, file, metadata, fileFormat, dataType, DatasetSite.SLAC);
+ return DatacatUtilities.addFile(datacatClient, folder, file, metadata, fileFormat, dataType, site);
}
/**
@@ -56,14 +57,14 @@
final Map<String, Object> metadata, final DatasetFileFormat fileFormat, final DatasetDataType dataType,
final DatasetSite site) {
- // Strip out cache dir prefix.
- String filePath = file.getAbsolutePath();
- if (filePath.startsWith("/cache")) {
- filePath = filePath.replace("/cache", "");
+ // Get the cache file if this file is on JLAB MSS.
+ File actualFile = file;
+ if (EvioFileUtilities.isMssFile(file)) {
+ actualFile = EvioFileUtilities.getCachedFile(file);
}
// Add the dataset to the data catalog using the REST API.
- final int response = client.addDataset(folder, dataType, filePath, file.length(), site, fileFormat,
+ final int response = client.addDataset(folder, dataType, file.getAbsolutePath(), actualFile.length(), site, fileFormat,
file.getName(), metadata);
return response;
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java Fri Nov 6 16:07:23 2015
@@ -2,14 +2,19 @@
import java.io.File;
import java.io.IOException;
-import java.util.Date;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.logging.Level;
import java.util.logging.Logger;
+import org.hps.record.evio.EventTagBitMask;
import org.hps.record.evio.EventTagConstant;
import org.hps.record.evio.EvioEventUtilities;
import org.hps.record.evio.EvioFileUtilities;
+import org.hps.record.scalers.ScalersEvioProcessor;
import org.jlab.coda.jevio.EvioEvent;
import org.jlab.coda.jevio.EvioException;
import org.jlab.coda.jevio.EvioReader;
@@ -35,114 +40,209 @@
@Override
public Map<String, Object> getMetadata(final File file) throws IOException {
- Date startDate = null;
- Date endDate = null;
- int badEventCount = 0;
+ Integer firstTimestamp = null;
+ Integer lastTimestamp = null;
+ int badEvents = 0;
int eventCount = 0;
- int byteCount = 0;
- boolean hasPrestart = false;
- boolean hasEnd = false;
- int[] eventIdData = null;
+ int epicsEvents = 0;
+ int scalerBanks = 0;
+ boolean prestart = false;
+ boolean end = false;
+ boolean go = false;
+ boolean blinded = true;
Integer run = null;
- Integer endEvent = null;
- Integer startEvent = null;
- Long lastTimestamp = null;
-
+ Integer lastPhysicsEvent = null;
+ Integer firstPhysicsEvent = null;
+
+ // Create map for counting event masks.
+ Map<EventTagBitMask, Integer> triggerCounts = new HashMap<EventTagBitMask, Integer>();
+ for (EventTagBitMask mask : EventTagBitMask.values()) {
+ triggerCounts.put(mask, 0);
+ }
+
+ // Scaler processor to check for scaler bank.
+ ScalersEvioProcessor scalersProcessor = new ScalersEvioProcessor();
+
+ // Get the file number from the name.
+ final int fileNumber = EvioFileUtilities.getSequenceFromName(file);
+
+ // Only files divisible by 10 are unblinded (Eng Run 2015 scheme).
+ if (fileNumber % 10 == 0) {
+ blinded = false;
+ }
+
EvioReader evioReader = null;
try {
- evioReader = EvioFileUtilities.open(file, false);
+
+ evioReader = EvioFileUtilities.open(file, true);
+
+ EvioEvent evioEvent = null;
+
+ // Event read loop.
+ while (true) {
+
+ // Read in an EVIO event, trapping exceptions in case a parse error occurs.
+ boolean badEvent = false;
+ try {
+ evioEvent = evioReader.parseNextEvent();
+ } catch (IOException | EvioException e) {
+ badEvent = true;
+ LOGGER.warning("bad EVIO event " + evioEvent.getEventNumber() + " could not be parsed");
+ }
+
+ // Increment bad event count and continue.
+ if (badEvent) {
+ badEvents++;
+ continue;
+ }
+
+ // End of file.
+ if (evioEvent == null) {
+ LOGGER.info("end of file reached after " + eventCount + " events");
+ break;
+ }
+
+ // Process different event types.
+ if (EventTagConstant.PRESTART.equals(evioEvent)) {
+
+ // File has PRESTART event.
+ LOGGER.info("found PRESTART event " + evioEvent.getEventNumber());
+ prestart = true;
+
+ // Set the run number from the PRESTART event.
+ final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent);
+ if (run == null) {
+ run = controlEventData[1];
+ LOGGER.info("set run to " + run + " from PRESTART");
+ }
+
+ } else if (EventTagConstant.GO.equals(evioEvent)) {
+
+ // File has GO event.
+ go = true;
+
+ // Set the first timestamp from the GO event.
+ final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent);
+ firstTimestamp = controlEventData[0];
+ LOGGER.info("set first timestamp to " + firstTimestamp + " from GO event " + evioEvent.getEventNumber());
+
+ } else if (EventTagConstant.END.equals(evioEvent)) {
+
+ // File has END event.
+ LOGGER.info("got END event");
+ end = true;
+
+ // Set the last timestamp from the END event.
+ final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent);
+ lastTimestamp = controlEventData[0];
+ LOGGER.info("set last timestamp " + lastTimestamp + " from END event " + evioEvent.getEventNumber());
+ if (run == null) {
+ run = controlEventData[1];
+ LOGGER.info("set run to " + run);
+ }
+
+ } else if (EvioEventUtilities.isPhysicsEvent(evioEvent)) {
+
+ // Event count on the file only includes physics events.
+ eventCount++;
+
+ // Get head bank.
+ final int[] headBankData = EvioEventUtilities.getHeadBankData(evioEvent);
+
+ // Set first timestamp from head bank.
+ if (firstTimestamp == null) {
+ if (headBankData[3] != 0) {
+ firstTimestamp = headBankData[3];
+ LOGGER.info("set first timestamp to " + firstTimestamp + " from physics event " + evioEvent.getEventNumber());
+ }
+ }
+
+ // Set run number from head bank if not set already.
+ if (run == null) {
+ run = headBankData[1];
+ LOGGER.info("set run to " + run + " from physics event " + evioEvent.getEventNumber());
+ }
+
+ // Get the event ID data.
+ final int[] eventIdData = EvioEventUtilities.getEventIdData(evioEvent);
+ if (eventIdData == null) {
+ throw new RuntimeException("The event ID data bank for event " + evioEvent.getEventNumber() + " is null.");
+ }
+
+ // Set the first physics event from event ID data.
+ if (firstPhysicsEvent == null) {
+ firstPhysicsEvent = eventIdData[0];
+ LOGGER.info("set start event " + firstPhysicsEvent + " from physics event " + evioEvent.getEventNumber());
+ }
+
+ // Set the last physics event from the event ID data.
+ lastPhysicsEvent = eventIdData[0];
+
+ // Set the last timestamp from head bank.
+ if (headBankData[3] != 0) {
+ lastTimestamp = headBankData[3];
+ }
+
+ // Increment scaler bank count if exists in event.
+ scalersProcessor.process(evioEvent);
+ if (scalersProcessor.getCurrentScalerData() != null) {
+ scalerBanks++;
+ }
+
+ // Increment event mask counts for each type.
+ Set<EventTagBitMask> masks = EventTagBitMask.getEventTagBitMasks(evioEvent);
+ for (EventTagBitMask mask : masks) {
+ int count = triggerCounts.get(mask) + 1;
+ triggerCounts.put(mask, count);
+ }
+
+ } else if (EventTagConstant.EPICS.equals(evioEvent)) {
+ // Count EPICS events.
+ ++epicsEvents;
+ }
+ }
+
} catch (final EvioException e) {
+ // Error reading the EVIO file.
throw new IOException(e);
- }
-
- final int fileNumber = EvioFileUtilities.getSequenceFromName(file);
-
- EvioEvent evioEvent = null;
-
- while (true) {
- try {
- evioEvent = evioReader.parseNextEvent();
- } catch (IOException | EvioException e) {
- ++badEventCount;
- continue;
+ } finally {
+ // Close the reader.
+ if (evioReader != null) {
+ try {
+ evioReader.close();
+ } catch (IOException e) {
+ LOGGER.log(Level.WARNING, "error closing EVIO reader", e);
+ }
}
- if (evioEvent == null) {
- break;
- }
- byteCount += evioEvent.getTotalBytes();
- if (EventTagConstant.PRESTART.equals(evioEvent)) {
- LOGGER.info("found PRESTART");
- hasPrestart = true;
- final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent);
- final long timestamp = controlEventData[0] * 1000L;
- startDate = new Date(timestamp);
- LOGGER.info("set start date to " + startDate + " from PRESTART");
- if (run == null) {
- run = controlEventData[1];
- LOGGER.info("set run to " + run);
- }
- } else if (EventTagConstant.END.equals(evioEvent)) {
- LOGGER.info("found END event");
- hasEnd = true;
- final int[] controlEventData = EvioEventUtilities.getControlEventData(evioEvent);
- final long timestamp = controlEventData[0] * 1000L;
- endDate = new Date(timestamp);
- LOGGER.info("set end date to " + endDate);
- if (run == null) {
- run = controlEventData[1];
- LOGGER.info("set run to " + run);
- }
- } else if (EvioEventUtilities.isPhysicsEvent(evioEvent)) {
- final int[] headBankData = EvioEventUtilities.getHeadBankData(evioEvent);
- if (startDate == null) {
- if (headBankData[3] != 0) {
- startDate = new Date(headBankData[3] * 1000L);
- LOGGER.info("set start date to " + startDate + " from physics event");
- }
- }
- if (run == null) {
- run = headBankData[1];
- LOGGER.info("set run to " + run + " from physics event");
- }
- eventIdData = EvioEventUtilities.getEventIdData(evioEvent);
- if (startEvent == null) {
- startEvent = eventIdData[0];
- LOGGER.info("set start event " + startEvent);
- }
- if (headBankData[3] != 0) {
- lastTimestamp = headBankData[3] * 1000L;
- }
- ++eventCount;
- }
- }
-
- // Set end date from last valid timestamp.
- if (endDate == null) {
- endDate = new Date(lastTimestamp);
- LOGGER.info("set end date to " + endDate + " from last timestamp " + lastTimestamp);
- }
-
- // Set end event number.
- if (eventIdData != null) {
- endEvent = eventIdData[0];
- LOGGER.info("set end event " + endEvent);
- }
-
- final Map<String, Object> metaDataMap = new HashMap<String, Object>();
-
+ }
+
+ // Create and fill the metadata map.
+ final Map<String, Object> metaDataMap = new LinkedHashMap<String, Object>();
metaDataMap.put("runMin", run);
metaDataMap.put("runMax", run);
metaDataMap.put("eventCount", eventCount);
- metaDataMap.put("size", byteCount);
- metaDataMap.put("fileNumber", fileNumber);
- metaDataMap.put("badEventCount", badEventCount);
- metaDataMap.put("endTimestamp", endDate.getTime());
- metaDataMap.put("startTimestamp", startDate.getTime());
- metaDataMap.put("startEvent", startEvent);
- metaDataMap.put("endEvent", endEvent);
- metaDataMap.put("hasEnd", hasEnd ? 1 : 0);
- metaDataMap.put("hasPrestart", hasPrestart ? 1 : 0);
-
+ metaDataMap.put("FILE", fileNumber);
+ metaDataMap.put("FIRST_TIMESTAMP", firstTimestamp);
+ metaDataMap.put("LAST_TIMESTAMP", lastTimestamp);
+ metaDataMap.put("FIRST_PHYSICS_EVENT", firstPhysicsEvent);
+ metaDataMap.put("LAST_PHYSICS_EVENT", lastPhysicsEvent);
+ metaDataMap.put("BAD_EVENTS", badEvents);
+ metaDataMap.put("EPICS_EVENTS", epicsEvents);
+ metaDataMap.put("SCALER_BANKS", scalerBanks);
+ metaDataMap.put("END", end);
+ metaDataMap.put("PRESTART", prestart);
+ metaDataMap.put("GO", go);
+ metaDataMap.put("BLINDED", blinded);
+
+ // Add the event mask counts.
+ for (Entry<EventTagBitMask, Integer> entry : triggerCounts.entrySet()) {
+ // These two keys aren't working right so don't include them.
+ if (EventTagBitMask.PHYSICS != entry.getKey() && EventTagBitMask.SYNC != entry.getKey()) {
+ metaDataMap.put(entry.getKey().name(), entry.getValue());
+ }
+ }
+
+ // Return the completed metadata map.
return metaDataMap;
}
}
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileFormatFilter.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileFormatFilter.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileFormatFilter.java Fri Nov 6 16:07:23 2015
@@ -3,7 +3,6 @@
import java.io.File;
import java.io.FileFilter;
import java.util.Set;
-import java.util.logging.Logger;
import org.hps.datacat.client.DatasetFileFormat;
@@ -15,11 +14,6 @@
* @author Jeremy McCormick, SLAC
*/
public class FileFormatFilter implements FileFilter {
-
- /**
- * Initialize the logger.
- */
- private static final Logger LOGGER = Logger.getLogger(FileFormatFilter.class.getPackage().getName());
/**
* The file format.
@@ -48,13 +42,10 @@
*/
@Override
public boolean accept(final File pathname) {
- LOGGER.info(pathname.getPath());
final DatasetFileFormat fileFormat = DatacatUtilities.getFileFormat(pathname);
if (fileFormat != null) {
- LOGGER.info("file " + pathname.getPath() + " has format " + fileFormat.name());
return formats.contains(fileFormat);
} else {
- LOGGER.info("rejected file " + pathname.getPath() + " with unknown format");
return false;
}
}
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileMetadataReader.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileMetadataReader.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileMetadataReader.java Fri Nov 6 16:07:23 2015
@@ -4,8 +4,19 @@
import java.io.IOException;
import java.util.Map;
-
+/**
+ * Interface for reading metadata for the datacat from files.
+ *
+ * @author Jeremy McCormick, SLAC
+ */
public interface FileMetadataReader {
+ /**
+ * Create a metadata map with keys and values from the contents of a file.
+ *
+ * @param the input file for extracting metadata
+ * @return the metadata map
+ * @throws IOException if there is an error reading the file
+ */
public Map<String, Object> getMetadata(File file) throws IOException;
}
Modified: java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java
=============================================================================
--- java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java (original)
+++ java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java Fri Nov 6 16:07:23 2015
@@ -28,7 +28,7 @@
private static Logger LOGGER = Logger.getLogger(DatacatClientImpl.class.getPackage().getName());
/**
- * The root directory (e.g. should be 'HPS').
+ * The root directory (should be 'HPS').
*/
private final String rootDir;
@@ -46,7 +46,7 @@
* Create client with default parameters.
*/
DatacatClientImpl() {
- this(DatacatConstants.BASE_URL, DatasetSite.SLAC, DatacatConstants.ROOT_DIR);
+ this(DatacatConstants.BASE_URL, DatasetSite.JLAB, DatacatConstants.ROOT_DIR);
}
/**
@@ -99,14 +99,14 @@
final Map<String, Object> parameters = new HashMap<String, Object>();
parameters.put("dataType", dataType.toString());
parameters.put("resource", resource);
- parameters.put("site", DatasetSite.SLAC.name());
+ parameters.put("site", site);
parameters.put("fileFormat", fileFormat.toString());
parameters.put("name", name);
parameters.put("size", size);
final JSONObject jsonDataset = JSONUtilities.createJSONDataset(parameters, metadata);
final String urlLocation = url + "/datasets.json/" + this.rootDir + "/" + folder;
- LOGGER.info("addDataset: " + urlLocation);
- LOGGER.info("dataset JSON: " + jsonDataset.toString());
+ LOGGER.info("add dataset " + urlLocation);
+ LOGGER.info("dataset JSON " + jsonDataset.toString());
return HttpUtilities.doPost(urlLocation, jsonDataset.toString());
}
Modified: java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/JSONUtilities.java
=============================================================================
--- java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/JSONUtilities.java (original)
+++ java/branches/jeremy-dev/datacat-client/src/main/java/org/hps/datacat/client/JSONUtilities.java Fri Nov 6 16:07:23 2015
@@ -84,16 +84,22 @@
JSONObject metadataObject = new JSONObject();
metadataObject.put("key", entry.getKey());
Object rawValue = entry.getValue();
+ if (rawValue == null) {
+ throw new IllegalArgumentException("The metadata key " + entry.getKey() + " has a null value.");
+ }
if (rawValue instanceof String) {
metadataObject.put("type", "string");
} else if (rawValue instanceof Integer | rawValue instanceof Long) {
metadataObject.put("type", "integer");
} else if (rawValue instanceof Float | rawValue instanceof Double) {
metadataObject.put("type", "decimal");
+ } else if (rawValue instanceof Boolean) {
+ metadataObject.put("type", "integer");
+ rawValue = (Boolean)rawValue ? 1 : 0;
} else {
- throw new IllegalArgumentException("Do not know how to handle type: " + rawValue.getClass().getName());
- }
- metadataObject.put("value", entry.getValue());
+ throw new IllegalArgumentException("Metadata value " + rawValue + " with key " + entry.getKey() + " has unknown type.");
+ }
+ metadataObject.put("value", rawValue);
array.put(metadataObject);
}
return array;
Modified: java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EventTagBitMask.java
=============================================================================
--- java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EventTagBitMask.java (original)
+++ java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EventTagBitMask.java Fri Nov 6 16:07:23 2015
@@ -20,7 +20,7 @@
/** Pair 1 trigger. */
PAIRS1(3),
/** Physics event. */
- PHYSICS(7),
+ PHYSICS(7), // FIXME: Doesn't work!
/** Pulser triggered event. */
PULSER(5),
/** Single 0 trigger. */
Modified: java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioEventUtilities.java
=============================================================================
--- java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioEventUtilities.java (original)
+++ java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioEventUtilities.java Fri Nov 6 16:07:23 2015
@@ -13,9 +13,6 @@
import org.hps.conditions.database.DatabaseConditionsManager;
import org.hps.record.daqconfig.EvioDAQParser;
-import org.hps.record.epics.EpicsData;
-import org.hps.record.epics.EpicsHeader;
-import org.hps.record.scalers.ScalerData;
import org.jlab.coda.jevio.BaseStructure;
import org.jlab.coda.jevio.EvioEvent;
import org.lcsim.conditions.ConditionsManager.ConditionsNotFoundException;
Modified: java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java
=============================================================================
--- java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java (original)
+++ java/branches/jeremy-dev/record-util/src/main/java/org/hps/record/evio/EvioFileUtilities.java Fri Nov 6 16:07:23 2015
@@ -28,19 +28,23 @@
/**
* Get a cached file path, assuming that the input file path is on the JLAB MSS e.g. it starts with "/mss".
+ * If the file is not on the JLAB MSS an error will be thrown.
+ * <p>
+ * If the file is already on the cache disk just return the same file.
*
- * @param file the MSS file path
+ * @param mssFile the MSS file path
* @return the cached file path (prepends "/cache" to the path)
* @throws IllegalArgumentException if the file is not on the MSS (e.g. path does not start with "/mss")
*/
- public static File getCachedFile(final File file) {
- if (!isMssFile(file)) {
- throw new IllegalArgumentException("File " + file.getPath() + " is not on the JLab MSS.");
+ public static File getCachedFile(final File mssFile) {
+ if (!isMssFile(mssFile)) {
+ throw new IllegalArgumentException("File " + mssFile.getPath() + " is not on the JLab MSS.");
}
- if (isCachedFile(file)) {
- throw new IllegalArgumentException("File " + file.getPath() + " is already on the cache disk.");
- }
- return new File("/cache" + file.getPath());
+ File cacheFile = mssFile;
+ if (!isCachedFile(mssFile)) {
+ cacheFile = new File("/cache" + mssFile.getAbsolutePath());
+ }
+ return cacheFile;
}
/**
@@ -98,7 +102,7 @@
* @throws EvioException if there is an error reading the EVIO data
*/
public static EvioReader open(final File file) throws IOException, EvioException {
- return open(file, false);
+ return open(file, true);
}
/**
@@ -111,6 +115,7 @@
* @throws EvioException if there is an error reading the EVIO data
*/
public static EvioReader open(final File file, final boolean sequential) throws IOException, EvioException {
+ LOGGER.info("opening " + file.getPath() + " in " + (sequential ? "sequential" : "mmap" + " mode"));
File openFile = file;
if (isMssFile(file)) {
openFile = getCachedFile(file);
Modified: java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/KinkAnalysisDriver.java
=============================================================================
--- java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/KinkAnalysisDriver.java (original)
+++ java/branches/jeremy-dev/users/src/main/java/org/hps/users/meeg/KinkAnalysisDriver.java Fri Nov 6 16:07:23 2015
@@ -78,7 +78,6 @@
// }
// }
// }
-
if (event.hasCollection(ReconstructedParticle.class, "AprimeBeamspotConstrained")) {
List<ReconstructedParticle> particles = event.get(ReconstructedParticle.class, "AprimeBeamspotConstrained");
int nvertices = 0;
@@ -101,47 +100,14 @@
for (MCParticle particle : MCParticles) {
if (particle.getOrigin().magnitude() > 10.0) {
- hardScatters.add(VecOp.neg(particle.getOrigin()));
- }
- }
-
+ hardScatters.add(particle.getOrigin());
+ }
+ }
List<SimTrackerHit> trackerHits = event.get(SimTrackerHit.class, "TrackerHits");
// Map<MCParticle, List<SimTrackerHit>> hitMap = new HashMap<MCParticle, List<SimTrackerHit>>();
- Map<MCParticle, Map<Integer, SimTrackerHit>> trackMap = new HashMap<MCParticle, Map<Integer, SimTrackerHit>>();
-
- for (SimTrackerHit hit : trackerHits) {
-// List hitList = hitMap.get(hit.getMCParticle());
-// if (hitList == null) {
-// hitList = new ArrayList<SimTrackerHit>();
-// hitMap.put(hit.getMCParticle(), hitList);
-// }
-// hitList.add(hit);
-
- Map<Integer, SimTrackerHit> layerMap = trackMap.get(hit.getMCParticle());
- if (layerMap == null) {
- layerMap = new HashMap<Integer, SimTrackerHit>();
- trackMap.put(hit.getMCParticle(), layerMap);
- }
- int layer = hit.getIdentifierFieldValue("layer");
- if (layerMap.containsKey(layer)) {
- boolean nearHardScatter = false;
- for (Hep3Vector scatter : hardScatters) {
- if (VecOp.add(hit.getPositionVec(), scatter).magnitude() < 5.0) {
- nearHardScatter = true;
- }
- }
- if (!nearHardScatter) {
- hardScatters.add(VecOp.neg(hit.getPositionVec()));
- }
-// System.out.format("Double hit in layer %d, %s\n", layer, nearHardScatter ? "near hard scatter" : "not near hard scatter");
- if (layerMap.get(layer).getPathLength() < hit.getPathLength()) {
- continue;
- }
- }
- layerMap.put(layer, hit);
- }
+ Map<MCParticle, Map<Integer, SimTrackerHit>> trackMap = makeTrackHitMap(trackerHits, hardScatters);
List<MCParticle> particlesWithoutTracks = new ArrayList<MCParticle>();
for (MCParticle particle : trackMap.keySet()) {
@@ -208,13 +174,12 @@
frontDP.fill(p1.magnitude() - p2.magnitude());
frontDT.fill(deflection12);
-
for (int i = 0; i < layers.size() - 1; i++) {
SimTrackerHit hit = layerMap.get(layers.get(i));
boolean nearHardScatter = false;
for (Hep3Vector scatter : hardScatters) {
- if (VecOp.add(hit.getPositionVec(), scatter).magnitude() < 5.0) {
+ if (VecOp.sub(hit.getPositionVec(), scatter).magnitude() < 5.0) {
nearHardScatter = true;
}
}
@@ -268,7 +233,7 @@
}
}
- private double angle(SimTrackerHit hit1, SimTrackerHit hit2) {
+ private static double angle(SimTrackerHit hit1, SimTrackerHit hit2) {
double y1 = hit2.getMCParticle().getOriginY();
// double z1 = hit2.getMCParticle().getOriginZ();
double s1 = hit2.getMCParticle().getProductionTime() * PhysicalConstants.c_light;
@@ -286,7 +251,7 @@
return Math.asin((y2 - y1) / (s2 - s1));
}
- private double angle(List<Integer> layers, Map<Integer, SimTrackerHit> layerMap, int layer1, int layer2) {
+ private static double angle(List<Integer> layers, Map<Integer, SimTrackerHit> layerMap, int layer1, int layer2) {
SimTrackerHit hit1 = null;
if (layer1 > 0) {
hit1 = layerMap.get(layers.get(layer1 - 1));
@@ -296,7 +261,7 @@
return angle(hit1, hit2);
}
- private double deflection(Map<Integer, SimTrackerHit> layerMap, int layer1, int layer2) {
+ static double deflection(Map<Integer, SimTrackerHit> layerMap, int layer1, int layer2) {
List<Integer> layers = new ArrayList<Integer>(layerMap.keySet());
Collections.sort(layers);
@@ -304,6 +269,38 @@
double angle2 = angle(layers, layerMap, layer2, layer2 + 1);
return (angle2 - angle1) * Math.signum(angle1);
+ }
+
+ static Map<MCParticle, Map<Integer, SimTrackerHit>> makeTrackHitMap(List<SimTrackerHit> trackerHits, List<Hep3Vector> hardScatters) {
+ Map<MCParticle, Map<Integer, SimTrackerHit>> trackMap = new HashMap<MCParticle, Map<Integer, SimTrackerHit>>();
+
+ for (SimTrackerHit hit : trackerHits) {
+ Map<Integer, SimTrackerHit> layerMap = trackMap.get(hit.getMCParticle());
+ if (layerMap == null) {
+ layerMap = new HashMap<Integer, SimTrackerHit>();
+ trackMap.put(hit.getMCParticle(), layerMap);
+ }
+ int layer = hit.getIdentifierFieldValue("layer");
+ if (layerMap.containsKey(layer)) {
+ if (hardScatters != null) {
+ boolean nearHardScatter = false;
+ for (Hep3Vector scatter : hardScatters) {
+ if (VecOp.sub(hit.getPositionVec(), scatter).magnitude() < 5.0) {
+ nearHardScatter = true;
+ }
+ }
+ if (!nearHardScatter) {
+ hardScatters.add(hit.getPositionVec());
+ }
+ }
+// System.out.format("Double hit in layer %d, %s\n", layer, nearHardScatter ? "near hard scatter" : "not near hard scatter");
+ if (layerMap.get(layer).getPathLength() < hit.getPathLength()) {
+ continue;
+ }
+ }
+ layerMap.put(layer, hit);
+ }
+ return trackMap;
}
@Override
|
