Author: [log in to unmask]
Date: Fri Jul 10 16:05:22 2015
New Revision: 3256
Log:
Add missing javadoc. Simplify how physics event counting works. Fix a few bug related to finding the run's end date.
Modified:
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/DateFileFilter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventTypeLog.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummary.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/ScalerDataUpdater.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/package-info.java
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java Fri Jul 10 16:05:22 2015
@@ -24,8 +24,8 @@
import org.lcsim.util.log.LogUtil;
/**
- * Crawls EVIO files in a directory tree, groups the files that are found by run, and optionally performs various tasks based on the run summary
- * information that is accumulated, including printing a summary, caching the files from JLAB MSS, and updating a run database.
+ * Search for EVIO files in a directory tree, group the files that are found by run, extract meta data from these
+ * files, and optionally update a run database with the information that was found.
*
* @author Jeremy McCormick, SLAC
*/
@@ -51,12 +51,12 @@
*/
static {
OPTIONS.addOption("a", "runs", true, "list of run numbers to accept (others will be excluded)");
- OPTIONS.addOption("b", "min-date", true, "min date for files (example 2015-03-26 11:28:59)");
- OPTIONS.addOption("c", "cache", false, "automatically cache files from MSS (JLAB only)");
+ OPTIONS.addOption("b", "min-date", true, "min date for a file (example \"2015-03-26 11:28:59\")");
+ OPTIONS.addOption("c", "cache", false, "automatically cache files from MSS to cache disk (JLAB only)");
OPTIONS.addOption("C", "connection-properties", true, "database connection properties file (required)");
OPTIONS.addOption("d", "directory", true, "root directory to start crawling (default is current dir)");
- OPTIONS.addOption("E", "evio-processor", true, "full class name of an EvioEventProcessor to execute (can be used multiple times)");
- OPTIONS.addOption("h", "help", false, "print help and exit");
+ OPTIONS.addOption("E", "evio-processor", true, "class name of an EvioEventProcessor to execute");
+ OPTIONS.addOption("h", "help", false, "print help and exit (overrides all other arguments)");
OPTIONS.addOption("m", "max-files", true, "max number of files to process per run (mostly for debugging)");
OPTIONS.addOption("p", "print", true, "set event printing interval during EVIO processing");
OPTIONS.addOption("r", "insert", false, "insert information into the run database (not done by default)");
@@ -67,7 +67,7 @@
}
/**
- * Support running the crawler from the command line.
+ * Run the crawler from the command line.
*
* @param args the command line arguments
*/
@@ -88,24 +88,24 @@
* The options parser.
*/
private final PosixParser parser = new PosixParser();
-
+
/**
* Configuration options from the command line.
*/
private CrawlerConfig config;
-
- /**
- * Parse command line options into internal configuration object.
+
+ /**
+ * Parse command line options and create a new {@link Crawler} object from the configuration.
*
* @param args the command line arguments
* @return the configured crawler object
*/
private Crawler parse(final String args[]) {
-
+
LOGGER.info("parsing command line options");
-
+
config = new CrawlerConfig();
-
+
try {
final CommandLine cl = this.parser.parse(OPTIONS, args);
@@ -126,12 +126,15 @@
final String dbPropPath = cl.getOptionValue("C");
final File dbPropFile = new File(dbPropPath);
if (!dbPropFile.exists()) {
- throw new IllegalArgumentException("Connection properties file " + dbPropFile.getPath() + " does not exist.");
- }
- config.setConnection(ConnectionParameters.fromProperties(dbPropFile));
+ throw new IllegalArgumentException("Connection properties file " + dbPropFile.getPath()
+ + " does not exist.");
+ }
+ ConnectionParameters connectionParameters = ConnectionParameters.fromProperties(dbPropFile);
+ config.setConnection(connectionParameters);
LOGGER.config("using " + dbPropPath + " for db connection properties");
} else {
- throw new RuntimeException("The -C switch providing the database connection properties file is a required argument.");
+ throw new RuntimeException(
+ "The -C switch providing the database connection properties file is a required argument.");
}
// Root directory for file crawling.
@@ -155,17 +158,19 @@
try {
// Create new time stamp file which will have its date updated at the end of the job.
LOGGER.config("creating new timestamp file " + timestampFile.getPath());
- timestampFile.createNewFile();
+ timestampFile.createNewFile();
} catch (final IOException e) {
throw new IllegalArgumentException("Error creating timestamp file: " + timestampFile.getPath());
}
} else {
try {
// Get the date filter for files from an existing time stamp file provided by the user.
- Date timestamp = new Date(Files.readAttributes(config.timestampFile().toPath(), BasicFileAttributes.class).lastModifiedTime()
- .toMillis());
+ Date timestamp = new Date(Files
+ .readAttributes(config.timestampFile().toPath(), BasicFileAttributes.class)
+ .lastModifiedTime().toMillis());
config.setTimestamp(timestamp);
- LOGGER.config("got timestamp " + timestamp + " from existing file " + config.timestampFile().getPath());
+ LOGGER.config("got timestamp " + timestamp + " from existing file "
+ + config.timestampFile().getPath());
} catch (final IOException e) {
throw new RuntimeException("Error getting attributes of timestamp file.", e);
}
@@ -226,7 +231,8 @@
if (cl.hasOption("b")) {
try {
if (config.timestamp() != null) {
- LOGGER.warning("existing timestamp from file " + config.timestamp() + " will be overridden by date from -b argument");
+ LOGGER.warning("existing timestamp from file " + config.timestamp()
+ + " will be overridden by date from -b argument");
}
config.setTimestamp(cl.getOptionValue("b"));
LOGGER.config("set timestamp to " + config.timestamp() + " from -b argument");
@@ -250,30 +256,32 @@
} catch (final ParseException e) {
throw new RuntimeException("Error parsing options.", e);
}
-
+
LOGGER.info("done parsing command line options");
return this;
}
/**
- * Print the usage statement for this tool to the console and exit.
+ * Print the usage statement for this tool to the console and then exit the program.
*/
private void printUsage() {
final HelpFormatter help = new HelpFormatter();
help.printHelp("EvioFileCrawler", "", OPTIONS, "");
System.exit(0);
}
-
- /**
- * Run the full crawler job, including all configured file-processing steps, which may take a very long time!
+
+ /**
+ * Run the full crawler job.
+ * <p>
+ * This might take quite a long time!
*
* @throws Exception if there is some error during the job
*/
public void run() throws Exception {
LOGGER.info("running Crawler job");
-
+
// Create the file visitor for crawling the root directory with the given date filter.
final EvioFileVisitor visitor = new EvioFileVisitor(config.timestamp());
@@ -284,12 +292,12 @@
final RunLog runs = visitor.getRunLog();
// Print the run numbers that were found.
- printRunNumbers(runs);
+ runs.printRunNumbers();
// Sort the files on their sequence numbers.
- runs.sortAllFiles();
-
- // Process all the files, performing caching from MSS if necessary.
+ runs.sortFiles();
+
+ // Process all the files, performing caching from the MSS if necessary.
RunProcessor.processRuns(this.cacheManager, runs, config);
// Print the summary information after the run processing is done.
@@ -320,15 +328,6 @@
// Close the DB connection.
connection.close();
}
- }
-
- private void printRunNumbers(final RunLog runs) {
- // Print the list of runs that were found.
- final StringBuffer sb = new StringBuffer();
- for (final Integer run : runs.getSortedRunNumbers()) {
- sb.append(run + " ");
- }
- LOGGER.info("found EVIO files from runs: " + sb.toString());
}
private void walk(final EvioFileVisitor visitor) {
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java Fri Jul 10 16:05:22 2015
@@ -12,15 +12,19 @@
import org.hps.record.evio.EvioEventProcessor;
/**
- * Crawls EVIO files in a directory tree, groups the files that are found by run, and optionally performs various tasks based on the run summary
- * information that is accumulated, including printing a summary, caching the files from JLAB MSS, and updating a run database.
+ * Full configuration information for the {@link Crawler class}.
+ * <p>
+ * The setter methods use the builder pattern so method chaining them is possible.
*
* @author Jeremy McCormick, SLAC
*/
final class CrawlerConfig {
+ /**
+ * The format for input timestamps used for file filtering.
+ */
private static final SimpleDateFormat TIMESTAMP_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-
+
/**
* Default event print interval.
*/
@@ -30,14 +34,20 @@
* Interval for printing out event number while running EVIO processors.
*/
private int eventPrintInterval = DEFAULT_EVENT_PRINT_INTERVAL;
-
+
/**
* A list of run numbers to accept in the job.
*/
private Set<Integer> acceptRuns;
+ /**
+ * <code>true</code> if database updates are allowed meaning existing records can be deleted and replaced.
+ */
private boolean allowUpdates = false;
+ /**
+ * The database connection parameters which must be provided by command line argument.
+ */
private ConnectionParameters connectionParameters;
/**
@@ -45,10 +55,13 @@
*/
private int maxFiles = -1;
+ /**
+ * A list of extra {@link org.hps.record.evio.EvioEventProcessor}s to run with the job.
+ */
private final List<EvioEventProcessor> processors = new ArrayList<EvioEventProcessor>();
/**
- * The root directory to crawl which defaults to the current directory.
+ * The root directory to search for files, which defaults to the current directory.
*/
private File rootDir = new File(System.getProperty("user.dir"));
@@ -58,17 +71,17 @@
private Date timestamp = null;
/**
- * A file to use for the timestamp date.
+ * A file to use for getting the timestamp date.
*/
private File timestampFile = null;
/**
- * Flag indicating if the run database should be updated from results of the job.
+ * <code>true</code> if the run database should be updated from results of the job.
*/
private boolean updateRunLog = false;
/**
- * Flag indicating if the file cache should be used (e.g. jcache automatically executed to move files to the cache disk from tape).
+ * <code>true</code> if file caching should be used to move files to the cache disk from tape at JLAB.
*/
private boolean useFileCache = false;
@@ -77,11 +90,23 @@
*/
private Long waitTime;
+ /**
+ * Add an {@link org.hps.record.evio.EvioEventProcessor} to the job.
+ *
+ * @param processor
+ * @return this object
+ */
CrawlerConfig addProcessor(final EvioEventProcessor processor) {
this.processors.add(processor);
return this;
}
-
+
+ /**
+ * Add an {@link org.hps.record.evio.EvioEventProcessor} to the job by its class name.
+ *
+ * @param processor the <code>EvioEventProcessor</code> to instantiate
+ * @return this object
+ */
CrawlerConfig addProcessor(final String className) {
try {
this.processors.add(EvioEventProcessor.class.cast(Class.forName(className).newInstance()));
@@ -91,111 +116,261 @@
return this;
}
+ /**
+ * Set the list of run numbers that should be accepted.
+ *
+ * @param acceptRuns the list of acceptable run numbers
+ * @return this object
+ */
CrawlerConfig setAcceptRuns(final Set<Integer> acceptRuns) {
this.acceptRuns = acceptRuns;
return this;
}
+ /**
+ * Set whether database updates are allowed, i.e. replacement of existing records.
+ *
+ * @param allowUpdates <code>true</code> to allow database record deletion/updates
+ * @return this object
+ */
CrawlerConfig setAllowUpdates(final boolean allowUpdates) {
this.allowUpdates = allowUpdates;
return this;
}
+ /**
+ * Set the database connection parameters.
+ *
+ * @param connectionParameters the database connection parameters
+ * @return this object
+ */
CrawlerConfig setConnection(final ConnectionParameters connectionParameters) {
this.connectionParameters = connectionParameters;
return this;
}
+ /**
+ * Set the maximum number of files that will be processed by the job.
+ * <p>
+ * This should only be used for debugging purposes as it results in incorrect event counts for the run.
+ *
+ * @param maxFiles the maximum number of files to process or -1 for unlimited
+ * @return this object
+ */
CrawlerConfig setMaxFiles(final int maxFiles) {
this.maxFiles = maxFiles;
return this;
}
-
+
+ /**
+ * Set the root directory for the file search.
+ *
+ * @param rootDir the root directory for the file search
+ * @return this object
+ */
CrawlerConfig setRootDir(File rootDir) {
this.rootDir = rootDir;
return this;
}
-
+
+ /**
+ * Set a date for filtering input files.
+ * <p>
+ * Those files created before this date will not be processed.
+ *
+ * @param timestamp the date for filtering files
+ * @return this object
+ */
CrawlerConfig setTimestamp(Date timestamp) {
this.timestamp = timestamp;
return this;
}
-
+
+ /**
+ * Set a date for filtering input files using a string in the default format defined by
+ * <code>TIMESTAMP_DATE_FORMAT</code>.
+ * <p>
+ * Those files created before this date will not be processed.
+ *
+ * @param timestamp the date string for filtering files
+ * @return this object
+ */
CrawlerConfig setTimestamp(String timestampString) throws ParseException {
TIMESTAMP_DATE_FORMAT.parse(timestampString);
return this;
}
-
+
+ /**
+ * Set a date for filtering files based on the modification date of a timestamp file.
+ *
+ * @param timestampFile the timestamp file for date filtering
+ * @return this object
+ */
CrawlerConfig setTimestampFile(File timestampFile) {
this.timestampFile = timestampFile;
return this;
}
-
+
+ /**
+ * Set whether the run database should be updated in the job.
+ * <p>
+ * This will not allow replacement of existing run log records. The {@link #allowUpdates()} flag must be on for this
+ * be allowed.
+ *
+ * @param updateRunLog <code>true</code> if the run database should be updated
+ * @return this object
+ */
CrawlerConfig setUpdateRunLog(boolean updateRunLog) {
this.updateRunLog = updateRunLog;
return this;
}
-
+
+ /**
+ * Set whether file caching using the 'jcache' program should be enabled.
+ * <p>
+ * This is only relevant for jobs run at JLAB.
+ *
+ * @param useFileCache <code>true</code> to allow file caching
+ * @return this object
+ */
CrawlerConfig setUseFileCache(boolean useFileCache) {
this.useFileCache = useFileCache;
return this;
}
-
+
+ /**
+ * Set the max wait time in seconds for all file caching operations to complete.
+ * <p>
+ * If this time is exceeded then the job will fail with an error.
+ *
+ * @param waitTime the max wait time in seconds allowed for file caching to complete
+ * @return this object
+ */
CrawlerConfig setWaitTime(long waitTime) {
this.waitTime = waitTime;
return this;
}
-
+
+ /**
+ * Set the interval for printing the EVIO event numbers during processing.
+ *
+ * @param eventPrintInterval the event print interval
+ * @return this object
+ */
CrawlerConfig setEventPrintInterval(int eventPrintInterval) {
this.eventPrintInterval = eventPrintInterval;
return this;
}
-
+
+ /**
+ * Get the set of runs that will be accepted for the job.
+ *
+ * @return the list of runs that will be accepted
+ */
Set<Integer> acceptRuns() {
return acceptRuns;
}
-
+
+ /**
+ * Return <code>true</code> if updates/deletions of existing records in the database is allowed.
+ *
+ * @return <code>true</code> if updating/deleting records in the database is allowed
+ */
boolean allowUpdates() {
return allowUpdates;
}
+ /**
+ * Get the database connection parameters.
+ *
+ * @return the database connection parameters
+ */
ConnectionParameters connectionParameters() {
return connectionParameters;
}
+ /**
+ * Get the maximum number of files that the job can process.
+ *
+ * @return the maximum number of files
+ */
int maxFiles() {
return maxFiles;
}
+ /**
+ * Get the list of extra event processors that will run with the job.
+ * <p>
+ * Required (default) processors for the job are not included here.
+ *
+ * @return the list of extra event processors
+ */
List<EvioEventProcessor> processors() {
return processors;
}
+ /**
+ * Get the root directory for the file search.
+ *
+ * @return the root directory for the file search
+ */
File rootDir() {
return rootDir;
}
+ /**
+ * Get the timestamp for file filtering.
+ * <p>
+ * Files older than this will not be included in the job.
+ *
+ * @return the timestamp for file filtering
+ */
Date timestamp() {
return timestamp;
}
-
+
+ /**
+ * Get the timestamp file using for filtering EVIO files.
+ *
+ * @return the timestamp file used for filtering EVIO files (can be null)
+ */
File timestampFile() {
return timestampFile;
}
+ /**
+ * Return <code>true</code> if the run database should be updated.
+ *
+ * @return <code>true</code> if the run database should be updated
+ */
boolean updateRunLog() {
return updateRunLog;
}
+ /**
+ * Return <code>true</code> if file caching should be enabled.
+ *
+ * @return <code>true</code> if file caching should be enabled
+ */
boolean useFileCache() {
return useFileCache;
}
+ /**
+ * Get the max wait time in seconds to allow for file caching operations to complete.
+ *
+ * @return the max wait time in seconds to allow for file caching operations to complete
+ */
Long waitTime() {
return waitTime;
- }
-
+ }
+
+ /**
+ * Get the event print interval.
+ *
+ * @return the event print interval
+ */
int eventPrintInterval() {
return this.eventPrintInterval;
- }
+ }
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/DateFileFilter.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/DateFileFilter.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/DateFileFilter.java Fri Jul 10 16:05:22 2015
@@ -17,7 +17,7 @@
final class DateFileFilter implements FileFilter {
/**
- * The cut off time stamp.
+ * The cut off timestamp.
*/
private final Date date;
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java Fri Jul 10 16:05:22 2015
@@ -16,7 +16,8 @@
final class EpicsLog extends EvioEventProcessor {
/**
- * A count of how many times a given EPICS variable is found in the input, e.g. for computing the mean value across the run.
+ * A count of how many times a given EPICS variable is found in the input, e.g. for computing the mean value across
+ * the entire run.
*/
private final Map<String, Integer> counts = new HashMap<String, Integer>();
@@ -80,7 +81,7 @@
/**
* Update state from the current EPICS data.
* <p>
- * If the current data is null, this method does nothing.
+ * If the current data is <code>null</code>, this method does nothing.
*/
private void update() {
if (this.currentData != null) {
@@ -96,8 +97,9 @@
this.counts.put(name, count);
final double value = this.logData.getValue(name) + this.currentData.getValue(name);
this.logData.setValue(name, value);
- //System.out.println(name + " => added " + this.currentData.getValue(name) + "; total = " + value + "; mean = " + value / count);
+ // System.out.println(name + " => added " + this.currentData.getValue(name) + "; total = " + value +
+ // "; mean = " + value / count);
}
}
- }
+ }
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventTypeLog.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventTypeLog.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventTypeLog.java Fri Jul 10 16:05:22 2015
@@ -6,6 +6,7 @@
import org.hps.record.evio.EventTagBitMask;
import org.hps.record.evio.EventTagConstant;
import org.hps.record.evio.EvioEventProcessor;
+import org.hps.record.evio.EvioEventUtilities;
import org.jlab.coda.jevio.EvioEvent;
/**
@@ -24,6 +25,11 @@
* The run summary to update.
*/
private final RunSummary runSummary;
+
+ /**
+ * The total number of physics events processed.
+ */
+ private int physicsEventCount = 0;
/**
* Create the log pointing to a run summary.
@@ -58,23 +64,41 @@
}
/**
+ * Get the number of physics events counted.
+ *
+ * @return the number of physics events counted
+ */
+ int getPhysicsEventCount() {
+ return physicsEventCount;
+ }
+
+ /**
* Process an EVIO event and add its type to the map.
*
* @param event the EVIO event
*/
@Override
public void process(final EvioEvent event) {
+
+ // Increment counts for exact event tag values.
for (final EventTagConstant constant : EventTagConstant.values()) {
if (constant.isEventTag(event)) {
final int count = this.eventTypeCounts.get(constant) + 1;
this.eventTypeCounts.put(constant, count);
}
}
+
+ // Increment counts for bit masking of tags.
for (final EventTagBitMask mask : EventTagBitMask.values()) {
if (mask.isEventTag(event)) {
final int count = this.eventTypeCounts.get(mask) + 1;
this.eventTypeCounts.put(mask, count);
}
}
+
+ // Increment physics event count.
+ if (EvioEventUtilities.isPhysicsEvent(event)) {
+ ++this.physicsEventCount;
+ }
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java Fri Jul 10 16:05:22 2015
@@ -4,8 +4,8 @@
import java.io.FileFilter;
/**
- * This is a simple file filter that will accept EVIO files with a certain convention to their naming which looks like <i>FILENAME.evio.SEQUENCE</i>.
- * This matches the convention used by the CODA DAQ software.
+ * This is a simple file filter that will accept EVIO files with a certain convention to their naming which looks like
+ * <i>FILENAME.evio.SEQUENCE</i>. This matches the convention used by the CODA DAQ software.
*
* @author Jeremy McCormick, SLAC
*/
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java Fri Jul 10 16:05:22 2015
@@ -3,30 +3,17 @@
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.logging.Logger;
import org.lcsim.util.log.LogUtil;
/**
- * This is a list of <code>File</code> objects that are assumed to be EVIO files. There are some added utilities for getting the total number of
- * events in all the files.
+ * This is a list of <code>File</code> objects that are assumed to be EVIO files.
*
* @author Jeremy McCormick, SLAC
*/
final class EvioFileList extends ArrayList<File> {
-
- /**
- * Setup logger.
- */
- private static final Logger LOGGER = LogUtil.create(EvioFileList.class);
-
- /**
- * Event count by file.
- */
- private final Map<File, Integer> eventCounts = new HashMap<File, Integer>();
/**
* Get the first file.
@@ -38,57 +25,12 @@
}
/**
- * Get the event count for an EVIO file.
- *
- * @param file the EVIO file
- * @return the event count for the file
- * @throws RuntimeException if the count was never computed (file is not in map)
- */
- int getEventCount(final File file) {
- if (this.eventCounts.get(file) == null) {
- throw new RuntimeException("The event count for " + file.getPath() + " was never computed.");
- }
- return this.eventCounts.get(file);
- }
-
- /**
- * Get the total number of events.
- * <p>
- * Files which do not have their event counts computed will be ignored.
- *
- * @return the total number of events
- */
- int getTotalEvents() {
- int totalEvents = 0;
- for (final File file : this) {
- if (this.eventCounts.containsKey(file)) {
- totalEvents += this.eventCounts.get(file);
- } else {
- // Warn about non-computed count.
- // FIXME: Perhaps this should actually be a fatal error.
- LOGGER.warning("event count for " + file.getPath() + " was not computed and will not be reflected in total");
- }
- }
- return totalEvents;
- }
-
- /**
* Get the last file.
*
* @return the last file
*/
File last() {
return this.get(this.size() - 1);
- }
-
- /**
- * Set the event count for a file.
- *
- * @param file the EVIO file
- * @param eventCount the event count
- */
- void setEventCount(final File file, final Integer eventCount) {
- this.eventCounts.put(file, eventCount);
}
/**
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java Fri Jul 10 16:05:22 2015
@@ -14,7 +14,7 @@
import org.lcsim.util.log.LogUtil;
/**
- * A miscellaneous collection of EVIO file utility methods used by classes in the crawler package.
+ * A miscellaneous collection of EVIO file utility methods used by the crawler package.
*
* @author Jeremy McCormick, SLAC
*/
@@ -108,23 +108,33 @@
}
/**
- * Get the run end date which is taken either from the END event or the last physics event is the END event is not found.
+ * Get the run end date which is taken either from the END event or the last physics event if the END event is not
+ * found in the file.
*
* @param file the EVIO file
* @return the run end date
*/
static Date getRunEnd(final File file) {
- Date date = getControlDate(file, EvioEventConstants.END_EVENT_TAG, -10);
- if (date == null) {
+
+ // Search for the END event in the last 10 events of the file.
+ Date endDate = getControlDate(file, EvioEventConstants.END_EVENT_TAG, -10);
+
+ // Was the end date found from the END event?
+ if (endDate == null) {
+
EvioReader reader = null;
try {
reader = open(file, true);
- reader.gotoEventNumber(reader.getEventCount() - 11);
+
+ // Search for the last physics event in the last 10 events of the file.
+ reader.gotoEventNumber(reader.getEventCount() - 10);
EvioEvent event = null;
while ((event = reader.parseNextEvent()) != null) {
if (EvioEventUtilities.isPhysicsEvent(event)) {
- if ((date = getHeadBankDate(event)) != null) {
- break;
+ Date eventDate = getHeadBankDate(event);
+ if (eventDate != null) {
+ // This might be set multiple times but should result in the time of the last physics event.
+ endDate = eventDate;
}
}
}
@@ -140,7 +150,7 @@
}
}
}
- return date;
+ return endDate;
}
/**
@@ -158,16 +168,22 @@
}
/**
- * Get the run start date from an EVIO file (should be the first in the run).
+ * Get the run start date from an EVIO file (should be the first file in the run).
* <p>
- * This is taken from the PRESTART event.
+ * This is taken from the PRESTART event if available or the first physics event.
*
* @param file the EVIO file
* @return the run start date
*/
static Date getRunStart(final File file) {
+
+ // First try to find the start date in the special PRESTART event.
Date date = getControlDate(file, EvioEventConstants.PRESTART_EVENT_TAG, 0);
+
+ // Was start date not found from PRESTART?
if (date == null) {
+
+ // Read events until there is a physics event and use its time for the start date.
EvioReader reader = null;
try {
reader = open(file, true);
@@ -195,7 +211,7 @@
}
/**
- * Get the EVIO file sequence number (the number at the end of the file name).
+ * Get the EVIO file sequence number, which is the number at the end of the file name.
*
* @param file the EVIO file
* @return the file's sequence number
@@ -207,7 +223,7 @@
}
/**
- * Return <code>true</code> if this is a cached file e.g. the path starts with "/cache".
+ * Return <code>true</code> if this is a file on the cache disk e.g. the path starts with "/cache".
*
* @param file the file
* @return <code>true</code> if the file is a cached file
@@ -227,7 +243,7 @@
}
/**
- * Open an EVIO file using a <code>EvioReader</code>.
+ * Open an EVIO file using an <code>EvioReader</code> in memory mapping mode.
*
* @param file the EVIO file
* @return the new <code>EvioReader</code> for the file
@@ -260,14 +276,14 @@
}
/**
- * Open an EVIO from a path.
- *
- * @param path the file path
+ * Open an EVIO file from a path string.
+ *
+ * @param path the path string
* @return the new <code>EvioReader</code> for the file
* @throws IOException if there is an IO problem
* @throws EvioException if there is an error reading the EVIO data
*/
static EvioReader open(final String path) throws IOException, EvioException {
- return open(new File(path), true);
+ return open(new File(path), false);
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java Fri Jul 10 16:05:22 2015
@@ -16,9 +16,7 @@
import org.lcsim.util.log.LogUtil;
/**
- * A file visitor that crawls directories for EVIO files.
- * <p>
- * It updates a run log to keep track of which files are associated with which run numbers.
+ * A file visitor that crawls directories for EVIO files and returns the information as a {@link RunLog}.
*
* @author Jeremy McCormick, SLAC
*/
@@ -62,7 +60,7 @@
boolean accept = true;
for (final FileFilter filter : this.filters) {
accept = filter.accept(file);
- if (accept == false) {
+ if (!accept) {
LOGGER.finer(filter.getClass().getSimpleName() + " rejected " + file.getPath());
break;
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java Fri Jul 10 16:05:22 2015
@@ -5,6 +5,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
+import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -20,7 +21,7 @@
import org.xml.sax.InputSource;
/**
- * Utility class for using the <i>jcache</i> command at JLAB.
+ * Utility class for caching files from the MSS to cache disk at JLAB.
*
* @author Jeremy McCormick, SLAC
*/
@@ -50,7 +51,7 @@
* The current status from executing the 'jcache request' command.
*/
private String status;
-
+
/**
* The xml node with request data.
*/
@@ -98,9 +99,7 @@
} catch (final IOException e) {
throw new RuntimeException(e);
}
- // LOGGER.finer("raw XML: " + xmlString);
xmlString = xmlString.substring(xmlString.trim().indexOf("<jcache>"));
- // LOGGER.finer("cleaned XML: " + xmlString);
return buildDocument(xmlString).getRootElement();
}
@@ -152,14 +151,14 @@
boolean isPending() {
return "pending".equals(this.status);
}
-
+
/**
* Return <code>true</code> if status is "failed".
*/
boolean isFailed() {
return "failed".equals(this.status);
}
-
+
/**
* Get the error message from the XML request.
*
@@ -172,7 +171,7 @@
return "";
}
}
-
+
/**
* Run the <i>jcache request</i> command for this request ID and return the XML output.
*
@@ -194,7 +193,7 @@
if (status != 0) {
throw new RuntimeException("The jcache request returned an error status: " + status);
}
- return this.getRequestXml(process.getInputStream());
+ return this.getRequestXml(process.getInputStream());
}
/**
@@ -203,10 +202,10 @@
void update() {
// Request status update and get the XML from that process.
this.xml = request();
-
+
// Update the status from the XML.
this.status = this.xml.getChild("request").getChild("file").getChildText("status");
-
+
// Is request done or file already in cache?
if (this.isDone() || this.isHit()) {
// Flag file as cached.
@@ -231,9 +230,9 @@
private static Logger LOGGER = LogUtil.create(JCacheManager.class, new DefaultLogFormatter(), Level.FINE);
/**
- * Time to wait between polling of all files (~10 seconds).
- */
- private static final long POLL_WAIT_TIME = 10000;
+ * Time to wait between re-polling all files (30 seconds).
+ */
+ private static final long POLL_WAIT_TIME = 30000;
/**
* Build an XML document from a string.
@@ -242,7 +241,7 @@
* @return the XML document
*/
private static Document buildDocument(final String xmlString) {
- LOGGER.fine("building doc from string: " + xmlString);
+ LOGGER.fine(xmlString);
final SAXBuilder builder = new SAXBuilder();
Document document = null;
try {
@@ -293,14 +292,14 @@
private long maxWaitTime = DEFAULT_MAX_WAIT_TIME;
/**
- * The time when the caching operation starts.
+ * The time in milliseconds when the caching operation started.
*/
long start = 0;
/**
* Cache a file by submitting a 'jcache submit' process.
* <p>
- * The resulting cache request will be registered with this manager until the {@link #clear()} method is called.
+ * The resulting cache request will be registered with this manager.
*
* @param file the file to cache which should be a path on the JLAB MSS (e.g. starts with '/mss')
*/
@@ -311,7 +310,7 @@
}
if (EvioFileUtilities.getCachedFile(file).exists()) {
- // Assume (maybe unreasonably?!) that since the file already exists it will stay in the cache for the duration of the job.
+ // Assume that since the file already exists it will stay in the cache for the duration of the job.
LOGGER.fine(file.getPath() + " is already on the cache disk so cache request is ignored");
} else {
@@ -330,7 +329,7 @@
}
/**
- * Submit cache request for every file in a list.
+ * Submit a cache request for every file in the list.
*
* @param files the list of files
*/
@@ -356,7 +355,8 @@
// Get the cache status for a single file.
final CacheStatus cacheStatus = entry.getValue();
- LOGGER.info("checking status of " + cacheStatus.getFile().getPath() + " with req ID '" + cacheStatus.getRequestId() + "' ...");
+ LOGGER.info("checking status of " + cacheStatus.getFile().getPath() + " with req ID '"
+ + cacheStatus.getRequestId() + "' ...");
// Is this file flagged as not non-cached?
if (!cacheStatus.isCached()) {
@@ -375,9 +375,10 @@
LOGGER.info(entry.getKey() + " is NOT cached with status " + cacheStatus.getStatus(false));
} else {
// Log that this file is now cached. It will not be checked next time.
- LOGGER.info(cacheStatus.getFile().getPath() + " is cached with status " + cacheStatus.getStatus(false));
+ LOGGER.info(cacheStatus.getFile().getPath() + " is cached with status "
+ + cacheStatus.getStatus(false));
}
-
+
// Did the request fail?
if (cacheStatus.isFailed()) {
// Cache failure is a fatal error.
@@ -535,6 +536,11 @@
break;
}
}
+
+ double end = (double) (System.currentTimeMillis() - this.start);
+
+ LOGGER.info("caching took " + new DecimalFormat("#.##").format(end / 1000. / 60.) + " minutes");
+
if (cached) {
LOGGER.info("all files cached successfully!");
} else {
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java Fri Jul 10 16:05:22 2015
@@ -5,7 +5,7 @@
import java.util.Set;
/**
- * A filter which rejects files that have a run number not in the accept list.
+ * A filter which rejects files with run numbers not in a specified set.
*
* @author Jeremy McCormick, SLAC
*/
@@ -17,7 +17,7 @@
private final Set<Integer> acceptRuns;
/**
- * Create a new <code>RunFilter</code> with a set of runs to accept.
+ * Create a new <code>RunFilter</code> with a set of run numbers to accept.
*
* @param acceptRuns the set of runs to accept
*/
@@ -29,7 +29,7 @@
}
/**
- * Returns <code>true</code> if file is accepted (its run number is in the set).
+ * Returns <code>true</code> if file is accepted.
*
* @param file the EVIO file
* @return <code>true</code> if file is accepted
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java Fri Jul 10 16:05:22 2015
@@ -11,9 +11,7 @@
import org.lcsim.util.log.LogUtil;
/**
- * This class contains summary information about a series of runs which are modeled with the {@link RunSummary} class.
- * <p>
- * These can be looked up by their run number {@link #getRunSummary(int)}.
+ * This class contains information about a series of runs which each have a {@link RunSummary} object.
*
* @author Jeremy McCormick, SLAC
*/
@@ -30,9 +28,9 @@
private final Map<Integer, RunSummary> runs = new LinkedHashMap<Integer, RunSummary>();
/**
- * Get a run summary by run number.
+ * Get a {@link RunSummary} by its run number.
* <p>
- * It will be created if it does not exist.
+ * It will be created if it does not exist already.
*
* @param run the run number
* @return the <code>RunSummary</code> for the run number
@@ -44,15 +42,20 @@
}
return this.runs.get(run);
}
-
+
+ /**
+ * Get the collection of {@link RunSummary} objects.
+ *
+ * @return the collection of {@link RunSummary} objects
+ */
public Collection<RunSummary> getRunSummaries() {
return this.runs.values();
}
/**
- * Get a list of sorted run numbers in this run log.
+ * Get a list of sorted run numbers from this run log.
* <p>
- * This is a copy of the keys from the map so modifying it will have no effect on this class.
+ * This is a copy of the keys from the map, so modifying it will have no effect on the original.
*
* @return the list of sorted run numbers
*/
@@ -63,7 +66,7 @@
}
/**
- * Print out the run summaries to <code>System.out</code>.
+ * Print out each {@link RunSummary} to <code>System.out</code>.
*/
void printRunSummaries() {
for (final int run : this.runs.keySet()) {
@@ -72,11 +75,24 @@
}
/**
- * Sort all the file lists in place (by sequence number).
+ * Sort the file list for each run in place by EVIO sequence numbers.
*/
- void sortAllFiles() {
+ void sortFiles() {
for (final Integer run : this.runs.keySet()) {
this.runs.get(run).sortFiles();
}
}
+
+ /**
+ * Print the run numbers to the log.
+ */
+ void printRunNumbers() {
+ // Print the list of runs that were found.
+ final StringBuffer sb = new StringBuffer();
+ for (final Integer run : getSortedRunNumbers()) {
+ sb.append(run + " ");
+ }
+ LOGGER.info("found EVIO files from runs: " + sb.toString());
+ }
+
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java Fri Jul 10 16:05:22 2015
@@ -15,27 +15,27 @@
* @author Jeremy McCormick, SLAC
*/
public class RunLogUpdater {
-
+
/**
* Setup logging.
*/
private static final Logger LOGGER = LogUtil.create(RunLogUpdater.class);
-
+
/**
* The run log with information for all runs.
*/
private RunLog runLog;
-
+
/**
* The database connection.
*/
private final Connection connection;
-
+
/**
* <code>true</code> if updates should be allowed in database or only inserts.
*/
private boolean allowUpdates = false;
-
+
/**
* Create a new updater.
*
@@ -43,54 +43,54 @@
* @param allowUpdates <code>true</code> if updates should be allowed
*/
RunLogUpdater(Connection connection, RunLog runLog, boolean allowUpdates) {
-
+
// Set the DB connection.
this.connection = connection;
-
+
// Set the run log with the run info to update.
this.runLog = runLog;
-
+
// Set whether db updates are allowed (replacement of existing records).
this.allowUpdates = allowUpdates;
}
-
+
/**
- * Insert the run summary information into the database, including updating the <i>run_log_files</i>
- * and <i>run_log_epics</i> tables.
+ * Insert the run summary information into the database, including updating the <i>run_log_files</i> and
+ * <i>run_log_epics</i> tables.
*
* @param connection the database connection
* @throws SQLException if there is an error querying the database
*/
void insert() throws SQLException {
-
+
LOGGER.info("inserting run data into database ...");
try {
connection.setAutoCommit(false);
-
+
// Loop over all runs found while crawling.
for (final Integer run : runLog.getSortedRunNumbers()) {
-
+
LOGGER.info("beginning transaction for run " + run);
-
+
// Get the RunSummary data for the run.
RunSummary runSummary = runLog.getRunSummary(run);
-
+
LOGGER.info("updating " + runSummary);
-
- // Create the db updater for the RunSummary.
+
+ // Create the database updater for the RunSummary.
RunSummaryUpdater runUpdater = new RunSummaryUpdater(connection, runSummary);
-
+
// Set whether existing records can be replaced.
runUpdater.setAllowDeleteExisting(allowUpdates);
-
+
LOGGER.info("allow updates: " + allowUpdates);
-
+
// Insert the run records.
runUpdater.insert();
-
+
LOGGER.info("run " + runSummary.getRun() + " summary inserted successfully");
}
-
+
} catch (final SQLException e) {
LOGGER.log(Level.SEVERE, "rolling back transaction", e);
try {
@@ -99,8 +99,8 @@
LOGGER.log(Level.SEVERE, "error rolling back transaction", e2);
throw new RuntimeException(e);
}
- }
-
+ }
+
LOGGER.info("done inserting run data");
- }
+ }
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java Fri Jul 10 16:05:22 2015
@@ -20,19 +20,34 @@
/**
* Processes EVIO files from a run in order to extract various meta data information including start and end dates.
* <p>
- * This class is a wrapper for activating different sub-tasks, including optionally caching all files from the JLAB MSS to the cache disk.
+ * This class is a wrapper for activating different sub-tasks, including optionally caching all files from the JLAB MSS
+ * to the cache disk.
* <p>
* There is also a list of processors which is run on all events from the run.
*
* @author Jeremy McCormick, SLAC
*/
-// FIXME: Replace this class with Freehep loop implementation, EvioFileSource, etc.
final class RunProcessor {
/**
* Setup logger.
*/
private static final Logger LOGGER = LogUtil.create(RunProcessor.class, new DefaultLogFormatter(), Level.FINE);
+
+ /**
+ * Processor for extracting EPICS information.
+ */
+ private EpicsLog epicsLog;
+
+ /**
+ * Processor for extracting scaler data.
+ */
+ private ScalersEvioProcessor scalersProcessor;
+
+ /**
+ * Processor for extracting event type counts (sync, physics, trigger types, etc.).
+ */
+ private EventTypeLog eventTypeLog;
/**
* Create the processor for a single run.
@@ -40,34 +55,34 @@
* @param runSummary the run summary for the run
* @return the run processor
*/
- static RunProcessor createRunProcessor(final JCacheManager cacheManager, final RunSummary runSummary, CrawlerConfig config) {
-
- // Create new run processor.
- final RunProcessor processor = new RunProcessor(runSummary, cacheManager);
+ RunProcessor(final JCacheManager cacheManager, final RunSummary runSummary, CrawlerConfig config) {
+
+ this.runSummary = runSummary;
+ this.cacheManager = cacheManager;
// EPICS processor.
- processor.addProcessor(new EpicsLog(runSummary));
+ epicsLog = new EpicsLog(runSummary);
+ addProcessor(epicsLog);
// Scaler data processor.
- final ScalersEvioProcessor scalersProcessor = new ScalersEvioProcessor();
+ scalersProcessor = new ScalersEvioProcessor();
scalersProcessor.setResetEveryEvent(false);
- processor.addProcessor(scalersProcessor);
+ addProcessor(scalersProcessor);
// Event log processor.
- processor.addProcessor(new EventTypeLog(runSummary));
+ eventTypeLog = new EventTypeLog(runSummary);
+ addProcessor(eventTypeLog);
// Max files.
if (config.maxFiles() != -1) {
- processor.setMaxFiles(config.maxFiles());
+ setMaxFiles(config.maxFiles());
}
// Enable file caching.
- processor.useFileCache(config.useFileCache());
+ useFileCache(config.useFileCache());
// Set event printing interval.
- processor.setEventPrintInterval(config.eventPrintInterval());
-
- return processor;
+ setEventPrintInterval(config.eventPrintInterval());
}
/**
@@ -99,17 +114,6 @@
* Set to <code>true</code> to use file caching.
*/
private boolean useFileCache;
-
- /**
- * Create a new run processor.
- *
- * @param runSummary the run summary to update
- * @param cacheManager the cache manager for executing 'jcache' commands
- */
- RunProcessor(final RunSummary runSummary, final JCacheManager cacheManager) {
- this.runSummary = runSummary;
- this.cacheManager = cacheManager;
- }
/**
* Add a processor of EVIO events.
@@ -124,8 +128,8 @@
/**
* Cache all files and wait for the operation to complete.
* <p>
- * Potentially, this operation can take a very long time. This can be managed using the {@link JCacheManager#setWaitTime(long)} method to set a
- * timeout.
+ * Potentially, this operation can take a very long time. This can be managed using the
+ * {@link JCacheManager#setWaitTime(long)} method to set a timeout.
*/
private void cacheFiles() {
@@ -166,8 +170,8 @@
if (this.maxFiles != -1) {
int toIndex = this.maxFiles;
if (toIndex > files.size()) {
- toIndex = files.size();
- }
+ toIndex = files.size();
+ }
files = files.subList(0, toIndex);
}
return files;
@@ -182,17 +186,9 @@
return this.processors;
}
- ScalersEvioProcessor getScalersProcessor() {
- for (final EvioEventProcessor processor : this.processors) {
- if (processor instanceof ScalersEvioProcessor) {
- return ScalersEvioProcessor.class.cast(processor);
- }
- }
- return null;
- }
-
- /**
- * Return <code>true</code> if a valid CODA <i>END</i> event can be located in the <code>EvioReader</code>'s current file.
+ /**
+ * Return <code>true</code> if a valid CODA <i>END</i> event can be located in the <code>EvioReader</code>'s current
+ * file.
*
* @param reader the EVIO reader
* @return <code>true</code> if valid END event is located
@@ -218,8 +214,8 @@
}
/**
- * Process the run by executing the registered {@link org.hps.record.evio.EvioEventProcessor}s and performing special tasks such as the extraction
- * of start and end dates.
+ * Process the run by executing the registered {@link org.hps.record.evio.EvioEventProcessor}s and performing
+ * special tasks such as the extraction of start and end dates.
* <p>
* This method will also activate file caching, if enabled by the {@link #useFileCache} option.
*
@@ -240,9 +236,9 @@
}
List<File> files = this.getFiles();
-
+
LOGGER.info("processing " + files.size() + " from run " + this.runSummary.getRun());
-
+
// Process all the files.
for (final File file : files) {
this.process(file);
@@ -281,12 +277,6 @@
LOGGER.fine("got run start " + runStart);
this.runSummary.setStartDate(runStart);
}
-
- // Compute the event count for the file and store the value in the run summary's file list.
- LOGGER.info("getting event count for " + file.getPath() + "...");
- final int eventCount = this.computeEventCount(reader);
- this.runSummary.getEvioFileList().setEventCount(file, eventCount);
- LOGGER.info("set event count " + eventCount + " for " + file.getPath());
// Process the events using the EVIO processors.
LOGGER.info("running EVIO processors ...");
@@ -319,10 +309,10 @@
}
// Pull scaler data from EVIO processor into run summary.
- final ScalersEvioProcessor scalersProcessor = this.getScalersProcessor();
- if (scalersProcessor != null) {
- runSummary.setScalerData(scalersProcessor.getScalerData());
- }
+ runSummary.setScalerData(this.scalersProcessor.getScalerData());
+
+ // Set total number of physics events.
+ runSummary.setTotalEvents(this.eventTypeLog.getPhysicsEventCount());
} finally {
if (reader != null) {
@@ -356,7 +346,8 @@
/**
* Set whether or not to use the file caching, which copies files from the JLAB MSS to the cache disk.
* <p>
- * Since EVIO data files at JLAB are primarily kept on the MSS, running without this option enabled there will likely cause the job to fail.
+ * Since EVIO data files at JLAB are primarily kept on the MSS, running without this option enabled there will
+ * likely cause the job to fail.
*
* @param cacheFiles <code>true</code> to enabled file caching
*/
@@ -364,7 +355,7 @@
this.useFileCache = cacheFiles;
LOGGER.config("file caching enabled");
}
-
+
/**
* Process all the runs that were found.
*
@@ -372,27 +363,28 @@
* @throws Exception if there is an error processing one of the runs
*/
static void processRuns(JCacheManager cacheManager, final RunLog runs, CrawlerConfig config) throws Exception {
-
+
// Configure max wait time of jcache manager.
if (config.waitTime() != null && config.waitTime() > 0L) {
cacheManager.setWaitTime(config.waitTime());
LOGGER.config("JCacheManager max wait time set to " + config.waitTime());
- }
-
+ }
+
// Process all of the runs that were found.
for (final int run : runs.getSortedRunNumbers()) {
-
+
// Get the run summary.
RunSummary runSummary = runs.getRunSummary(run);
-
+
// Clear the cache manager.
if (config.useFileCache()) {
cacheManager.clear();
}
// Create a processor to process all the EVIO events in the run.
- final RunProcessor runProcessor = RunProcessor.createRunProcessor(cacheManager, runSummary, config);
-
+ final RunProcessor runProcessor = new RunProcessor(cacheManager, runSummary, config);
+
+ // Add extra processors.
for (final EvioEventProcessor processor : config.processors()) {
runProcessor.addProcessor(processor);
LOGGER.config("added extra EVIO processor " + processor.getClass().getName());
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummary.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummary.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummary.java Fri Jul 10 16:05:22 2015
@@ -15,8 +15,8 @@
import org.lcsim.util.log.LogUtil;
/**
- * This class models the run summary information which is persisted as a row in the <i>run_log</i>
- * table of the run database.
+ * This class models the run summary information which is persisted as a row in the <i>run_log</i> table of the run
+ * database.
* <p>
* This information includes:
* <ul>
@@ -40,7 +40,7 @@
static {
DATE_DISPLAY.setCalendar(new GregorianCalendar(TimeZone.getTimeZone("America/New_York")));
}
-
+
/**
* Setup logger.
*/
@@ -151,7 +151,7 @@
public EvioFileList getEvioFileList() {
return this.files;
}
-
+
/**
* Get the run number.
*
@@ -185,12 +185,13 @@
* @return the total events in the run
*/
public int getTotalEvents() {
- if (this.totalEvents == -1) {
- this.totalEvents = this.files.getTotalEvents();
- }
return this.totalEvents;
}
-
+
+ void setTotalEvents(int totalEvents) {
+ this.totalEvents = totalEvents;
+ }
+
/**
* Get the number of seconds in the run which is the difference between the start and end times.
*
@@ -205,10 +206,10 @@
}
return (getEndDate().getTime() - getStartDate().getTime()) / 1000;
}
-
- /**
- * Get the event rate (effectively the trigger rate) which is the total events divided by the number
- * of seconds in the run.
+
+ /**
+ * Get the event rate (effectively the trigger rate) which is the total events divided by the number of seconds in
+ * the run.
*
* @return the event rate
*/
@@ -289,6 +290,11 @@
this.eventTypeCounts = eventTypeCounts;
}
+ /**
+ * Set the scaler data of the run.
+ *
+ * @param scalerData the scaler data
+ */
void setScalerData(final ScalerData scalerData) {
this.scalerData = scalerData;
}
@@ -316,7 +322,7 @@
*/
@Override
public String toString() {
- return "RunSummary { run: " + this.run + ", started: " + this.getStartDate() + ", ended: " + this.getEndDate() + ", events: "
- + this.getTotalEvents() + ", endOkay: " + endOkay + " }";
+ return "RunSummary { run: " + this.run + ", started: " + this.getStartDate() + ", ended: " + this.getEndDate()
+ + ", events: " + this.getTotalEvents() + ", endOkay: " + endOkay + " }";
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java Fri Jul 10 16:05:22 2015
@@ -74,13 +74,13 @@
// Delete scaler data.
this.deleteScalerData();
-
+
// Delete file list.
this.deleteFiles();
// Delete run log.
this.deleteRun();
-
+
LOGGER.info("deleted run " + runSummary.getRun() + " info successfully");
}
@@ -94,7 +94,7 @@
statement.setInt(1, this.run);
statement.executeUpdate();
}
-
+
/**
* Delete existing EPICS data from the run_log_epics table.
*
@@ -136,7 +136,7 @@
}
void insert() throws SQLException {
-
+
LOGGER.info("performing db insert for " + runSummary);
// Turn auto-commit off as this whole method is a single transaction.
@@ -150,7 +150,8 @@
this.delete();
} else {
// Rows exist but updating is disallowed which is a fatal error.
- final RuntimeException x = new RuntimeException("Run " + runSummary.getRun() + " already exists and deleting is not allowed.");
+ final RuntimeException x = new RuntimeException("Run " + runSummary.getRun()
+ + " already exists and deleting is not allowed.");
LOGGER.log(Level.SEVERE, x.getMessage(), x);
throw x;
}
@@ -184,7 +185,8 @@
* @throws SQLException if there is an error performing the db query
*/
private void insertEpics() throws SQLException {
- final PreparedStatement statement = connection.prepareStatement("INSERT INTO run_epics (run, variable_name, value) values (?, ?, ?)");
+ final PreparedStatement statement = connection
+ .prepareStatement("INSERT INTO run_epics (run, variable_name, value) values (?, ?, ?)");
final EpicsData data = runSummary.getEpicsData();
if (data != null) {
for (final String variableName : data.getUsedNames()) {
@@ -254,6 +256,11 @@
return rs.first();
}
+ /**
+ * Set whether deletion and replacement of existing run information is allowed.
+ *
+ * @param allowDeleteExisting <code>true</code> to allow deletion and replacement of existing information
+ */
void setAllowDeleteExisting(final boolean allowDeleteExisting) {
this.allowDeleteExisting = allowDeleteExisting;
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/ScalerDataUpdater.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/ScalerDataUpdater.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/ScalerDataUpdater.java Fri Jul 10 16:05:22 2015
@@ -8,38 +8,77 @@
import org.hps.record.scalers.ScalerData;
/**
- * Database interface for inserting scaler data into the run log.
+ * Inserts scaler data into the run database.
*
* @author Jeremy McCormick, SLAC
*/
public class ScalerDataUpdater {
+ /**
+ * <code>true</code> if existing data can be replaced and deleted.
+ */
private boolean allowDeleteExisting = false;
+ /**
+ * The database connection.
+ */
private final Connection connection;
+ /**
+ * The scaler data to insert into the database.
+ */
private final ScalerData data;
+ /**
+ * The run number for recording the scaler data.
+ */
private final int run;
+ /**
+ * SQL delete query string.
+ */
private final String SQL_DELETE = "DELETE FROM run_scalers WHERE run = ?";
+ /**
+ * SQL insert query string.
+ */
private final String SQL_INSERT = "INSERT INTO run_scalers (run, idx, value) VALUES (?, ?, ?)";
+ /**
+ * SQL select query string.
+ */
private final String SQL_SELECT = "SELECT run FROM run_scalers WHERE run = ?";
+ /**
+ * Create an updater for the given scaler data.
+ *
+ * @param connection the database connection
+ * @param data the scaler data
+ * @param run the run number
+ */
ScalerDataUpdater(final Connection connection, final ScalerData data, final int run) {
this.connection = connection;
this.data = data;
this.run = run;
}
+ /**
+ * Delete scaler data for the run.
+ *
+ * @throws SQLException if there is a SQL query error
+ */
private void delete() throws SQLException {
final PreparedStatement s = connection.prepareStatement(SQL_DELETE);
s.setInt(1, run);
s.executeUpdate();
}
+ /**
+ * Return <code>true</code> if scaler data exists for the run number.
+ *
+ * @return <code>true</code> if scaler data exists
+ * @throws SQLException if there is a SQL query error
+ */
private boolean exists() throws SQLException {
final PreparedStatement s = connection.prepareStatement(SQL_SELECT);
s.setInt(1, run);
@@ -47,6 +86,14 @@
return rs.first();
}
+ /**
+ * Insert scaler data for this run number into the database.
+ * <p>
+ * Includes check on whether data exists and can be replaced or not.
+ *
+ * @throws SQLException if there is a SQL query error
+ * @throws RuntimeException if data exists and cannot be deleted
+ */
void insert() throws SQLException {
if (this.exists()) {
if (allowDeleteExisting) {
@@ -62,6 +109,11 @@
}
}
+ /**
+ * Insert scaler data into the database.
+ *
+ * @throws SQLException if there is a SQL query error
+ */
private void insertScalerData() throws SQLException {
final PreparedStatement statement;
try {
@@ -77,6 +129,11 @@
}
}
+ /**
+ * Set whether existing data can be deleted and replaced.
+ *
+ * @param allowDeleteExisting <code>true</code> if existing data can be replaced
+ */
void setAllowDeleteExisting(final boolean allowDeleteExisting) {
this.allowDeleteExisting = allowDeleteExisting;
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/package-info.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/package-info.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/package-info.java Fri Jul 10 16:05:22 2015
@@ -1,8 +1,7 @@
/**
- * Implements an EVIO file crawler to extract run and configuration information.
+ * Implements an EVIO file crawler to extract run meta data and update a run database with this information.
* <p>
- * This information includes run start and end dates, event counts, EPICS data, scaler data,
- * and the list of associated EVIO files.
+ * This includes run start and end dates, event counts, EPICS data, scaler data, and the list of associated EVIO files.
*
* @author Jeremy McCormick, SLAC
*/
|
