Author: [log in to unmask]
Date: Tue Aug 4 15:11:36 2015
New Revision: 3333
Log:
Change crawler to use UTC times in milliseconds instead of SQL timestamps. HPSJAVA-570
Modified:
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileSequenceComparator.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java Tue Aug 4 15:11:36 2015
@@ -308,7 +308,7 @@
runs.sortFiles();
// Process all the files, performing caching from the MSS if necessary.
- RunProcessor.processRuns(this.cacheManager, runs, config);
+ RunProcessor.processAllRuns(this.cacheManager, runs, config);
// Print the summary information after the run processing is done.
runs.printRunSummaries();
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java Tue Aug 4 15:11:36 2015
@@ -21,7 +21,7 @@
final boolean isEvio = pathname.getName().contains(".evio");
boolean hasSeqNum = false;
try {
- EvioFileUtilities.getSequenceNumber(pathname);
+ EvioFileUtilities.getSequence(pathname);
hasSeqNum = true;
} catch (final Exception e) {
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileSequenceComparator.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileSequenceComparator.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileSequenceComparator.java Tue Aug 4 15:11:36 2015
@@ -17,8 +17,8 @@
*/
@Override
public int compare(final File o1, final File o2) {
- final Integer sequenceNumber1 = EvioFileUtilities.getSequenceNumber(o1);
- final Integer sequenceNumber2 = EvioFileUtilities.getSequenceNumber(o2);
+ final Integer sequenceNumber1 = EvioFileUtilities.getSequence(o1);
+ final Integer sequenceNumber2 = EvioFileUtilities.getSequence(o2);
return sequenceNumber1.compareTo(sequenceNumber2);
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java Tue Aug 4 15:11:36 2015
@@ -2,7 +2,6 @@
import java.io.File;
import java.io.IOException;
-import java.util.Date;
import java.util.logging.Logger;
import org.hps.record.evio.EvioEventConstants;
@@ -48,44 +47,50 @@
}
/**
- * Get the date from the control bank of an EVIO event.
- *
- * @param file the EVIO file
- * @param eventTag the event tag on the bank
- * @param gotoEvent an event to start the scanning
- * @return the control bank date or null if not found
- */
- static Date getControlDate(final File file, final int eventTag, final int gotoEvent) {
- Date date = null;
- EvioReader reader = null;
+ * Get the end date
+ *
+ * @param evioReader the <code>EvioReader</code>
+ * @return the run end date
+ */
+ static Long getEndTimestamp(final EvioReader evioReader) {
+
+ // Date endDate = null;
+ Long timestamp = null;
+
try {
- reader = open(file, true);
- EvioEvent event;
- if (gotoEvent > 0) {
- reader.gotoEventNumber(gotoEvent);
- } else if (gotoEvent < 0) {
- reader.gotoEventNumber(reader.getEventCount() + gotoEvent);
- }
- while ((event = reader.parseNextEvent()) != null) {
- if (event.getHeader().getTag() == eventTag) {
- final int[] data = EvioEventUtilities.getControlEventData(event);
- final long seconds = data[0];
- date = new Date(seconds * MILLISECONDS);
- break;
+ // Search for the last physics event in the last 5 events of the file.
+ System.out.println("going to event " + (evioReader.getEventCount() - 5) + " / "
+ + evioReader.getEventCount() + " to find end date");
+ evioReader.gotoEventNumber(evioReader.getEventCount() - 5);
+ EvioEvent evioEvent = null;
+ EvioEvent lastPhysicsEvent = null;
+
+ // Find last physics event.
+ while ((evioEvent = evioReader.parseNextEvent()) != null) {
+ if (EvioEventUtilities.isPhysicsEvent(evioEvent)) {
+ lastPhysicsEvent = evioEvent;
}
}
+
+ // If there is no physics event found this is an error.
+ if (lastPhysicsEvent == null) {
+ throw new RuntimeException("No physics event found.");
+ }
+
+ // Get the timestamp from the head bank of the physics event.
+ LOGGER.info("getting head bank date from " + lastPhysicsEvent.getEventNumber());
+ final Long eventTimestamp = getHeadBankTimestamp(lastPhysicsEvent);
+ if (eventTimestamp != null) {
+ LOGGER.info("found end timestamp " + eventTimestamp);
+ timestamp = eventTimestamp;
+ } else {
+ throw new RuntimeException("No timestamp found in head bank.");
+ }
+
} catch (EvioException | IOException e) {
throw new RuntimeException(e);
- } finally {
- if (reader != null) {
- try {
- reader.close();
- } catch (final IOException e) {
- e.printStackTrace();
- }
- }
- }
- return date;
+ }
+ return timestamp;
}
/**
@@ -94,63 +99,18 @@
* @param event the EVIO file
* @return the date from the head bank or null if not found
*/
- static Date getHeadBankDate(final EvioEvent event) {
- Date date = null;
+ static Long getHeadBankTimestamp(final EvioEvent event) {
+ // Date date = null;
+ Long timestamp = null;
final BaseStructure headBank = EvioEventUtilities.getHeadBank(event);
if (headBank != null) {
final int[] data = headBank.getIntData();
final long time = data[3];
if (time != 0L) {
- date = new Date(time * MILLISECONDS);
- }
- }
- return date;
- }
-
- /**
- * Get the run end date which is taken either from the END event or the last physics event if the END event is not
- * found in the file.
- *
- * @param file the EVIO file
- * @return the run end date
- */
- static Date getRunEnd(final File file) {
-
- // Search for the END event in the last 10 events of the file.
- Date endDate = getControlDate(file, EvioEventConstants.END_EVENT_TAG, -10);
-
- // Was the end date found from the END event?
- if (endDate == null) {
-
- EvioReader reader = null;
- try {
- reader = open(file, true);
-
- // Search for the last physics event in the last 10 events of the file.
- reader.gotoEventNumber(reader.getEventCount() - 10);
- EvioEvent event = null;
- while ((event = reader.parseNextEvent()) != null) {
- if (EvioEventUtilities.isPhysicsEvent(event)) {
- final Date eventDate = getHeadBankDate(event);
- if (eventDate != null) {
- // This might be set multiple times but should result in the time of the last physics event.
- endDate = eventDate;
- }
- }
- }
- } catch (EvioException | IOException e) {
- throw new RuntimeException(e);
- } finally {
- if (reader != null) {
- try {
- reader.close();
- } catch (final IOException e) {
- e.printStackTrace();
- }
- }
- }
- }
- return endDate;
+ timestamp = time * MILLISECONDS;
+ }
+ }
+ return timestamp;
}
/**
@@ -160,7 +120,7 @@
* @return the run number
* @throws Exception if there is a problem parsing out the run number
*/
- static Integer getRunFromName(final File file) {
+ static Integer getRun(final File file) {
final String name = file.getName();
final int startIndex = name.lastIndexOf("_") + 1;
final int endIndex = name.indexOf(".");
@@ -168,58 +128,71 @@
}
/**
- * Get the run start date from an EVIO file (should be the first file in the run).
- * <p>
- * This is taken from the PRESTART event if available or the first physics event.
- *
- * @param file the EVIO file
+ * Get the EVIO file sequence number, which is the number at the end of the file name.
+ *
+ * @param file the EVIO file
+ * @return the file's sequence number
+ * @throws Exception if there is an error parsing out the sequence number
+ */
+ static Integer getSequence(final File file) {
+ final String name = file.getName();
+ return Integer.parseInt(name.substring(name.lastIndexOf(".") + 1));
+ }
+
+ /**
+ * Get the start date from the first physics event.
+ *
+ * @param evioReader the <code>EvioReader</code>
* @return the run start date
*/
- static Date getRunStart(final File file) {
-
- // First try to find the start date in the special PRESTART event.
- Date date = getControlDate(file, EvioEventConstants.PRESTART_EVENT_TAG, 0);
-
- // Was start date not found from PRESTART?
- if (date == null) {
-
- // Read events until there is a physics event and use its time for the start date.
- EvioReader reader = null;
- try {
- reader = open(file, true);
- EvioEvent event = null;
- while ((event = reader.parseNextEvent()) != null) {
- if (EvioEventUtilities.isPhysicsEvent(event)) {
- if ((date = getHeadBankDate(event)) != null) {
- break;
- }
+ static Long getStartTimestamp(final EvioReader evioReader) {
+
+ Long timestamp = null;
+
+ // Read events until there is a physics event and return its timestamp.
+ try {
+ EvioEvent event = null;
+ while ((event = evioReader.parseNextEvent()) != null) {
+ if (EvioEventUtilities.isPhysicsEvent(event)) {
+ if ((timestamp = getHeadBankTimestamp(event)) != null) {
+ break;
}
}
- } catch (EvioException | IOException e) {
- throw new RuntimeException(e);
- } finally {
- if (reader != null) {
- try {
- reader.close();
- } catch (final IOException e) {
- e.printStackTrace();
- }
+ }
+ } catch (EvioException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ return timestamp;
+ }
+
+ /**
+ * Get the date from the control bank of an EVIO event.
+ *
+ * @param file the EVIO file
+ * @param eventTag the event tag on the bank
+ * @param gotoEvent an event to start the scanning
+ * @return the control bank date or null if not found
+ */
+ static Long getTimestamp(final EvioReader evioReader, final int eventTag, final int gotoEvent) {
+ Long timestamp = null;
+ try {
+ EvioEvent evioEvent;
+ if (gotoEvent > 0) {
+ evioReader.gotoEventNumber(gotoEvent);
+ } else if (gotoEvent < 0) {
+ evioReader.gotoEventNumber(evioReader.getEventCount() + gotoEvent);
+ }
+ while ((evioEvent = evioReader.parseNextEvent()) != null) {
+ if (evioEvent.getHeader().getTag() == eventTag) {
+ final int[] data = EvioEventUtilities.getControlEventData(evioEvent);
+ timestamp = (long) (data[0] * MILLISECONDS);
+ break;
}
}
- }
- return date;
- }
-
- /**
- * Get the EVIO file sequence number, which is the number at the end of the file name.
- *
- * @param file the EVIO file
- * @return the file's sequence number
- * @throws Exception if there is an error parsing out the sequence number
- */
- static Integer getSequenceNumber(final File file) {
- final String name = file.getName();
- return Integer.parseInt(name.substring(name.lastIndexOf(".") + 1));
+ } catch (EvioException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ return timestamp;
}
/**
@@ -230,6 +203,33 @@
*/
static boolean isCachedFile(final File file) {
return file.getPath().startsWith("/cache");
+ }
+
+ /**
+ * Return <code>true</code> if a valid CODA <i>END</i> event can be located in the <code>EvioReader</code>'s current
+ * file.
+ *
+ * @param reader the EVIO reader
+ * @return <code>true</code> if valid END event is located
+ * @throws Exception if there are IO problems using the reader
+ */
+ static boolean isEndOkay(final EvioReader reader) throws Exception {
+ LOGGER.info("checking is END okay ...");
+
+ boolean endOkay = false;
+
+ // Go to second to last event for searching.
+ reader.gotoEventNumber(reader.getEventCount() - 2);
+
+ // Look for END event.
+ EvioEvent event = null;
+ while ((event = reader.parseNextEvent()) != null) {
+ if (event.getHeader().getTag() == EvioEventConstants.END_EVENT_TAG) {
+ endOkay = true;
+ break;
+ }
+ }
+ return endOkay;
}
/**
@@ -287,4 +287,10 @@
static EvioReader open(final String path) throws IOException, EvioException {
return open(new File(path), false);
}
+
+ /**
+ * Present class instantiation.
+ */
+ private EvioFileUtilities() {
+ }
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java Tue Aug 4 15:11:36 2015
@@ -99,10 +99,10 @@
if (this.accept(file)) {
// Get the run number from the file name.
- final Integer run = EvioFileUtilities.getRunFromName(file);
+ final Integer run = EvioFileUtilities.getRun(file);
// Get the sequence number from the file name.
- final Integer seq = EvioFileUtilities.getSequenceNumber(file);
+ final Integer seq = EvioFileUtilities.getSequence(file);
LOGGER.info("accepted file " + file.getPath() + " with run " + run + " and seq " + seq);
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java Tue Aug 4 15:11:36 2015
@@ -299,7 +299,7 @@
/**
* The time in milliseconds when the caching operation started.
*/
- long start = 0;
+ private long start = 0;
/**
* Cache a file by submitting a 'jcache submit' process.
@@ -357,7 +357,7 @@
// Loop over all files, refresh the status, and check that they are cached.
for (final Entry<File, CacheStatus> entry : this.cacheStatuses.entrySet()) {
- // Get the cache status the file.
+ // Get the cache status of the file.
final CacheStatus cacheStatus = entry.getValue();
LOGGER.info("checking status of " + cacheStatus.getFile().getPath() + " with req ID '"
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunFilter.java Tue Aug 4 15:11:36 2015
@@ -36,6 +36,6 @@
*/
@Override
public boolean accept(final File file) {
- return this.acceptRuns.contains(EvioFileUtilities.getRunFromName(file));
+ return this.acceptRuns.contains(EvioFileUtilities.getRun(file));
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java Tue Aug 4 15:11:36 2015
@@ -3,7 +3,6 @@
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Date;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -41,13 +40,12 @@
* @param runs the run log containing the list of run summaries
* @throws Exception if there is an error processing one of the runs
*/
- static void processRuns(final JCacheManager cacheManager, final RunLog runs, final CrawlerConfig config)
+ static void processAllRuns(final JCacheManager cacheManager, final RunLog runs, final CrawlerConfig config)
throws Exception {
- // Configure max wait time of jcache manager.
+ // Configure the max wait time for file caching operations.
if (config.waitTime() != null && config.waitTime() > 0L) {
cacheManager.setWaitTime(config.waitTime());
- LOGGER.config("JCacheManager max wait time set to " + config.waitTime());
}
// Process all of the runs that were found.
@@ -71,7 +69,7 @@
}
// Process all of the run's files.
- runProcessor.process();
+ runProcessor.processRun();
}
}
@@ -86,17 +84,17 @@
private final EpicsLog epicsLog;
/**
+ * Processor for extracting event type counts (sync, physics, trigger types, etc.).
+ */
+ private final EventCountProcessor eventCountProcessor;
+
+ /**
* The event printing interval when processing EVIO files.
*/
private int eventPrintInterval = 1000;
/**
- * Processor for extracting event type counts (sync, physics, trigger types, etc.).
- */
- private final EventCountProcessor eventCountProcessor;
-
- /**
- * Max files to read (defaults to unlimited).
+ * Max total files to read (default is unlimited).
*/
private int maxFiles = -1;
@@ -189,15 +187,55 @@
}
/**
- * Get the event count from the current <code>EvioReader</code>.
- *
- * @param reader the current <code>EvioReader</code>
- * @return the event count
- * @throws IOException if there is a generic IO error
- * @throws EvioException if there is an EVIO related error
- */
- Integer computeEventCount(final EvioReader reader) throws IOException, EvioException {
- return reader.getEventCount();
+ * Find the end date in the EVIO events.
+ *
+ * @param evioReader the open <code>EvioReader</code>
+ */
+ private void findEndDate(final EvioReader evioReader) {
+
+ // Try to get end date from END event.
+ Long endTimestamp = EvioFileUtilities.getTimestamp(evioReader, EvioEventConstants.END_EVENT_TAG, -5);
+
+ if (endTimestamp != null) {
+ // Flag end okay for the run.
+ this.runSummary.setEndOkay(true);
+ } else {
+ // Try to find the end date from the last physics event.
+ endTimestamp = EvioFileUtilities.getEndTimestamp(evioReader);
+ this.runSummary.setEndOkay(false);
+ }
+
+ if (endTimestamp == null) {
+ // Not finding the end date is a fatal error.
+ throw new RuntimeException("Failed to find end date.");
+ }
+
+ LOGGER.info("found end timestamp " + endTimestamp);
+ this.runSummary.setEndTimeUtc(endTimestamp);
+ }
+
+ /**
+ * Find the start date in the EVIO events.
+ *
+ * @param evioReader the open <code>EvioReader</code>
+ */
+ private void findStartDate(final EvioReader evioReader) {
+
+ // First try to find the start date in the PRESTART event.
+ Long startTimestamp = EvioFileUtilities.getTimestamp(evioReader, EvioEventConstants.PRESTART_EVENT_TAG, 0);
+
+ if (startTimestamp == null) {
+ // Search for start date in first physics event.
+ startTimestamp = EvioFileUtilities.getStartTimestamp(evioReader);
+ }
+
+ if (startTimestamp == null) {
+ // Not finding the start date is a fatal error.
+ throw new RuntimeException("Failed to find start date.");
+ }
+
+ LOGGER.fine("got run start " + startTimestamp);
+ this.runSummary.setStartTimeUtc(startTimestamp);
}
/**
@@ -228,81 +266,50 @@
}
/**
- * Return <code>true</code> if a valid CODA <i>END</i> event can be located in the <code>EvioReader</code>'s current
- * file.
- *
- * @param reader the EVIO reader
- * @return <code>true</code> if valid END event is located
- * @throws Exception if there are IO problems using the reader
- */
- boolean isEndOkay(final EvioReader reader) throws Exception {
- LOGGER.info("checking is END okay ...");
-
- boolean endOkay = false;
-
- // Go to second to last event for searching.
- reader.gotoEventNumber(reader.getEventCount() - 2);
-
- // Look for END event.
- EvioEvent event = null;
- while ((event = reader.parseNextEvent()) != null) {
- if (event.getHeader().getTag() == EvioEventConstants.END_EVENT_TAG) {
- endOkay = true;
- break;
- }
- }
- return endOkay;
- }
-
- /**
- * Process the run by executing the registered {@link org.hps.record.evio.EvioEventProcessor}s and performing
- * special tasks such as the extraction of start and end dates.
- * <p>
- * This method will also activate file caching, if enabled by the {@link #useFileCache} option.
- *
- * @throws Exception if there is an error processing a file
- */
- void process() throws Exception {
-
- LOGGER.info("processing run " + this.runSummary.getRun() + " ...");
-
- // First cache all the files we will process, if necessary.
- if (this.useFileCache) {
- this.cacheFiles();
- }
-
- // Run the start of job hooks.
- for (final EvioEventProcessor processor : this.processors) {
- processor.startJob();
- }
-
- final List<File> files = this.getFiles();
-
- LOGGER.info("processing " + files.size() + " from run " + this.runSummary.getRun());
-
- // Process all the files.
- for (final File file : files) {
- this.process(file);
- }
-
- // Run the end job hooks.
- for (final EvioEventProcessor processor : this.processors) {
- processor.endJob();
- }
-
- // Put scaler data from EVIO processor into run summary.
- runSummary.setScalerData(this.scalersProcessor.getScalerData());
-
- // Set the counts of event types on the run summary.
- runSummary.setEventTypeCounts(eventCountProcessor.getEventCounts());
-
- // Set total number of events on the run summary from the event counter.
- runSummary.setTotalEvents(this.eventCountProcessor.getTotalEventCount());
-
- // Set EpicsData for the run.
- runSummary.setEpicsData(this.epicsLog.getEpicsData());
-
- LOGGER.info("done processing run " + this.runSummary.getRun());
+ * Return <code>true</code> if the file is the first one in the list for the run.
+ *
+ * @param file the EVIO <code>File</code>
+ * @return <code>true</code> if the file is the first one in the list for the run
+ */
+ private boolean isFirstFile(final File file) {
+ return file.equals(this.runSummary.getEvioFileList().first());
+ }
+
+ /**
+ * Return <code>true</code> if the file is the last one in the list for the run.
+ *
+ * @param file the EVIO <code>File</code>
+ * @return <code>true</code> if the file is the last one in the list for the run
+ */
+ private boolean isLastFile(final File file) {
+ return file.equals(this.getFiles().get(this.getFiles().size() - 1));
+ }
+
+ /**
+ * Process events using the list of EVIO processors.
+ *
+ * @param evioReader the open <code>EvioReader</code>
+ * @throws IOException if there is a file IO error
+ * @throws EvioException if there is an EVIO error
+ * @throws Exception if there is some other error
+ */
+ private void processEvents(final EvioReader evioReader) throws IOException, EvioException, Exception {
+ LOGGER.finer("running EVIO processors ...");
+ evioReader.gotoEventNumber(0);
+ int nProcessed = 0;
+ if (!this.processors.isEmpty()) {
+ EvioEvent event = null;
+ while ((event = evioReader.parseNextEvent()) != null) {
+ for (final EvioEventProcessor processor : this.processors) {
+ processor.process(event);
+ }
+ ++nProcessed;
+ if (nProcessed % this.eventPrintInterval == 0) {
+ LOGGER.finer("processed " + nProcessed + " EVIO events");
+ }
+ }
+ LOGGER.info("done running EVIO processors");
+ }
}
/**
@@ -313,60 +320,91 @@
* @throws IOException if there is some kind of IO error
* @throws Exception if there is a generic error thrown by event processing
*/
- private void process(final File file) throws EvioException, IOException, Exception {
- LOGGER.fine("processing " + file.getPath() + " ...");
-
- EvioReader reader = null;
+ private void processFile(final File file) throws EvioException, IOException, Exception {
+
+ LOGGER.fine("processing file " + file.getPath() + " ...");
+
+ EvioReader evioReader = null;
try {
- // Open with wrapper method which will use the cached file path if necessary.
- LOGGER.fine("opening " + file.getPath() + " for reading ...");
- reader = EvioFileUtilities.open(file, true);
- LOGGER.fine("done opening " + file.getPath());
+
+ // Open file for reading (flag should be true for sequential or false for mem map).
+ evioReader = EvioFileUtilities.open(file, true);
// If this is the first file then get the start date.
- if (file.equals(this.runSummary.getEvioFileList().first())) {
- LOGGER.fine("getting run start ...");
- final Date runStart = EvioFileUtilities.getRunStart(file);
- LOGGER.fine("got run start " + runStart);
- this.runSummary.setStartDate(runStart);
- }
-
- // Process the events using the EVIO processors.
- LOGGER.info("running EVIO processors ...");
- reader.gotoEventNumber(0);
- int nProcessed = 0;
- if (!this.processors.isEmpty()) {
- EvioEvent event = null;
- while ((event = reader.parseNextEvent()) != null) {
- for (final EvioEventProcessor processor : this.processors) {
- processor.process(event);
- ++nProcessed;
- if (nProcessed % this.eventPrintInterval == 0) {
- LOGGER.finer("processed " + nProcessed + " EVIO events");
- }
- }
- }
- }
- LOGGER.info("done running EVIO processors");
-
- // Check if END event is present if this is the last file in the run.
- if (file.equals(this.getFiles().get(this.getFiles().size() - 1))) {
- final boolean endOkay = this.isEndOkay(reader);
- this.runSummary.setEndOkay(endOkay);
- LOGGER.info("endOkay set to " + endOkay);
-
- LOGGER.info("getting end date ...");
- final Date endDate = EvioFileUtilities.getRunEnd(file);
- this.runSummary.setEndDate(endDate);
- LOGGER.info("found end date " + endDate);
+ if (this.isFirstFile(file)) {
+ LOGGER.fine("getting run start from first file " + file.getPath() + " ...");
+ this.findStartDate(evioReader);
+ }
+
+ // Go back to the first event and process the events using the list of EVIO processors.
+ this.processEvents(evioReader);
+
+ // Find end date from last file in the run.
+ if (this.isLastFile(file)) {
+ LOGGER.fine("getting run end from last file " + file.getPath() + " ...");
+ this.findEndDate(evioReader);
}
} finally {
- if (reader != null) {
- reader.close();
+ // Close the EvioReader for the current file.
+ if (evioReader != null) {
+ evioReader.close();
}
}
LOGGER.fine("done processing " + file.getPath());
+ }
+
+ /**
+ * Process the run by executing the registered {@link org.hps.record.evio.EvioEventProcessor}s extracting the start
+ * and end dates.
+ * <p>
+ * This method will also activate file caching, if enabled by the {@link #useFileCache} option.
+ *
+ * @throws Exception if there is an error processing a file
+ */
+ void processRun() throws Exception {
+
+ LOGGER.info("processing run " + this.runSummary.getRun() + " ...");
+
+ // First cache all the files we will process, if necessary.
+ if (this.useFileCache) {
+ this.cacheFiles();
+ }
+
+ // Run the start of job hooks.
+ for (final EvioEventProcessor processor : this.processors) {
+ processor.startJob();
+ }
+
+ // Get the list of files, limited by max files setting.
+ final List<File> files = this.getFiles();
+
+ LOGGER.info("processing " + files.size() + " from run " + this.runSummary.getRun());
+
+ // Process all the files.
+ for (final File file : files) {
+ this.processFile(file);
+ }
+
+ // Run the end job hooks.
+ LOGGER.info("running end of job hooks on EVIO processors ...");
+ for (final EvioEventProcessor processor : this.processors) {
+ processor.endJob();
+ }
+
+ // Put scaler data from EVIO processor into run summary.
+ runSummary.setScalerData(this.scalersProcessor.getScalerData());
+
+ // Set the counts of event types on the run summary.
+ runSummary.setEventTypeCounts(eventCountProcessor.getEventCounts());
+
+ // Set total number of events on the run summary from the event counter.
+ runSummary.setTotalEvents(this.eventCountProcessor.getTotalEventCount());
+
+ // Set EpicsData for the run.
+ runSummary.setEpicsData(this.epicsLog.getEpicsData());
+
+ LOGGER.info("done processing run " + this.runSummary.getRun());
}
/**
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java Tue Aug 4 15:11:36 2015
@@ -239,11 +239,11 @@
*/
private void insertRun() throws SQLException {
final PreparedStatement statement = connection
- .prepareStatement("INSERT INTO runs (run, start_date, end_date, nevents, nfiles, end_ok, created) VALUES(?, ?, ?, ?, ?, ?, NOW())");
+ .prepareStatement("INSERT INTO runs (run, start_time_utc, end_time_utc, nevents, nfiles, end_ok, created) VALUES(?, ?, ?, ?, ?, ?, NOW())");
LOGGER.info("preparing to insert run " + run + " into runs table ..");
statement.setInt(1, run);
- statement.setTimestamp(2, new java.sql.Timestamp(runSummary.getStartDate().getTime()));
- statement.setTimestamp(3, new java.sql.Timestamp(runSummary.getEndDate().getTime()));
+ statement.setLong(2, runSummary.getStartTimeUtc());
+ statement.setLong(3, runSummary.getEndTimeUtc());
statement.setInt(4, runSummary.getTotalEvents());
statement.setInt(5, runSummary.getEvioFileList().size());
statement.setBoolean(6, runSummary.getEndOkay());
@@ -253,7 +253,7 @@
/**
* Insert scaler data into the database.
- *
+ *
* @throws SQLException if there is a SQL query error
*/
private void insertScalarData() throws SQLException {
