Author: [log in to unmask]
Date: Fri May 15 18:30:03 2015
New Revision: 2980
Log:
Minor improvements to file crawler.
Modified:
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileCrawler.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileCrawler.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileCrawler.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileCrawler.java Fri May 15 18:30:03 2015
@@ -17,6 +17,7 @@
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
+import org.lcsim.util.log.DefaultLogFormatter;
import org.lcsim.util.log.LogUtil;
/**
@@ -28,13 +29,9 @@
*/
public final class EvioFileCrawler {
- private static final Logger LOGGER = LogUtil.create(EvioFileVisitor.class);
+ private static final Logger LOGGER = LogUtil.create(EvioFileVisitor.class, new DefaultLogFormatter(), Level.ALL);
private static final Options OPTIONS = new Options();
-
- static {
- LOGGER.setLevel(Level.ALL);
- }
static {
OPTIONS.addOption("h", "help", false, "print help and exit");
@@ -217,7 +214,7 @@
final RunSummary runSummary = runs.getRunSummary(run);
// Create a processor to process all the EVIO records in the run.
- final RunProcessor processor = createRunProcessor(runSummary);
+ final RunProcessor processor = createRunProcessor(runSummary);
// Process the run, updating the run summary.
processor.process();
@@ -245,7 +242,15 @@
}
final RunLog runs = visitor.getRunLog();
-
+
+ // Print run numbers that were found.
+ StringBuffer sb = new StringBuffer();
+ for (Integer run : runs.getSortedRunNumbers()) {
+ sb.append(run + " ");
+ }
+ LOGGER.info("found files from runs: " + sb.toString());
+
+ // Sort files on their sequence numbers.
LOGGER.fine("sorting files by sequence ...");
runs.sortAllFiles();
@@ -257,7 +262,7 @@
// Process all the files in the runs.
processRuns(runs);
- // Print the run summaries.
+ // Print the run summary information.
if (this.printSummary) {
runs.printRunSummaries();
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileFilter.java Fri May 15 18:30:03 2015
@@ -5,8 +5,15 @@
final class EvioFileFilter implements FileFilter {
- @Override
+ @Override
public boolean accept(final File pathname) {
- return pathname.getName().contains(".evio");
+ boolean isEvio = pathname.getName().contains(".evio");
+ boolean hasSeqNum = false;
+ try {
+ EvioFileUtilities.getSequenceNumber(pathname);
+ hasSeqNum = true;
+ } catch (Exception e) {
+ }
+ return isEvio && hasSeqNum;
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java Fri May 15 18:30:03 2015
@@ -137,12 +137,16 @@
}
static EvioReader open(final File file) throws IOException, EvioException {
+ return open(file, false);
+ }
+
+ static EvioReader open(final File file, boolean sequential) throws IOException, EvioException {
File openFile = file;
if (isMssFile(file)) {
openFile = getCachedFile(file);
}
final long start = System.currentTimeMillis();
- final EvioReader reader = new EvioReader(openFile, false, false);
+ final EvioReader reader = new EvioReader(openFile, false, sequential);
final long end = System.currentTimeMillis() - start;
LOGGER.info("opened " + openFile.getPath() + " in " + end / MILLISECONDS + " seconds");
return reader;
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileVisitor.java Fri May 15 18:30:03 2015
@@ -8,6 +8,7 @@
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
+import java.util.logging.Level;
import java.util.logging.Logger;
import org.lcsim.util.log.LogUtil;
@@ -57,7 +58,6 @@
LOGGER.info("adding file: " + file.getPath() + "; run: " + run + "; seq = " + seq);
this.runs.getRunSummary(run).addFile(file);
-
} else {
LOGGER.info("rejected file: " + file.getPath());
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java Fri May 15 18:30:03 2015
@@ -20,14 +20,9 @@
Map<Integer, RunSummary> runs = new HashMap<Integer, RunSummary>();
- // void cache() {
- // for (final int run : getSortedRunNumbers()) {
- // this.runs.get(run).getFiles().cache();
- // }
- // }
-
public RunSummary getRunSummary(final int run) {
if (!this.runs.containsKey(run)) {
+ LOGGER.info("creating new RunSummary for run " + run);
this.runs.put(run, new RunSummary(run));
}
return this.runs.get(run);
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java Fri May 15 18:30:03 2015
@@ -34,14 +34,13 @@
}
void process() throws Exception {
- if (this.processors.isEmpty()) {
- throw new RuntimeException("The processors list is empty.");
- }
// Run the start of job hooks.
for (final EvioEventProcessor processor : this.processors) {
processor.startJob();
}
+
+
// Process the files.
for (final File file : this.runSummary.getFiles()) {
