Author: [log in to unmask]
Date: Fri May 1 19:03:59 2015
New Revision: 2881
Log:
Break EVIO crawler into proper class structure.
Added:
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/DateFileFilter.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileCrawler.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileList.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileSequenceComparator.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileUtilities.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileVisitor.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFilter.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunFilter.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunLog.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunSummary.java
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/DateFileFilter.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/DateFileFilter.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/DateFileFilter.java Fri May 1 19:03:59 2015
@@ -0,0 +1,28 @@
+package org.hps.users.jeremym.crawler;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Date;
+
+class DateFileFilter implements FileFilter {
+
+ private final Date date;
+
+ DateFileFilter(final Date date) {
+ this.date = date;
+ }
+
+ @Override
+ public boolean accept(final File pathname) {
+ BasicFileAttributes attr = null;
+ try {
+ attr = Files.readAttributes(pathname.toPath(), BasicFileAttributes.class);
+ } catch (final IOException e) {
+ throw new RuntimeException("Error getting file attributes.", e);
+ }
+ return attr.creationTime().toMillis() > this.date.getTime();
+ }
+}
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileCrawler.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileCrawler.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileCrawler.java Fri May 1 19:03:59 2015
@@ -0,0 +1,188 @@
+package org.hps.users.jeremym.crawler;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Files;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.lcsim.util.log.LogUtil;
+
+/**
+ * Crawls EVIO files in a directory tree, groups the files that are found by run, and optionally performs various tasks
+ * based on the run summary information, including printing a summary, caching the files from JLAB MSS, and updating a
+ * run database.
+ *
+ * @author <a href="mailto:[log in to unmask]">Jeremy McCormick</a>
+ */
+public class EvioFileCrawler {
+
+ private static final Logger LOGGER = LogUtil.create(EvioFileVisitor.class);
+
+ private static final Options OPTIONS = new Options();
+
+ static {
+ LOGGER.setLevel(Level.ALL);
+ }
+
+ static {
+ OPTIONS.addOption("t", "timestamp-file", true,
+ "timestamp file for date filtering; modified time will be set at end of job");
+ OPTIONS.addOption("d", "directory", true, "starting directory");
+ OPTIONS.addOption("r", "runs", true, "list of runs to accept (others will be excluded)");
+ OPTIONS.addOption("c", "cache", false, "cache files to /cache/mss from MSS (only works at JLAB)");
+ OPTIONS.addOption("p", "print", false, "print run summary at end of job");
+ OPTIONS.addOption("L", "log-level", true, "set log level (INFO, FINE, etc.)");
+ OPTIONS.addOption("u", "update", false, "update the run database");
+ }
+
+ public static void main(final String[] args) {
+ new EvioFileCrawler().parse(args).run();
+ }
+
+ final Set<Integer> acceptRuns = new HashSet<Integer>();
+
+ boolean cache = false;
+
+ final PosixParser parser = new PosixParser();
+
+ boolean printSummary = false;
+
+ File rootDir = new File(System.getProperty("user.dir"));
+
+ Date timestamp = null;
+
+ File timestampFile = null;
+
+ boolean update = false;
+
+ private EvioFileCrawler parse(final String args[]) {
+ try {
+ final CommandLine cl = this.parser.parse(OPTIONS, args);
+
+ if (cl.hasOption("L")) {
+ final Level level = Level.parse(cl.getOptionValue("L"));
+ LOGGER.info("setting log level to " + level);
+ LOGGER.setLevel(level);
+ }
+
+ if (cl.hasOption("d")) {
+ this.rootDir = new File(cl.getOptionValue("d"));
+ if (!this.rootDir.exists()) {
+ throw new IllegalArgumentException("The directory does not exist.");
+ }
+ if (!this.rootDir.isDirectory()) {
+ throw new IllegalArgumentException("The specified path is not a directory.");
+ }
+ }
+
+ if (cl.hasOption("t")) {
+ this.timestampFile = new File(cl.getOptionValue("t"));
+ if (!this.timestampFile.exists()) {
+ throw new IllegalArgumentException("The timestamp file does not exist: "
+ + this.timestampFile.getPath());
+ }
+ try {
+ this.timestamp = new Date(Files
+ .readAttributes(this.timestampFile.toPath(), BasicFileAttributes.class).lastModifiedTime()
+ .toMillis());
+ } catch (final IOException e) {
+ throw new RuntimeException("Error getting attributes of timestamp file.", e);
+ }
+ }
+
+ if (cl.hasOption("r")) {
+ for (final String runString : cl.getOptionValues("r")) {
+ final Integer acceptRun = Integer.parseInt(runString);
+ this.acceptRuns.add(acceptRun);
+ LOGGER.config("added accept run " + acceptRun);
+ }
+ }
+
+ if (cl.hasOption("p")) {
+ this.printSummary = true;
+ }
+
+ if (cl.hasOption("u")) {
+ this.update = true;
+ }
+
+ if (cl.hasOption("c")) {
+ this.cache = true;
+ }
+
+ if (this.cache && (this.printSummary || this.update)) {
+ // If file caching is selected, then printing run summary or updating the database won't work.
+ throw new IllegalArgumentException("File caching cannot be activated with the -p or -u options.");
+ }
+
+ } catch (final ParseException e) {
+ throw new RuntimeException("Error parsing options.", e);
+ }
+
+ return this;
+ }
+
+ public void run() {
+ final EnumSet<FileVisitOption> options = EnumSet.noneOf(FileVisitOption.class);
+ final EvioFileVisitor visitor = new EvioFileVisitor();
+ if (this.timestamp != null) {
+ visitor.addFilter(new DateFileFilter(this.timestamp));
+ LOGGER.config("added date filter with timestamp " + this.timestamp);
+ }
+ if (!this.acceptRuns.isEmpty()) {
+ visitor.addFilter(new RunFilter(this.acceptRuns));
+ LOGGER.config("added run filter");
+ }
+ try {
+ // Walk the file tree from the root directory.
+ Files.walkFileTree(this.rootDir.toPath(), options, Integer.MAX_VALUE, visitor);
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ final RunLog runs = visitor.getRunLog();
+
+ LOGGER.fine("sorting files by sequence ...");
+ runs.sortAllFiles();
+
+ if (this.cache) {
+ // Cache files from MSS.
+ runs.cache();
+ } else {
+
+ // Print the run summaries.
+ if (this.printSummary) {
+ runs.printRunSummaries();
+ }
+
+ // Insert run summary into run_log table.
+ if (this.update) {
+ runs.update();
+ }
+ }
+
+ // Update timestamp file.
+ if (this.timestampFile == null) {
+ this.timestampFile = new File("timestamp");
+ try {
+ this.timestampFile.createNewFile();
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ LOGGER.info("created new timestamp file: " + this.timestampFile.getPath());
+ }
+ this.timestampFile.setLastModified(System.currentTimeMillis());
+ LOGGER.info("set modified on timestamp file: " + new Date(this.timestampFile.lastModified()));
+ }
+}
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileList.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileList.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileList.java Fri May 1 19:03:59 2015
@@ -0,0 +1,56 @@
+package org.hps.users.jeremym.crawler;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.jlab.coda.jevio.EvioException;
+import org.jlab.coda.jevio.EvioReader;
+
+class EvioFileList extends ArrayList<File> {
+
+ void cache() {
+ for (final File file : this) {
+ EvioFileUtilities.cache(file);
+ }
+ }
+
+ int computeTotalEvents() {
+ int totalEvents = 0;
+ for (final File file : this) {
+ EvioReader reader = null;
+ try {
+ reader = new EvioReader(file, false);
+ totalEvents += reader.getEventCount();
+ } catch (EvioException | IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (final IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+ return totalEvents;
+ }
+
+ File first() {
+ return this.get(0);
+ }
+
+ File last() {
+ return this.get(this.size() - 1);
+ }
+
+ void sort() {
+ final List<File> fileList = new ArrayList<File>(this);
+ Collections.sort(fileList, new EvioFileSequenceComparator());
+ this.clear();
+ this.addAll(fileList);
+ }
+}
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileSequenceComparator.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileSequenceComparator.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileSequenceComparator.java Fri May 1 19:03:59 2015
@@ -0,0 +1,14 @@
+package org.hps.users.jeremym.crawler;
+
+import java.io.File;
+import java.util.Comparator;
+
+class EvioFileSequenceComparator implements Comparator<File> {
+
+ @Override
+ public int compare(final File o1, final File o2) {
+ final Integer sequenceNumber1 = EvioFileUtilities.getSequenceNumber(o1);
+ final Integer sequenceNumber2 = EvioFileUtilities.getSequenceNumber(o2);
+ return sequenceNumber1.compareTo(sequenceNumber2);
+ }
+}
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileUtilities.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileUtilities.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileUtilities.java Fri May 1 19:03:59 2015
@@ -0,0 +1,158 @@
+package org.hps.users.jeremym.crawler;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Date;
+import java.util.logging.Logger;
+
+import org.hps.record.evio.EvioEventConstants;
+import org.hps.record.evio.EvioEventUtilities;
+import org.jlab.coda.jevio.BaseStructure;
+import org.jlab.coda.jevio.EvioEvent;
+import org.jlab.coda.jevio.EvioException;
+import org.jlab.coda.jevio.EvioReader;
+import org.lcsim.util.log.LogUtil;
+
+public class EvioFileUtilities {
+
+ private static final Logger LOGGER = LogUtil.create(EvioFileUtilities.class);
+
+ private static final long MILLISECONDS = 1000L;
+
+ static void cache(final File file) {
+ if (!file.getPath().startsWith("/mss")) {
+ throw new IllegalArgumentException("Only files on /mss can be cached.");
+ }
+ try {
+ new ProcessBuilder("jcache", "submit", "default", file.getPath()).start();
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ LOGGER.info("process started to cache " + file.getPath());
+ }
+
+ static Date getControlDate(final File file, final int eventTag, final int gotoEvent) {
+ Date date = null;
+ EvioReader reader = null;
+ try {
+ reader = open(file);
+ EvioEvent event;
+ if (gotoEvent > 0) {
+ reader.gotoEventNumber(gotoEvent);
+ } else if (gotoEvent < 0) {
+ reader.gotoEventNumber(reader.getEventCount() + gotoEvent);
+ }
+ while ((event = reader.parseNextEvent()) != null) {
+ if (event.getHeader().getTag() == eventTag) {
+ final int[] data = EvioEventUtilities.getControlEventData(event);
+ final long seconds = data[0];
+ date = new Date(seconds * MILLISECONDS);
+ break;
+ }
+ }
+ } catch (EvioException | IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (final IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return date;
+ }
+
+ static Date getHeadBankDate(final EvioEvent event) {
+ Date date = null;
+ final BaseStructure headBank = EvioEventUtilities.getHeadBank(event);
+ if (headBank != null) {
+ final int[] data = headBank.getIntData();
+ final long time = data[3];
+ if (time != 0L) {
+ date = new Date(time * MILLISECONDS);
+ }
+ }
+ return date;
+ }
+
+ static Date getRunEnd(final File file) {
+ Date date = getControlDate(file, EvioEventConstants.END_EVENT_TAG, -10);
+ if (date == null) {
+ EvioReader reader = null;
+ try {
+ reader = open(file);
+ reader.gotoEventNumber(reader.getEventCount() - 11);
+ EvioEvent event = null;
+ while ((event = reader.parseNextEvent()) != null) {
+ if (EvioEventUtilities.isPhysicsEvent(event)) {
+ if ((date = getHeadBankDate(event)) != null) {
+ break;
+ }
+ }
+ }
+ } catch (EvioException | IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (final IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+ return date;
+ }
+
+ static Integer getRunFromName(final File file) {
+ final String name = file.getName();
+ final int startIndex = name.lastIndexOf("_") + 1;
+ final int endIndex = name.indexOf(".");
+ return Integer.parseInt(name.substring(startIndex, endIndex));
+ }
+
+ static Date getRunStart(final File file) {
+ Date date = getControlDate(file, EvioEventConstants.PRESTART_EVENT_TAG, 0);
+ if (date == null) {
+ EvioReader reader = null;
+ try {
+ reader = open(file);
+ EvioEvent event = null;
+ while ((event = reader.parseNextEvent()) != null) {
+ if (EvioEventUtilities.isPhysicsEvent(event)) {
+ if ((date = getHeadBankDate(event)) != null) {
+ break;
+ }
+ }
+ }
+ } catch (EvioException | IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (final IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+ return date;
+ }
+
+ static Integer getSequenceNumber(final File file) {
+ final String name = file.getName();
+ return Integer.parseInt(name.substring(name.lastIndexOf(".") + 1));
+ }
+
+ static EvioReader open(final File file) throws IOException, EvioException {
+ final long start = System.currentTimeMillis();
+ final EvioReader reader = new EvioReader(file, false, false);
+ final long end = System.currentTimeMillis() - start;
+ LOGGER.info("opened " + file.getPath() + " in " + end / MILLISECONDS + " seconds");
+ return reader;
+ }
+}
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileVisitor.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileVisitor.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileVisitor.java Fri May 1 19:03:59 2015
@@ -0,0 +1,65 @@
+package org.hps.users.jeremym.crawler;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+import org.lcsim.util.log.LogUtil;
+
+class EvioFileVisitor extends SimpleFileVisitor<Path> {
+
+ private static final Logger LOGGER = LogUtil.create(EvioFileVisitor.class);
+
+ List<FileFilter> filters = new ArrayList<FileFilter>();
+
+ RunLog runs = new RunLog();
+
+ EvioFileVisitor() {
+ addFilter(new EvioFilter());
+ }
+
+ boolean accept(final File file) {
+ boolean accept = true;
+ for (final FileFilter filter : this.filters) {
+ accept = filter.accept(file);
+ if (accept == false) {
+ LOGGER.fine(filter.getClass().getSimpleName() + " rejected file: " + file.getPath());
+ break;
+ }
+ }
+ return accept;
+ }
+
+ void addFilter(final FileFilter filter) {
+ this.filters.add(filter);
+ LOGGER.config("added filter: " + filter.getClass().getSimpleName());
+ }
+
+ RunLog getRunLog() {
+ return this.runs;
+ }
+
+ @Override
+ public FileVisitResult visitFile(final Path path, final BasicFileAttributes attrs) {
+
+ final File file = path.toFile();
+ if (accept(file)) {
+
+ final Integer run = EvioFileUtilities.getRunFromName(file);
+ final Integer seq = EvioFileUtilities.getSequenceNumber(file);
+
+ LOGGER.info("adding file: " + file.getPath() + "; run: " + run + "; seq = " + seq);
+
+ this.runs.getRunSummary(run).addFile(file);
+ } else {
+ LOGGER.fine("rejected file: " + file.getPath());
+ }
+ return FileVisitResult.CONTINUE;
+ }
+}
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFilter.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFilter.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFilter.java Fri May 1 19:03:59 2015
@@ -0,0 +1,12 @@
+package org.hps.users.jeremym.crawler;
+
+import java.io.File;
+import java.io.FileFilter;
+
+class EvioFilter implements FileFilter {
+
+ @Override
+ public boolean accept(final File pathname) {
+ return pathname.getName().contains(".evio");
+ }
+}
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunFilter.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunFilter.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunFilter.java Fri May 1 19:03:59 2015
@@ -0,0 +1,18 @@
+package org.hps.users.jeremym.crawler;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.util.Set;
+
+class RunFilter implements FileFilter {
+ Set<Integer> acceptRuns;
+
+ RunFilter(final Set<Integer> acceptRuns) {
+ this.acceptRuns = acceptRuns;
+ }
+
+ @Override
+ public boolean accept(final File file) {
+ return this.acceptRuns.contains(EvioFileUtilities.getRunFromName(file));
+ }
+}
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunLog.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunLog.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunLog.java Fri May 1 19:03:59 2015
@@ -0,0 +1,98 @@
+package org.hps.users.jeremym.crawler;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.hps.conditions.database.ConnectionParameters;
+import org.lcsim.util.log.LogUtil;
+
+class RunLog {
+
+ private static final Logger LOGGER = LogUtil.create(RunLog.class);
+
+ Map<Integer, RunSummary> runs = new HashMap<Integer, RunSummary>();
+
+ void cache() {
+ for (final int run : getSortedRunNumbers()) {
+ this.runs.get(run).getFiles().cache();
+ }
+ }
+
+ public RunSummary getRunSummary(final int run) {
+ if (!this.runs.containsKey(run)) {
+ this.runs.put(run, new RunSummary(run));
+ }
+ return this.runs.get(run);
+ }
+
+ List<Integer> getSortedRunNumbers() {
+ final List<Integer> runList = new ArrayList<Integer>(this.runs.keySet());
+ Collections.sort(runList);
+ return runList;
+ }
+
+ void printRunSummaries() {
+ for (final int run : this.runs.keySet()) {
+ this.runs.get(run).printRunSummary(System.out);
+ }
+ }
+
+ void sortAllFiles() {
+ for (final Integer run : this.runs.keySet()) {
+ this.runs.get(run).sortFiles();
+ }
+ }
+
+ void update() {
+ LOGGER.info("updating database from run log ...");
+ final ConnectionParameters cp = new ConnectionParameters("root", "derp", "hps_run_db", "localhost");
+ Connection connection = null;
+ PreparedStatement runLogStatement = null;
+ try {
+ connection = cp.createConnection();
+ connection.setAutoCommit(false);
+ runLogStatement = connection
+ .prepareStatement("INSERT INTO run_log (run, start_date, end_date, nevents, nfiles, end_ok, last_updated) VALUES(?, ?, ?, ?, ?, ?, NOW())");
+ for (final Integer run : getSortedRunNumbers()) {
+ LOGGER.info("inserting run " + run + " into database");
+ final RunSummary runSummary = this.runs.get(run);
+ runLogStatement.setInt(1, run);
+ runLogStatement.setTimestamp(2, new java.sql.Timestamp(runSummary.getStartDate().getTime()));
+ runLogStatement.setTimestamp(3, new java.sql.Timestamp(runSummary.getEndDate().getTime()));
+ runLogStatement.setInt(4, runSummary.getTotalEvents());
+ runLogStatement.setInt(5, runSummary.getFiles().size());
+ runLogStatement.setBoolean(6, runSummary.isEndOkay());
+ runLogStatement.executeUpdate();
+ connection.commit();
+ }
+ } catch (final SQLException e) {
+ LOGGER.log(Level.SEVERE, "rolling back transaction", e);
+ try {
+ connection.rollback();
+ } catch (final SQLException e2) {
+ throw new RuntimeException(e);
+ }
+ } finally {
+ if (connection != null) {
+ try {
+ connection.setAutoCommit(true);
+ if (!connection.isClosed()) {
+ connection.close();
+ }
+ } catch (final SQLException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ LOGGER.info("database was updated!");
+ }
+
+}
Added: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunSummary.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunSummary.java (added)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunSummary.java Fri May 1 19:03:59 2015
@@ -0,0 +1,110 @@
+package org.hps.users.jeremym.crawler;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Date;
+import java.util.logging.Logger;
+
+import org.hps.record.evio.EvioEventConstants;
+import org.jlab.coda.jevio.EvioEvent;
+import org.jlab.coda.jevio.EvioException;
+import org.jlab.coda.jevio.EvioReader;
+import org.lcsim.util.log.LogUtil;
+
+class RunSummary {
+
+ private static final Logger LOGGER = LogUtil.create(RunSummary.class);
+
+ private Date endDate;
+ private final EvioFileList files = new EvioFileList();
+ private Boolean isEndOkay;
+ private final int run;
+ private Date startDate;
+ private int totalEvents = -1;
+
+ RunSummary(final int run) {
+ this.run = run;
+ }
+
+ void addFile(final File file) {
+ this.files.add(file);
+
+ // Total events must be recomputed.
+ this.totalEvents = -1;
+ }
+
+ Date getEndDate() {
+ if (this.endDate == null) {
+ this.endDate = EvioFileUtilities.getRunEnd(this.files.last());
+ }
+ return this.endDate;
+ }
+
+ EvioFileList getFiles() {
+ return this.files;
+ }
+
+ Date getStartDate() {
+ if (this.startDate == null) {
+ this.startDate = EvioFileUtilities.getRunStart(this.files.first());
+ }
+ return this.startDate;
+ }
+
+ int getTotalEvents() {
+ if (this.totalEvents == -1) {
+ this.totalEvents = this.files.computeTotalEvents();
+ }
+ return this.totalEvents;
+ }
+
+ boolean isEndOkay() {
+ if (this.isEndOkay == null) {
+ LOGGER.info("checking is END okay ...");
+ this.isEndOkay = false;
+ final File lastFile = this.files.last();
+ EvioReader reader = null;
+ try {
+ reader = new EvioReader(lastFile, false);
+ reader.gotoEventNumber(reader.getEventCount() - 5);
+ EvioEvent event = null;
+ while ((event = reader.parseNextEvent()) != null) {
+ if (event.getHeader().getTag() == EvioEventConstants.END_EVENT_TAG) {
+ this.isEndOkay = true;
+ break;
+ }
+ }
+ } catch (EvioException | IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (final IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+ return this.isEndOkay;
+ }
+
+ void printRunSummary(final PrintStream ps) {
+ ps.println("--------------------------------------------");
+ ps.println("run: " + this.run);
+ ps.println("first file: " + this.files.first());
+ ps.println("last file: " + this.files.last());
+ ps.println("started: " + getStartDate());
+ ps.println("ended: " + getEndDate());
+ ps.println("total events: " + this.getTotalEvents());
+ ps.println("files: " + this.files.size());
+ for (final File file : this.files) {
+ ps.println(file.getPath());
+ }
+ }
+
+ void sortFiles() {
+ this.files.sort();
+ }
+}
|
