Author: [log in to unmask]
Date: Tue May 26 13:54:51 2015
New Revision: 3023
Log:
Miscellaneous updates to the EVIO crawler (still unstable).
Added:
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java
Modified:
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileCrawler.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java
java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileCrawler.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileCrawler.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileCrawler.java Tue May 26 13:54:51 2015
@@ -26,6 +26,9 @@
*
* @author <a href="mailto:[log in to unmask]">Jeremy McCormick</a>
*/
+// TODO: write out Auger XML (and don't actually execute job)
+// TODO: write summary EVIO file with control/EPICS events (maybe?)
+// TODO: flag to allow overwriting existing information in run table
public final class EvioFileCrawler {
/**
@@ -342,7 +345,7 @@
// Insert run information into the database.
if (this.update) {
// Update run log.
- runs.insert();
+ new RunLogUpdater(runs).insert();
}
// Update the timestamp file which can be used to tell which files have been processed.
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java Tue May 26 13:54:51 2015
@@ -1,9 +1,6 @@
package org.hps.record.evio.crawler;
import java.io.File;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -74,30 +71,7 @@
}
return totalEvents;
}
-
- /**
- * Insert the file names into the run database.
- *
- * @param connection the database connection
- * @param run the run number
- * @throws SQLException if there is a problem executing one of the database queries
- */
- void insert(final Connection connection, final int run) throws SQLException {
- LOGGER.info("updating file list ...");
- PreparedStatement filesStatement = null;
- filesStatement = connection.prepareStatement("INSERT INTO run_log_files (run, directory, name) VALUES(?, ?, ?)");
- LOGGER.info("inserting files from run " + run + " into database");
- for (final File file : this) {
- LOGGER.info("creating update statement for " + file.getPath());
- filesStatement.setInt(1, run);
- filesStatement.setString(2, file.getParentFile().getPath());
- filesStatement.setString(3, file.getName());
- LOGGER.info("executing statement: " + filesStatement);
- filesStatement.executeUpdate();
- }
- LOGGER.info("run_log_files was updated!");
- }
-
+
/**
* Get the last file.
*
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java Tue May 26 13:54:51 2015
@@ -59,7 +59,7 @@
Date date = null;
EvioReader reader = null;
try {
- reader = open(file);
+ reader = open(file, true);
EvioEvent event;
if (gotoEvent > 0) {
reader.gotoEventNumber(gotoEvent);
@@ -118,7 +118,7 @@
if (date == null) {
EvioReader reader = null;
try {
- reader = open(file);
+ reader = open(file, true);
reader.gotoEventNumber(reader.getEventCount() - 11);
EvioEvent event = null;
while ((event = reader.parseNextEvent()) != null) {
@@ -170,7 +170,7 @@
if (date == null) {
EvioReader reader = null;
try {
- reader = open(file);
+ reader = open(file, true);
EvioEvent event = null;
while ((event = reader.parseNextEvent()) != null) {
if (EvioEventUtilities.isPhysicsEvent(event)) {
@@ -268,6 +268,6 @@
* @throws EvioException if there is an error reading the EVIO data
*/
static EvioReader open(final String path) throws IOException, EvioException {
- return open(new File(path));
+ return open(new File(path), true);
}
}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java Tue May 26 13:54:51 2015
@@ -462,8 +462,10 @@
LOGGER.info("waiting for files to be cached ...");
+ // This can happen if all the files are already cached.
if (this.cacheStatuses.isEmpty()) {
- throw new IllegalStateException("There are no files registered with the cache manager.");
+ LOGGER.warning("no files to be cached");
+ return true;
}
// This is the return value which will be changed to true if all files are cached successfully.
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java Tue May 26 13:54:51 2015
@@ -1,17 +1,13 @@
package org.hps.record.evio.crawler;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.logging.Level;
import java.util.logging.Logger;
-import org.hps.conditions.database.ConnectionParameters;
import org.lcsim.util.log.LogUtil;
/**
@@ -32,7 +28,7 @@
/**
* A map between run numbers and the run summary information.
*/
- private final Map<Integer, RunSummary> runs = new HashMap<Integer, RunSummary>();
+ private final Map<Integer, RunSummary> runs = new LinkedHashMap<Integer, RunSummary>();
/**
* Get a run summary by run number.
@@ -49,6 +45,10 @@
}
return this.runs.get(run);
}
+
+ public Collection<RunSummary> getRunSummaries() {
+ return this.runs.values();
+ }
/**
* Get a list of sorted run numbers in this run log.
@@ -61,79 +61,6 @@
final List<Integer> runList = new ArrayList<Integer>(this.runs.keySet());
Collections.sort(runList);
return runList;
- }
-
- /**
- * Insert all the information from the run log into the run database.
- */
- void insert() {
-
- LOGGER.info("inserting runs into run_log ...");
- final ConnectionParameters cp = new ConnectionParameters("root", "derp", "hps_run_db", "localhost");
- final Connection connection = cp.createConnection();
- try {
- connection.setAutoCommit(false);
-
- this.insertRunLog(connection);
-
- this.insertFiles(connection);
-
- connection.commit();
-
- } catch (final SQLException e) {
- LOGGER.log(Level.SEVERE, "rolling back transaction", e);
- try {
- connection.rollback();
- } catch (final SQLException e2) {
- throw new RuntimeException(e);
- }
- } finally {
- if (connection != null) {
- try {
- connection.setAutoCommit(true);
- connection.close();
- } catch (final SQLException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- /**
- * Insert the file lists into the run database.
- *
- * @param connection the database connection
- * @throws SQLException if there is an error executing the SQL query
- */
- private void insertFiles(final Connection connection) throws SQLException {
- for (final int run : this.getSortedRunNumbers()) {
- this.getRunSummary(run).getEvioFileList().insert(connection, run);
- }
- }
-
- /**
- * Insert the run summary information into the database.
- *
- * @param connection the database connection
- * @throws SQLException if there is an error querying the database
- */
- private void insertRunLog(final Connection connection) throws SQLException {
- PreparedStatement runLogStatement = null;
- runLogStatement = connection
- .prepareStatement("INSERT INTO run_log (run, start_date, end_date, nevents, nfiles, end_ok, last_updated) VALUES(?, ?, ?, ?, ?, ?, NOW())");
- for (final Integer run : this.getSortedRunNumbers()) {
- LOGGER.info("preparing to insert run " + run + " into database ..");
- final RunSummary runSummary = this.runs.get(run);
- runLogStatement.setInt(1, run);
- runLogStatement.setTimestamp(2, new java.sql.Timestamp(runSummary.getStartDate().getTime()));
- runLogStatement.setTimestamp(3, new java.sql.Timestamp(runSummary.getEndDate().getTime()));
- runLogStatement.setInt(4, runSummary.getTotalEvents());
- runLogStatement.setInt(5, runSummary.getEvioFileList().size());
- runLogStatement.setBoolean(6, runSummary.isEndOkay());
- runLogStatement.executeUpdate();
- LOGGER.info("committed run " + run + " to run_log");
- }
- LOGGER.info("run_log was updated!");
}
/**
Added: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java (added)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java Tue May 26 13:54:51 2015
@@ -0,0 +1,151 @@
+package org.hps.record.evio.crawler;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.hps.conditions.database.ConnectionParameters;
+import org.lcsim.util.log.LogUtil;
+
+/**
+ * Updates the run database with run log information from crawler job.
+ *
+ * @author Jeremy McCormick
+ */
+public class RunLogUpdater {
+
+ /**
+ * Setup logging.
+ */
+ private static final Logger LOGGER = LogUtil.create(RunLogUpdater.class);
+
+ RunLog runLog;
+
+ final Connection connection;
+
+ RunLogUpdater(RunLog runLog) {
+ this.runLog = runLog;
+
+ // Create database connection to use in this session.
+ final ConnectionParameters cp = new ConnectionParameters("root", "derp", "hps_run_db", "localhost");
+ connection = cp.createConnection();
+ }
+
+ void close() {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+ boolean hasRun(int run) {
+ boolean hasRun = false;
+ PreparedStatement statement = null;
+ try {
+ statement = connection.prepareStatement("SELECT run from run_log where run = ?");
+ statement.setInt(1, run);
+ ResultSet rs = statement.executeQuery();
+ if (rs.next()) hasRun = true;
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ }
+ return hasRun;
+ }
+
+ /**
+ * Insert all the information from the run log into the run database.
+ */
+ void insert() {
+
+ LOGGER.info("inserting runs into run_log ...");
+ try {
+ connection.setAutoCommit(false);
+
+ this.insertRunLog(connection);
+
+ this.insertFiles(connection);
+
+ connection.commit();
+
+ } catch (final SQLException e) {
+ LOGGER.log(Level.SEVERE, "rolling back transaction", e);
+ try {
+ connection.rollback();
+ } catch (final SQLException e2) {
+ throw new RuntimeException(e);
+ }
+ } finally {
+ try {
+ connection.setAutoCommit(true);
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * Insert the file lists into the run database.
+ *
+ * @param connection the database connection
+ * @throws SQLException if there is an error executing the SQL query
+ */
+ private void insertFiles(final Connection connection) throws SQLException {
+ for (final int run : runLog.getSortedRunNumbers()) {
+ insertFiles(connection, run, runLog.getRunSummary(run).getEvioFileList());
+ }
+ }
+
+ /**
+ * Insert the run summary information into the database.
+ *
+ * @param connection the database connection
+ * @throws SQLException if there is an error querying the database
+ */
+ private void insertRunLog(final Connection connection) throws SQLException {
+ PreparedStatement runLogStatement = null;
+ runLogStatement = connection
+ .prepareStatement("INSERT INTO run_log (run, start_date, end_date, nevents, nfiles, end_ok, last_updated) VALUES(?, ?, ?, ?, ?, ?, NOW())");
+ for (final Integer run : runLog.getSortedRunNumbers()) {
+ LOGGER.info("preparing to insert run " + run + " into database ..");
+ final RunSummary runSummary = runLog.getRunSummary(run);
+ runLogStatement.setInt(1, run);
+ runLogStatement.setTimestamp(2, new java.sql.Timestamp(runSummary.getStartDate().getTime()));
+ runLogStatement.setTimestamp(3, new java.sql.Timestamp(runSummary.getEndDate().getTime()));
+ runLogStatement.setInt(4, runSummary.getTotalEvents());
+ runLogStatement.setInt(5, runSummary.getEvioFileList().size());
+ runLogStatement.setBoolean(6, runSummary.isEndOkay());
+ runLogStatement.executeUpdate();
+ LOGGER.info("committed run " + run + " to run_log");
+ }
+ LOGGER.info("run_log was updated!");
+ }
+
+ /**
+ * Insert the file names into the run database.
+ *
+ * @param connection the database connection
+ * @param run the run number
+ * @throws SQLException if there is a problem executing one of the database queries
+ */
+ void insertFiles(final Connection connection, final int run, List<File> files) throws SQLException {
+ LOGGER.info("updating file list ...");
+ PreparedStatement filesStatement = null;
+ filesStatement = connection.prepareStatement("INSERT INTO run_log_files (run, directory, name) VALUES(?, ?, ?)");
+ LOGGER.info("inserting files from run " + run + " into database");
+ for (final File file : files) {
+ LOGGER.info("creating update statement for " + file.getPath());
+ filesStatement.setInt(1, run);
+ filesStatement.setString(2, file.getParentFile().getPath());
+ filesStatement.setString(3, file.getName());
+ LOGGER.info("executing statement: " + filesStatement);
+ filesStatement.executeUpdate();
+ }
+ LOGGER.info("run_log_files was updated!");
+ }
+}
Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java
=============================================================================
--- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java (original)
+++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java Tue May 26 13:54:51 2015
@@ -134,6 +134,13 @@
return this.processors;
}
+ /**
+ * Return <code>true</code> if valid END event can be located.
+ *
+ * @param reader the EVIO reader
+ * @return <code>true</code> if valid END event is located
+ * @throws Exception if there are IO problems using the reader
+ */
boolean isEndOkay(final EvioReader reader) throws Exception {
LOGGER.info("checking is END okay ...");
boolean endOkay = false;
@@ -196,7 +203,7 @@
try {
// Open with wrapper method which will use the cached file path if necessary.
LOGGER.fine("opening " + file.getPath() + " for reading ...");
- reader = EvioFileUtilities.open(file);
+ reader = EvioFileUtilities.open(file, true);
LOGGER.fine("done opening " + file.getPath());
// If this is the first file then get the start date.
