Author: [log in to unmask]
Date: Mon May 4 13:56:13 2015
New Revision: 2894
Log:
Updates to EVIO file crawler.
Modified:
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileCrawler.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileList.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileVisitor.java
java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunLog.java
Modified: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileCrawler.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileCrawler.java (original)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileCrawler.java Mon May 4 13:56:13 2015
@@ -166,9 +166,10 @@
runs.printRunSummaries();
}
- // Insert run summary into run_log table.
+ // Insert run information into database.
if (this.update) {
- runs.update();
+ // Update run log.
+ runs.insert();
}
}
Modified: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileList.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileList.java (original)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileList.java Mon May 4 13:56:13 2015
@@ -2,22 +2,32 @@
import java.io.File;
import java.io.IOException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.logging.Logger;
import org.jlab.coda.jevio.EvioException;
import org.jlab.coda.jevio.EvioReader;
+import org.lcsim.util.log.LogUtil;
class EvioFileList extends ArrayList<File> {
+ private static final Logger LOGGER = LogUtil.create(EvioFileList.class);
+
void cache() {
+ LOGGER.info("running cache commands ...");
for (final File file : this) {
EvioFileUtilities.cache(file);
}
+ LOGGER.info("done running cache commands");
}
int computeTotalEvents() {
+ LOGGER.info("computing total events ...");
int totalEvents = 0;
for (final File file : this) {
EvioReader reader = null;
@@ -36,6 +46,7 @@
}
}
}
+ LOGGER.info("done computing total events");
return totalEvents;
}
@@ -53,4 +64,22 @@
this.clear();
this.addAll(fileList);
}
+
+ void insert(final Connection connection, final int run) throws SQLException {
+ LOGGER.info("updating file list ...");
+ PreparedStatement filesStatement = null;
+ filesStatement = connection
+ .prepareStatement("INSERT INTO run_log_files (run, directory, name) VALUES(?, ?, ?)");
+ LOGGER.info("inserting files from run " + run + " into database");
+ for (final File file : this) {
+ LOGGER.info("creating update statement for " + file.getPath());
+ filesStatement.setInt(1, run);
+ filesStatement.setString(2, file.getParentFile().getPath());
+ filesStatement.setString(3, file.getName());
+ LOGGER.info("executing statement: " + filesStatement);
+ filesStatement.executeUpdate();
+ // connection.commit();
+ }
+ LOGGER.info("run_log_files was updated!");
+ }
}
Modified: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileVisitor.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileVisitor.java (original)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/EvioFileVisitor.java Mon May 4 13:56:13 2015
@@ -58,7 +58,7 @@
this.runs.getRunSummary(run).addFile(file);
} else {
- LOGGER.fine("rejected file: " + file.getPath());
+ LOGGER.info("rejected file: " + file.getPath());
}
return FileVisitResult.CONTINUE;
}
Modified: java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunLog.java
=============================================================================
--- java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunLog.java (original)
+++ java/trunk/users/src/main/java/org/hps/users/jeremym/crawler/RunLog.java Mon May 4 13:56:13 2015
@@ -39,6 +39,65 @@
return runList;
}
+ void insert() {
+
+ LOGGER.info("inserting runs into run_log ...");
+ final ConnectionParameters cp = new ConnectionParameters("root", "derp", "hps_run_db", "localhost");
+ final Connection connection = cp.createConnection();
+ try {
+ connection.setAutoCommit(false);
+
+ insertRunLog(connection);
+
+ insertFiles(connection);
+
+ connection.commit();
+
+ } catch (final SQLException e) {
+ LOGGER.log(Level.SEVERE, "rolling back transaction", e);
+ try {
+ connection.rollback();
+ } catch (final SQLException e2) {
+ throw new RuntimeException(e);
+ }
+ } finally {
+ if (connection != null) {
+ try {
+ connection.setAutoCommit(true);
+ connection.close();
+ } catch (final SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ void insertFiles(final Connection connection) throws SQLException {
+ for (final int run : getSortedRunNumbers()) {
+ getRunSummary(run).getFiles().insert(connection, run);
+ }
+ }
+
+ void insertRunLog(final Connection connection) throws SQLException {
+ PreparedStatement runLogStatement = null;
+ runLogStatement = connection
+ .prepareStatement("INSERT INTO run_log (run, start_date, end_date, nevents, nfiles, end_ok, last_updated) VALUES(?, ?, ?, ?, ?, ?, NOW())");
+ for (final Integer run : getSortedRunNumbers()) {
+ LOGGER.info("preparing to insert run " + run + " into database ..");
+ final RunSummary runSummary = this.runs.get(run);
+ runLogStatement.setInt(1, run);
+ runLogStatement.setTimestamp(2, new java.sql.Timestamp(runSummary.getStartDate().getTime()));
+ runLogStatement.setTimestamp(3, new java.sql.Timestamp(runSummary.getEndDate().getTime()));
+ runLogStatement.setInt(4, runSummary.getTotalEvents());
+ runLogStatement.setInt(5, runSummary.getFiles().size());
+ runLogStatement.setBoolean(6, runSummary.isEndOkay());
+ runLogStatement.executeUpdate();
+ // connection.commit();
+ LOGGER.info("committed run " + run + " to run_log");
+ }
+ LOGGER.info("run_log was updated!");
+ }
+
void printRunSummaries() {
for (final int run : this.runs.keySet()) {
this.runs.get(run).printRunSummary(System.out);
@@ -50,49 +109,4 @@
this.runs.get(run).sortFiles();
}
}
-
- void update() {
- LOGGER.info("updating database from run log ...");
- final ConnectionParameters cp = new ConnectionParameters("root", "derp", "hps_run_db", "localhost");
- Connection connection = null;
- PreparedStatement runLogStatement = null;
- try {
- connection = cp.createConnection();
- connection.setAutoCommit(false);
- runLogStatement = connection
- .prepareStatement("INSERT INTO run_log (run, start_date, end_date, nevents, nfiles, end_ok, last_updated) VALUES(?, ?, ?, ?, ?, ?, NOW())");
- for (final Integer run : getSortedRunNumbers()) {
- LOGGER.info("inserting run " + run + " into database");
- final RunSummary runSummary = this.runs.get(run);
- runLogStatement.setInt(1, run);
- runLogStatement.setTimestamp(2, new java.sql.Timestamp(runSummary.getStartDate().getTime()));
- runLogStatement.setTimestamp(3, new java.sql.Timestamp(runSummary.getEndDate().getTime()));
- runLogStatement.setInt(4, runSummary.getTotalEvents());
- runLogStatement.setInt(5, runSummary.getFiles().size());
- runLogStatement.setBoolean(6, runSummary.isEndOkay());
- runLogStatement.executeUpdate();
- connection.commit();
- }
- } catch (final SQLException e) {
- LOGGER.log(Level.SEVERE, "rolling back transaction", e);
- try {
- connection.rollback();
- } catch (final SQLException e2) {
- throw new RuntimeException(e);
- }
- } finally {
- if (connection != null) {
- try {
- connection.setAutoCommit(true);
- if (!connection.isClosed()) {
- connection.close();
- }
- } catch (final SQLException e) {
- e.printStackTrace();
- }
- }
- }
- LOGGER.info("database was updated!");
- }
-
}
