Author: [log in to unmask] Date: Thu Jul 16 18:47:15 2015 New Revision: 3263 Log: Latest iteration of the crawler and run db API. Add separate run package for getting run db info into a user job. HPSJAVA-558 Added: java/trunk/record-util/src/main/java/org/hps/record/run/ java/trunk/record-util/src/main/java/org/hps/record/run/AbstractRunDatabaseReader.java java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataReader.java java/trunk/record-util/src/main/java/org/hps/record/run/EvioFileListReader.java java/trunk/record-util/src/main/java/org/hps/record/run/RunManager.java java/trunk/record-util/src/main/java/org/hps/record/run/RunSummary.java - copied, changed from r3257, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummary.java java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryReader.java java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataReader.java Removed: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummary.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/ScalerDataUpdater.java Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventTypeLog.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/Crawler.java Thu Jul 16 18:47:15 2015 @@ -24,8 +24,8 @@ import org.lcsim.util.log.LogUtil; /** - * Search for EVIO files in a directory tree, group the files that are found by run, extract meta data from these - * files, and optionally update a run database with the information that was found. + * Search for EVIO files in a directory tree, group the files that are found by run, extract meta data from these files, + * and optionally update a run database with the information that was found. * * @author Jeremy McCormick, SLAC */ @@ -64,6 +64,7 @@ OPTIONS.addOption("w", "max-cache-wait", true, "total time to allow for file caching (seconds)"); OPTIONS.addOption("L", "log-level", true, "set the log level (INFO, FINE, etc.)"); OPTIONS.addOption("u", "update", false, "allow overriding existing data in the run db (not allowed by default)"); + OPTIONS.addOption("x", "max-depth", true, "max depth to crawl in the directory tree"); } /** @@ -85,14 +86,14 @@ private final JCacheManager cacheManager = new JCacheManager(); /** + * Configuration options from the command line. + */ + private CrawlerConfig config; + + /** * The options parser. */ private final PosixParser parser = new PosixParser(); - - /** - * Configuration options from the command line. - */ - private CrawlerConfig config; /** * Parse command line options and create a new {@link Crawler} object from the configuration. @@ -129,7 +130,7 @@ throw new IllegalArgumentException("Connection properties file " + dbPropFile.getPath() + " does not exist."); } - ConnectionParameters connectionParameters = ConnectionParameters.fromProperties(dbPropFile); + final ConnectionParameters connectionParameters = ConnectionParameters.fromProperties(dbPropFile); config.setConnection(connectionParameters); LOGGER.config("using " + dbPropPath + " for db connection properties"); } else { @@ -139,7 +140,7 @@ // Root directory for file crawling. if (cl.hasOption("d")) { - File rootDir = new File(cl.getOptionValue("d")); + final File rootDir = new File(cl.getOptionValue("d")); if (!rootDir.exists()) { throw new IllegalArgumentException("The directory does not exist."); } @@ -152,7 +153,7 @@ // Timestamp file for date filtering. if (cl.hasOption("t")) { - File timestampFile = new File(cl.getOptionValue("t")); + final File timestampFile = new File(cl.getOptionValue("t")); config.setTimestampFile(timestampFile); if (!timestampFile.exists()) { try { @@ -165,7 +166,7 @@ } else { try { // Get the date filter for files from an existing time stamp file provided by the user. - Date timestamp = new Date(Files + final Date timestamp = new Date(Files .readAttributes(config.timestampFile().toPath(), BasicFileAttributes.class) .lastModifiedTime().toMillis()); config.setTimestamp(timestamp); @@ -179,7 +180,7 @@ // List of one or more runs to accept in the job. if (cl.hasOption("a")) { - Set<Integer> acceptRuns = new HashSet<Integer>(); + final Set<Integer> acceptRuns = new HashSet<Integer>(); for (final String runString : cl.getOptionValues("a")) { final Integer acceptRun = Integer.parseInt(runString); acceptRuns.add(acceptRun); @@ -191,7 +192,7 @@ // Enable run log updating (off by default). if (cl.hasOption("r")) { config.setUpdateRunLog(true); - LOGGER.config("updating run database is enabled"); + LOGGER.config("inserting into run database is enabled"); } // Enable file cache usage for running at JLAB. @@ -202,21 +203,21 @@ // Max wait time for file caching. if (cl.hasOption("w")) { - Long waitTime = Long.parseLong(cl.getOptionValue("w")) * MILLISECONDS; + final Long waitTime = Long.parseLong(cl.getOptionValue("w")) * MILLISECONDS; config.setWaitTime(waitTime); LOGGER.config("max time for file caching set to " + config.waitTime()); } // Max files to process per run; mostly just here for debugging purposes. if (cl.hasOption("m")) { - int maxFiles = Integer.parseInt(cl.getOptionValue("m")); + final int maxFiles = Integer.parseInt(cl.getOptionValue("m")); config.setMaxFiles(maxFiles); LOGGER.config("max files set to " + maxFiles); } // Event printing interval when doing EVIO event processing. if (cl.hasOption("p")) { - int eventPrintInterval = Integer.parseInt(cl.getOptionValue("p")); + final int eventPrintInterval = Integer.parseInt(cl.getOptionValue("p")); config.setEventPrintInterval(eventPrintInterval); LOGGER.config("event print interval set to " + eventPrintInterval); } @@ -224,7 +225,7 @@ // Flag to allow replacement of existing records in the database; not allowed by default. if (cl.hasOption("u")) { config.setAllowUpdates(true); - LOGGER.config("replacement of existing run log information in database is enabled"); + LOGGER.config("deletion and replacement of existing runs in the database is enabled"); } // User supplied timestamp string that is converted to a date for file filtering. @@ -253,6 +254,15 @@ } } + // Max depth to crawl. + if (cl.hasOption("x")) { + final Integer maxDepth = Integer.parseInt(cl.getOptionValue("x")); + if (maxDepth < 1) { + throw new IllegalArgumentException("invalid -x argument for maxDepth: " + maxDepth); + } + config.setMaxDepth(maxDepth); + } + } catch (final ParseException e) { throw new RuntimeException("Error parsing options.", e); } @@ -286,7 +296,7 @@ final EvioFileVisitor visitor = new EvioFileVisitor(config.timestamp()); // Walk the file tree using the visitor. - walk(visitor); + this.walk(visitor); // Get the list of run data created by the visitor. final RunLog runs = visitor.getRunLog(); @@ -303,16 +313,22 @@ // Print the summary information after the run processing is done. runs.printRunSummaries(); - // Execute the database update. - executeRunLogUpdate(runs); + // Execute the run database update. + this.updateRunDatabase(runs); // Update the timestamp output file. - updateTimestamp(); + this.updateTimestamp(); LOGGER.info("Crawler job is done!"); } - private void executeRunLogUpdate(final RunLog runs) throws SQLException { + /** + * Update the database with information found from crawling the files. + * + * @param runs the list of runs to update + * @throws SQLException if there is a database query error + */ + private void updateRunDatabase(final RunLog runs) throws SQLException { // Insert the run information into the database. if (config.updateRunLog()) { @@ -330,28 +346,11 @@ } } - private void walk(final EvioFileVisitor visitor) { - if (config.timestamp() != null) { - // Date filter from timestamp. - visitor.addFilter(new DateFileFilter(config.timestamp())); - LOGGER.config("added date filter " + config.timestamp()); - } - - if (!config.acceptRuns().isEmpty()) { - // List of run numbers to accept. - visitor.addFilter(new RunFilter(config.acceptRuns())); - LOGGER.config("added run number filter"); - } - - try { - // Walk the file tree from the root directory. - final EnumSet<FileVisitOption> options = EnumSet.noneOf(FileVisitOption.class); - Files.walkFileTree(config.rootDir().toPath(), options, Integer.MAX_VALUE, visitor); - } catch (final IOException e) { - throw new RuntimeException("Error while walking the directory tree.", e); - } - } - + /** + * Update the timestamp file's modification date to the time when this job ended. + * <p> + * This can be then be used in subsequent crawl jobs to filter out files that have already been seen. + */ private void updateTimestamp() { // Update the timestamp file which can be used to tell which files have been processed by their creation date. if (config.timestampFile() == null) { @@ -366,4 +365,33 @@ config.timestampFile().setLastModified(System.currentTimeMillis()); LOGGER.config("set modified on timestamp file: " + new Date(config.timestampFile().lastModified())); } + + /** + * Walk the directory tree to find EVIO files for the runs that are being processed in the job. + * + * @param visitor the file visitor + */ + private void walk(final EvioFileVisitor visitor) { + if (config.timestamp() != null) { + // Date filter from timestamp. + visitor.addFilter(new DateFileFilter(config.timestamp())); + LOGGER.config("added date filter with time stamp " + config.timestamp()); + } + + if (!config.acceptRuns().isEmpty()) { + // List of run numbers to accept. + visitor.addFilter(new RunFilter(config.acceptRuns())); + LOGGER.config("added run number filter"); + } else { + LOGGER.config("no run number filter used"); + } + + try { + // Walk the file tree from the root directory. + final EnumSet<FileVisitOption> options = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(config.rootDir().toPath(), options, config.maxDepth(), visitor); + } catch (final IOException e) { + throw new RuntimeException("Error while walking the directory tree.", e); + } + } } Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/CrawlerConfig.java Thu Jul 16 18:47:15 2015 @@ -14,7 +14,7 @@ /** * Full configuration information for the {@link Crawler class}. * <p> - * The setter methods use the builder pattern so method chaining them is possible. + * Method chaining of setters is supported. * * @author Jeremy McCormick, SLAC */ @@ -23,7 +23,22 @@ /** * The format for input timestamps used for file filtering. */ - private static final SimpleDateFormat TIMESTAMP_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static final SimpleDateFormat TIMESTAMP_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + /** + * A list of run numbers to accept in the job. + */ + private Set<Integer> acceptRuns; + + /** + * <code>true</code> if database updates are allowed meaning existing records can be deleted and replaced. + */ + private boolean allowUpdates = false; + + /** + * The database connection parameters which must be provided by command line argument. + */ + private ConnectionParameters connectionParameters; /** * Default event print interval. @@ -36,19 +51,9 @@ private int eventPrintInterval = DEFAULT_EVENT_PRINT_INTERVAL; /** - * A list of run numbers to accept in the job. - */ - private Set<Integer> acceptRuns; - - /** - * <code>true</code> if database updates are allowed meaning existing records can be deleted and replaced. - */ - private boolean allowUpdates = false; - - /** - * The database connection parameters which must be provided by command line argument. - */ - private ConnectionParameters connectionParameters; + * The maximum depth to crawl. + */ + private Integer maxDepth = Integer.MAX_VALUE; /** * The maximum number of files to accept (just used for debugging purposes). @@ -91,8 +96,17 @@ private Long waitTime; /** + * Get the set of runs that will be accepted for the job. + * + * @return the list of runs that will be accepted + */ + Set<Integer> acceptRuns() { + return acceptRuns; + } + + /** * Add an {@link org.hps.record.evio.EvioEventProcessor} to the job. - * + * * @param processor * @return this object */ @@ -103,22 +117,87 @@ /** * Add an {@link org.hps.record.evio.EvioEventProcessor} to the job by its class name. - * + * * @param processor the <code>EvioEventProcessor</code> to instantiate * @return this object */ CrawlerConfig addProcessor(final String className) { try { this.processors.add(EvioEventProcessor.class.cast(Class.forName(className).newInstance())); - } catch (Exception e) { + } catch (final Exception e) { throw new RuntimeException("Error creating EvioEventProcessor with type: " + className, e); } return this; } /** + * Return <code>true</code> if updates/deletions of existing records in the database is allowed. + * + * @return <code>true</code> if updating/deleting records in the database is allowed + */ + boolean allowUpdates() { + return allowUpdates; + } + + /** + * Get the database connection parameters. + * + * @return the database connection parameters + */ + ConnectionParameters connectionParameters() { + return connectionParameters; + } + + /** + * Get the event print interval. + * + * @return the event print interval + */ + int eventPrintInterval() { + return this.eventPrintInterval; + } + + /** + * Get the max depth in the directory tree to crawl. + * + * @return the max depth + */ + Integer maxDepth() { + return maxDepth; + } + + /** + * Get the maximum number of files that the job can process. + * + * @return the maximum number of files + */ + int maxFiles() { + return maxFiles; + } + + /** + * Get the list of extra event processors that will run with the job. + * <p> + * Required (default) processors for the job are not included here. + * + * @return the list of extra event processors + */ + List<EvioEventProcessor> processors() { + return processors; + } + + /** + * Get the root directory for the file search. + * + * @return the root directory for the file search + */ + File rootDir() { + return rootDir; + } + + /** * Set the list of run numbers that should be accepted. - * + * * @param acceptRuns the list of acceptable run numbers * @return this object */ @@ -129,7 +208,7 @@ /** * Set whether database updates are allowed, i.e. replacement of existing records. - * + * * @param allowUpdates <code>true</code> to allow database record deletion/updates * @return this object */ @@ -140,7 +219,7 @@ /** * Set the database connection parameters. - * + * * @param connectionParameters the database connection parameters * @return this object */ @@ -150,10 +229,30 @@ } /** + * Set the interval for printing the EVIO event numbers during processing. + * + * @param eventPrintInterval the event print interval + * @return this object + */ + CrawlerConfig setEventPrintInterval(final int eventPrintInterval) { + this.eventPrintInterval = eventPrintInterval; + return this; + } + + /** + * Set the max depth. + * + * @param maxDepth the max depth + */ + void setMaxDepth(final Integer maxDepth) { + this.maxDepth = maxDepth; + } + + /** * Set the maximum number of files that will be processed by the job. * <p> * This should only be used for debugging purposes as it results in incorrect event counts for the run. - * + * * @param maxFiles the maximum number of files to process or -1 for unlimited * @return this object */ @@ -164,11 +263,11 @@ /** * Set the root directory for the file search. - * + * * @param rootDir the root directory for the file search * @return this object */ - CrawlerConfig setRootDir(File rootDir) { + CrawlerConfig setRootDir(final File rootDir) { this.rootDir = rootDir; return this; } @@ -177,11 +276,11 @@ * Set a date for filtering input files. * <p> * Those files created before this date will not be processed. - * + * * @param timestamp the date for filtering files * @return this object */ - CrawlerConfig setTimestamp(Date timestamp) { + CrawlerConfig setTimestamp(final Date timestamp) { this.timestamp = timestamp; return this; } @@ -191,22 +290,22 @@ * <code>TIMESTAMP_DATE_FORMAT</code>. * <p> * Those files created before this date will not be processed. - * + * * @param timestamp the date string for filtering files * @return this object */ - CrawlerConfig setTimestamp(String timestampString) throws ParseException { - TIMESTAMP_DATE_FORMAT.parse(timestampString); + CrawlerConfig setTimestamp(final String timestampString) throws ParseException { + TIMESTAMP_FORMAT.parse(timestampString); return this; } /** * Set a date for filtering files based on the modification date of a timestamp file. - * + * * @param timestampFile the timestamp file for date filtering * @return this object */ - CrawlerConfig setTimestampFile(File timestampFile) { + CrawlerConfig setTimestampFile(final File timestampFile) { this.timestampFile = timestampFile; return this; } @@ -216,11 +315,11 @@ * <p> * This will not allow replacement of existing run log records. The {@link #allowUpdates()} flag must be on for this * be allowed. - * + * * @param updateRunLog <code>true</code> if the run database should be updated * @return this object */ - CrawlerConfig setUpdateRunLog(boolean updateRunLog) { + CrawlerConfig setUpdateRunLog(final boolean updateRunLog) { this.updateRunLog = updateRunLog; return this; } @@ -229,11 +328,11 @@ * Set whether file caching using the 'jcache' program should be enabled. * <p> * This is only relevant for jobs run at JLAB. - * + * * @param useFileCache <code>true</code> to allow file caching * @return this object */ - CrawlerConfig setUseFileCache(boolean useFileCache) { + CrawlerConfig setUseFileCache(final boolean useFileCache) { this.useFileCache = useFileCache; return this; } @@ -242,87 +341,20 @@ * Set the max wait time in seconds for all file caching operations to complete. * <p> * If this time is exceeded then the job will fail with an error. - * + * * @param waitTime the max wait time in seconds allowed for file caching to complete * @return this object */ - CrawlerConfig setWaitTime(long waitTime) { + CrawlerConfig setWaitTime(final long waitTime) { this.waitTime = waitTime; return this; } /** - * Set the interval for printing the EVIO event numbers during processing. - * - * @param eventPrintInterval the event print interval - * @return this object - */ - CrawlerConfig setEventPrintInterval(int eventPrintInterval) { - this.eventPrintInterval = eventPrintInterval; - return this; - } - - /** - * Get the set of runs that will be accepted for the job. - * - * @return the list of runs that will be accepted - */ - Set<Integer> acceptRuns() { - return acceptRuns; - } - - /** - * Return <code>true</code> if updates/deletions of existing records in the database is allowed. - * - * @return <code>true</code> if updating/deleting records in the database is allowed - */ - boolean allowUpdates() { - return allowUpdates; - } - - /** - * Get the database connection parameters. - * - * @return the database connection parameters - */ - ConnectionParameters connectionParameters() { - return connectionParameters; - } - - /** - * Get the maximum number of files that the job can process. - * - * @return the maximum number of files - */ - int maxFiles() { - return maxFiles; - } - - /** - * Get the list of extra event processors that will run with the job. - * <p> - * Required (default) processors for the job are not included here. - * - * @return the list of extra event processors - */ - List<EvioEventProcessor> processors() { - return processors; - } - - /** - * Get the root directory for the file search. - * - * @return the root directory for the file search - */ - File rootDir() { - return rootDir; - } - - /** * Get the timestamp for file filtering. * <p> * Files older than this will not be included in the job. - * + * * @return the timestamp for file filtering */ Date timestamp() { @@ -331,7 +363,7 @@ /** * Get the timestamp file using for filtering EVIO files. - * + * * @return the timestamp file used for filtering EVIO files (can be null) */ File timestampFile() { @@ -340,7 +372,7 @@ /** * Return <code>true</code> if the run database should be updated. - * + * * @return <code>true</code> if the run database should be updated */ boolean updateRunLog() { @@ -349,7 +381,7 @@ /** * Return <code>true</code> if file caching should be enabled. - * + * * @return <code>true</code> if file caching should be enabled */ boolean useFileCache() { @@ -358,19 +390,10 @@ /** * Get the max wait time in seconds to allow for file caching operations to complete. - * + * * @return the max wait time in seconds to allow for file caching operations to complete */ Long waitTime() { return waitTime; } - - /** - * Get the event print interval. - * - * @return the event print interval - */ - int eventPrintInterval() { - return this.eventPrintInterval; - } } Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EpicsLog.java Thu Jul 16 18:47:15 2015 @@ -16,18 +16,18 @@ final class EpicsLog extends EvioEventProcessor { /** - * A count of how many times a given EPICS variable is found in the input, e.g. for computing the mean value across - * the entire run. + * A count of how many times a given EPICS variable is found in the input for computing the mean value across the + * entire run. */ private final Map<String, Integer> counts = new HashMap<String, Integer>(); /** * The current EPICS data block from the EVIO events (last one that was found). */ - private EpicsData currentData; + private EpicsData currentEpicsData; /** - * The summary information for the variables from computing the mean across the whole run. + * The summary information for the variables computed from the mean values across the whole run. */ private final EpicsData logData = new EpicsData(); @@ -37,17 +37,9 @@ private final EpicsEvioProcessor processor = new EpicsEvioProcessor(); /** - * Reference to the run summary which will contain the EPICs information. + * Create an EPICs log. */ - private final RunSummary runSummary; - - /** - * Create an EPICs log pointing to a run summary. - * - * @param runSummary the run summary - */ - EpicsLog(final RunSummary runSummary) { - this.runSummary = runSummary; + EpicsLog() { } /** @@ -63,9 +55,15 @@ final double mean = total / this.counts.get(name); this.logData.setValue(name, mean); } + } - // Set the EPICS data on the run summary. - this.runSummary.setEpicsData(this.logData); + /** + * Get the {@link org.hps.record.epics.EpicsData} which contains mean values for the run. + * + * @return the {@link org.hps.record.epics.EpicsData} for the run + */ + EpicsData getEpicsData() { + return this.logData; } /** @@ -74,7 +72,7 @@ @Override public void process(final EvioEvent evioEvent) { this.processor.process(evioEvent); - this.currentData = this.processor.getEpicsData(); + this.currentEpicsData = this.processor.getEpicsData(); this.update(); } @@ -84,8 +82,8 @@ * If the current data is <code>null</code>, this method does nothing. */ private void update() { - if (this.currentData != null) { - for (final String name : this.currentData.getUsedNames()) { + if (this.currentEpicsData != null) { + for (final String name : this.currentEpicsData.getUsedNames()) { if (!this.logData.getUsedNames().contains(name)) { this.logData.setValue(name, 0.); } @@ -95,7 +93,7 @@ int count = this.counts.get(name); count += 1; this.counts.put(name, count); - final double value = this.logData.getValue(name) + this.currentData.getValue(name); + final double value = this.logData.getValue(name) + this.currentEpicsData.getValue(name); this.logData.setValue(name, value); // System.out.println(name + " => added " + this.currentData.getValue(name) + "; total = " + value + // "; mean = " + value / count); Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventTypeLog.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventTypeLog.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EventTypeLog.java Thu Jul 16 18:47:15 2015 @@ -22,11 +22,6 @@ private final Map<Object, Integer> eventTypeCounts = new HashMap<Object, Integer>(); /** - * The run summary to update. - */ - private final RunSummary runSummary; - - /** * The total number of physics events processed. */ private int physicsEventCount = 0; @@ -36,22 +31,13 @@ * * @param runSummary the run summary */ - EventTypeLog(final RunSummary runSummary) { - this.runSummary = runSummary; + EventTypeLog() { for (final EventTagConstant constant : EventTagConstant.values()) { this.eventTypeCounts.put(constant, 0); } for (final EventTagBitMask mask : EventTagBitMask.values()) { this.eventTypeCounts.put(mask, 0); } - } - - /** - * End of job hook which sets the event type counts on the run summary. - */ - @Override - public void endJob() { - this.runSummary.setEventTypeCounts(this.eventTypeCounts); } /** @@ -65,7 +51,7 @@ /** * Get the number of physics events counted. - * + * * @return the number of physics events counted */ int getPhysicsEventCount() { @@ -79,7 +65,7 @@ */ @Override public void process(final EvioEvent event) { - + // Increment counts for exact event tag values. for (final EventTagConstant constant : EventTagConstant.values()) { if (constant.isEventTag(event)) { @@ -87,7 +73,7 @@ this.eventTypeCounts.put(constant, count); } } - + // Increment counts for bit masking of tags. for (final EventTagBitMask mask : EventTagBitMask.values()) { if (mask.isEventTag(event)) { Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileList.java Thu Jul 16 18:47:15 2015 @@ -4,23 +4,21 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.logging.Logger; - -import org.lcsim.util.log.LogUtil; /** * This is a list of <code>File</code> objects that are assumed to be EVIO files. * * @author Jeremy McCormick, SLAC */ -final class EvioFileList extends ArrayList<File> { +@SuppressWarnings("serial") +public final class EvioFileList extends ArrayList<File> { /** * Get the first file. * * @return the first file */ - File first() { + public File first() { return this.get(0); } @@ -29,14 +27,14 @@ * * @return the last file */ - File last() { + public File last() { return this.get(this.size() - 1); } /** * Sort the files in-place by their sequence number. */ - void sort() { + public void sort() { final List<File> fileList = new ArrayList<File>(this); Collections.sort(fileList, new EvioFileSequenceComparator()); this.clear(); Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/EvioFileUtilities.java Thu Jul 16 18:47:15 2015 @@ -115,23 +115,23 @@ * @return the run end date */ static Date getRunEnd(final File file) { - + // Search for the END event in the last 10 events of the file. Date endDate = getControlDate(file, EvioEventConstants.END_EVENT_TAG, -10); - + // Was the end date found from the END event? if (endDate == null) { - + EvioReader reader = null; try { reader = open(file, true); - + // Search for the last physics event in the last 10 events of the file. reader.gotoEventNumber(reader.getEventCount() - 10); EvioEvent event = null; while ((event = reader.parseNextEvent()) != null) { if (EvioEventUtilities.isPhysicsEvent(event)) { - Date eventDate = getHeadBankDate(event); + final Date eventDate = getHeadBankDate(event); if (eventDate != null) { // This might be set multiple times but should result in the time of the last physics event. endDate = eventDate; @@ -176,13 +176,13 @@ * @return the run start date */ static Date getRunStart(final File file) { - + // First try to find the start date in the special PRESTART event. Date date = getControlDate(file, EvioEventConstants.PRESTART_EVENT_TAG, 0); - + // Was start date not found from PRESTART? if (date == null) { - + // Read events until there is a physics event and use its time for the start date. EvioReader reader = null; try { @@ -271,7 +271,7 @@ final long start = System.currentTimeMillis(); final EvioReader reader = new EvioReader(openFile, false, sequential); final long end = System.currentTimeMillis() - start; - LOGGER.info("opened " + openFile.getPath() + " in " + end / MILLISECONDS + " seconds"); + LOGGER.info("opened " + openFile.getPath() + " in " + (double) end / (double) MILLISECONDS + " seconds"); return reader; } Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/JCacheManager.java Thu Jul 16 18:47:15 2015 @@ -22,6 +22,11 @@ /** * Utility class for caching files from the MSS to cache disk at JLAB. + * <p> + * This class should <b>not</b> be activated when running the crawler on the Auger batch system as it will take up a lot + * of job time caching files, which is part of Auger's job staging that doesn't count towards the job time. + * <p> + * It is fine to use running on an interactive <i>ifarm</i> node at JLAB. * * @author Jeremy McCormick, SLAC */ @@ -66,6 +71,19 @@ CacheStatus(final File file, final Integer requestId) { this.file = file; this.requestId = requestId; + } + + /** + * Get the error message from the XML request. + * + * @return the error message from the XML request + */ + String getErrorMessage() { + if (this.xml.getChild("request").getChild("file").getChild("error") != null) { + return this.xml.getChild("request").getChild("file").getChild("error").getText(); + } else { + return ""; + } } /** @@ -135,6 +153,13 @@ } /** + * Return <code>true</code> if status is "failed". + */ + boolean isFailed() { + return "failed".equals(this.status); + } + + /** * Return <code>true</code> if status is "hit". * * @return <code>true</code> if status is "hit" @@ -153,28 +178,8 @@ } /** - * Return <code>true</code> if status is "failed". - */ - boolean isFailed() { - return "failed".equals(this.status); - } - - /** - * Get the error message from the XML request. - * - * @return the error message from the XML request - */ - String getErrorMessage() { - if (this.xml.getChild("request").getChild("file").getChild("error") != null) { - return this.xml.getChild("request").getChild("file").getChild("error").getText(); - } else { - return ""; - } - } - - /** * Run the <i>jcache request</i> command for this request ID and return the XML output. - * + * * @return the XML output from the <i>jcache request</i> command */ private Element request() { @@ -201,7 +206,7 @@ */ void update() { // Request status update and get the XML from that process. - this.xml = request(); + this.xml = this.request(); // Update the status from the XML. this.status = this.xml.getChild("request").getChild("file").getChildText("status"); @@ -340,60 +345,57 @@ } /** - * Return <code>true</code> if all files registered with the manager are cached. + * Return <code>true</code> if all files registered with the manager have been cached. * * @return <code>true</code> if all files registered with the manager are cached */ boolean checkCacheStatus() { - // Flag which will be changed to false if we find non-cached files in the loop. + // Flag which will be changed to false if we find files that are not cached yet. boolean allCached = true; - // Loop over all cache statuses and refresh/check them. + // Loop over all files, refresh the status, and check that they are cached. for (final Entry<File, CacheStatus> entry : this.cacheStatuses.entrySet()) { - // Get the cache status for a single file. + // Get the cache status the file. final CacheStatus cacheStatus = entry.getValue(); LOGGER.info("checking status of " + cacheStatus.getFile().getPath() + " with req ID '" + cacheStatus.getRequestId() + "' ..."); - // Is this file flagged as not non-cached? + // Does the file's status indicate it is not cached yet? if (!cacheStatus.isCached()) { + // Update the cache status to see if it has changed. LOGGER.info("updating status of " + cacheStatus.getFile().getPath() + " ..."); - - // Update the cache status to see if it changed since last check. cacheStatus.update(); - // Is status still non-cached after status update? + // Is this file's status still non-cached after the status update? if (!cacheStatus.isCached()) { // Set flag which indicates at least one file is not cached yet. allCached = false; LOGGER.info(entry.getKey() + " is NOT cached with status " + cacheStatus.getStatus(false)); - } else { - // Log that this file is now cached. It will not be checked next time. - LOGGER.info(cacheStatus.getFile().getPath() + " is cached with status " - + cacheStatus.getStatus(false)); - } - - // Did the request fail? - if (cacheStatus.isFailed()) { + + break; + + } else if (cacheStatus.isFailed()) { // Cache failure is a fatal error. LOGGER.severe("cache request failed with error: " + cacheStatus.getErrorMessage()); throw new RuntimeException("Cache request failed."); + } else { + // Log that this file is now cached. It will not be checked next time this method is called. + LOGGER.info(cacheStatus.getFile().getPath() + " is now cached with status " + + cacheStatus.getStatus(false)); } - } else { - LOGGER.info(cacheStatus.getFile().getPath() + " is already cached"); } } return allCached; } /** - * Clear all cache statuses, which means files are no longer tracked by this manager. + * Clear the cache statuses which means the manager will no longer track any of the files that were registered. */ void clear() { this.cacheStatuses.clear(); @@ -402,7 +404,7 @@ } /** - * Get the request ID from a process that ran the 'jcache request' command. + * Parse out the request ID from the output of a 'jcache request' process. * * @param process the system process * @return the request ID @@ -524,7 +526,7 @@ cached = true; break; } else { - LOGGER.info(this.getUncachedCount() + " files still uncached"); + LOGGER.info(this.getUncachedCount() + " files are not cached"); } // Sleep for awhile before checking the cache statuses again. @@ -537,7 +539,7 @@ } } - double end = (double) (System.currentTimeMillis() - this.start); + final double end = System.currentTimeMillis() - this.start; LOGGER.info("caching took " + new DecimalFormat("#.##").format(end / 1000. / 60.) + " minutes"); Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLog.java Thu Jul 16 18:47:15 2015 @@ -8,6 +8,7 @@ import java.util.Map; import java.util.logging.Logger; +import org.hps.record.run.RunSummary; import org.lcsim.util.log.LogUtil; /** @@ -70,7 +71,7 @@ */ void printRunSummaries() { for (final int run : this.runs.keySet()) { - this.runs.get(run).printRunSummary(System.out); + this.runs.get(run).printOut(System.out); } } Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunLogUpdater.java Thu Jul 16 18:47:15 2015 @@ -5,6 +5,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import org.hps.record.run.RunSummary; import org.lcsim.util.log.LogUtil; /** Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunProcessor.java Thu Jul 16 18:47:15 2015 @@ -10,6 +10,7 @@ import org.hps.record.evio.EvioEventConstants; import org.hps.record.evio.EvioEventProcessor; +import org.hps.record.run.RunSummary; import org.hps.record.scalers.ScalersEvioProcessor; import org.jlab.coda.jevio.EvioEvent; import org.jlab.coda.jevio.EvioException; @@ -35,19 +36,89 @@ private static final Logger LOGGER = LogUtil.create(RunProcessor.class, new DefaultLogFormatter(), Level.FINE); /** + * Process all the runs that were found. + * + * @param runs the run log containing the list of run summaries + * @throws Exception if there is an error processing one of the runs + */ + static void processRuns(final JCacheManager cacheManager, final RunLog runs, final CrawlerConfig config) + throws Exception { + + // Configure max wait time of jcache manager. + if (config.waitTime() != null && config.waitTime() > 0L) { + cacheManager.setWaitTime(config.waitTime()); + LOGGER.config("JCacheManager max wait time set to " + config.waitTime()); + } + + // Process all of the runs that were found. + for (final int run : runs.getSortedRunNumbers()) { + + // Get the run summary. + final RunSummary runSummary = runs.getRunSummary(run); + + // Clear the cache manager. + if (config.useFileCache()) { + cacheManager.clear(); + } + + // Create a processor to process all the EVIO events in the run. + final RunProcessor runProcessor = new RunProcessor(cacheManager, runSummary, config); + + // Add extra processors. + for (final EvioEventProcessor processor : config.processors()) { + runProcessor.addProcessor(processor); + LOGGER.config("added extra EVIO processor " + processor.getClass().getName()); + } + + // Process all of the run's files. + runProcessor.process(); + } + } + + /** + * The cache manager. + */ + private final JCacheManager cacheManager; + + /** * Processor for extracting EPICS information. */ - private EpicsLog epicsLog; + private final EpicsLog epicsLog; + + /** + * The event printing interval when processing EVIO files. + */ + private int eventPrintInterval = 1000; + + /** + * Processor for extracting event type counts (sync, physics, trigger types, etc.). + */ + private final EventTypeLog eventTypeLog; + + /** + * Max files to read (defaults to unlimited). + */ + private int maxFiles = -1; + + /** + * The list of EVIO processors to run on the files that are found. + */ + private final List<EvioEventProcessor> processors = new ArrayList<EvioEventProcessor>(); + + /** + * The run summary information updated by running this processor. + */ + private final RunSummary runSummary; /** * Processor for extracting scaler data. */ - private ScalersEvioProcessor scalersProcessor; - - /** - * Processor for extracting event type counts (sync, physics, trigger types, etc.). - */ - private EventTypeLog eventTypeLog; + private final ScalersEvioProcessor scalersProcessor; + + /** + * Set to <code>true</code> to use file caching. + */ + private boolean useFileCache; /** * Create the processor for a single run. @@ -55,65 +126,35 @@ * @param runSummary the run summary for the run * @return the run processor */ - RunProcessor(final JCacheManager cacheManager, final RunSummary runSummary, CrawlerConfig config) { + RunProcessor(final JCacheManager cacheManager, final RunSummary runSummary, final CrawlerConfig config) { this.runSummary = runSummary; this.cacheManager = cacheManager; // EPICS processor. - epicsLog = new EpicsLog(runSummary); - addProcessor(epicsLog); + epicsLog = new EpicsLog(); + this.addProcessor(epicsLog); // Scaler data processor. scalersProcessor = new ScalersEvioProcessor(); scalersProcessor.setResetEveryEvent(false); - addProcessor(scalersProcessor); + this.addProcessor(scalersProcessor); // Event log processor. - eventTypeLog = new EventTypeLog(runSummary); - addProcessor(eventTypeLog); + eventTypeLog = new EventTypeLog(); + this.addProcessor(eventTypeLog); // Max files. if (config.maxFiles() != -1) { - setMaxFiles(config.maxFiles()); + this.setMaxFiles(config.maxFiles()); } // Enable file caching. - useFileCache(config.useFileCache()); + this.useFileCache(config.useFileCache()); // Set event printing interval. - setEventPrintInterval(config.eventPrintInterval()); - } - - /** - * The cache manager. - */ - private final JCacheManager cacheManager; - - /** - * The event printing interval when processing EVIO files. - */ - private int eventPrintInterval = 1000; - - /** - * Max files to read (defaults to unlimited). - */ - private int maxFiles = -1; - - /** - * The list of EVIO processors to run on the files that are found. - */ - private final List<EvioEventProcessor> processors = new ArrayList<EvioEventProcessor>(); - - /** - * The run summary information updated by running this processor. - */ - private final RunSummary runSummary; - - /** - * Set to <code>true</code> to use file caching. - */ - private boolean useFileCache; + this.setEventPrintInterval(config.eventPrintInterval()); + } /** * Add a processor of EVIO events. @@ -235,7 +276,7 @@ processor.startJob(); } - List<File> files = this.getFiles(); + final List<File> files = this.getFiles(); LOGGER.info("processing " + files.size() + " from run " + this.runSummary.getRun()); @@ -248,6 +289,18 @@ for (final EvioEventProcessor processor : this.processors) { processor.endJob(); } + + // Put scaler data from EVIO processor into run summary. + runSummary.setScalerData(this.scalersProcessor.getScalerData()); + + // Set the counts of event types on the run summary. + runSummary.setEventTypeCounts(eventTypeLog.getEventTypeCounts()); + + // Set total number of physics events on the run summary from the event counter. + runSummary.setTotalEvents(this.eventTypeLog.getPhysicsEventCount()); + + // Set EpicsData for the run. + runSummary.setEpicsData(this.epicsLog.getEpicsData()); LOGGER.info("done processing run " + this.runSummary.getRun()); } @@ -297,7 +350,7 @@ LOGGER.info("done running EVIO processors"); // Check if END event is present if this is the last file in the run. - if (file.equals(getFiles().get(getFiles().size() - 1))) { + if (file.equals(this.getFiles().get(this.getFiles().size() - 1))) { final boolean endOkay = this.isEndOkay(reader); this.runSummary.setEndOkay(endOkay); LOGGER.info("endOkay set to " + endOkay); @@ -308,12 +361,6 @@ LOGGER.info("found end date " + endDate); } - // Pull scaler data from EVIO processor into run summary. - runSummary.setScalerData(this.scalersProcessor.getScalerData()); - - // Set total number of physics events. - runSummary.setTotalEvents(this.eventTypeLog.getPhysicsEventCount()); - } finally { if (reader != null) { reader.close(); @@ -355,43 +402,4 @@ this.useFileCache = cacheFiles; LOGGER.config("file caching enabled"); } - - /** - * Process all the runs that were found. - * - * @param runs the run log containing the list of run summaries - * @throws Exception if there is an error processing one of the runs - */ - static void processRuns(JCacheManager cacheManager, final RunLog runs, CrawlerConfig config) throws Exception { - - // Configure max wait time of jcache manager. - if (config.waitTime() != null && config.waitTime() > 0L) { - cacheManager.setWaitTime(config.waitTime()); - LOGGER.config("JCacheManager max wait time set to " + config.waitTime()); - } - - // Process all of the runs that were found. - for (final int run : runs.getSortedRunNumbers()) { - - // Get the run summary. - RunSummary runSummary = runs.getRunSummary(run); - - // Clear the cache manager. - if (config.useFileCache()) { - cacheManager.clear(); - } - - // Create a processor to process all the EVIO events in the run. - final RunProcessor runProcessor = new RunProcessor(cacheManager, runSummary, config); - - // Add extra processors. - for (final EvioEventProcessor processor : config.processors()) { - runProcessor.addProcessor(processor); - LOGGER.config("added extra EVIO processor " + processor.getClass().getName()); - } - - // Process all of the run's files. - runProcessor.process(); - } - } } Modified: java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummaryUpdater.java Thu Jul 16 18:47:15 2015 @@ -9,6 +9,8 @@ import java.util.logging.Logger; import org.hps.record.epics.EpicsData; +import org.hps.record.run.RunSummary; +import org.hps.record.scalers.ScalerData; import org.lcsim.util.log.LogUtil; /** @@ -65,6 +67,11 @@ this.run = this.runSummary.getRun(); } + /** + * Delete all information for this run from all tables in the database. + * + * @throws SQLException if there is a SQL query error + */ private void delete() throws SQLException { LOGGER.info("deleting existing information for run " + runSummary.getRun()); @@ -91,17 +98,6 @@ */ private void deleteEpics() throws SQLException { final PreparedStatement statement = connection.prepareStatement("DELETE FROM run_epics WHERE run = ?"); - statement.setInt(1, this.run); - statement.executeUpdate(); - } - - /** - * Delete existing EPICS data from the run_log_epics table. - * - * @throws SQLException if there is an error performing the db query - */ - private void deleteScalerData() throws SQLException { - final PreparedStatement statement = connection.prepareStatement("DELETE FROM run_scalers WHERE run = ?"); statement.setInt(1, this.run); statement.executeUpdate(); } @@ -122,8 +118,6 @@ /** * Delete the row for this run from the <i>runs</i> table. - * <p> - * This doesn't delete the rows from <i>run_epics</i> or <i>run_files</i>. * * @throws SQLException if there is an error executing the SQL query */ @@ -135,6 +129,22 @@ LOGGER.info("deleted rows from runs for " + run); } + /** + * Delete existing EPICS data from the run_log_epics table. + * + * @throws SQLException if there is an error performing the db query + */ + private void deleteScalerData() throws SQLException { + final PreparedStatement statement = connection.prepareStatement("DELETE FROM run_scalers WHERE run = ?"); + statement.setInt(1, this.run); + statement.executeUpdate(); + } + + /** + * Insert the current {@link RunSummary} into the run database. + * + * @throws SQLException if there is a SQL query error + */ void insert() throws SQLException { LOGGER.info("performing db insert for " + runSummary); @@ -167,9 +177,7 @@ this.insertEpics(); // Insert scaler data. - if (runSummary.getScalerData() != null) { - new ScalerDataUpdater(connection, runSummary.getScalerData(), run).insert(); - } + this.insertScalarData(); // Commit the transactions for this run. LOGGER.info("committing transaction for run " + run); @@ -238,9 +246,33 @@ statement.setTimestamp(3, new java.sql.Timestamp(runSummary.getEndDate().getTime())); statement.setInt(4, runSummary.getTotalEvents()); statement.setInt(5, runSummary.getEvioFileList().size()); - statement.setBoolean(6, runSummary.isEndOkay()); + statement.setBoolean(6, runSummary.getEndOkay()); statement.executeUpdate(); LOGGER.info("inserted run " + run + " to runs table"); + } + + /** + * Insert scaler data into the database. + * + * @throws SQLException if there is a SQL query error + */ + private void insertScalarData() throws SQLException { + final PreparedStatement statement; + final ScalerData scalerData = this.runSummary.getScalerData(); + if (scalerData == null) { + throw new RuntimeException("scaler data is missing"); + } + try { + statement = connection.prepareStatement("INSERT INTO run_scalers (run, idx, value) VALUES (?, ?, ?)"); + for (int idx = 0; idx < scalerData.size(); idx++) { + statement.setInt(1, run); + statement.setInt(2, idx); + statement.setInt(3, scalerData.getValue(idx)); + statement.executeUpdate(); + } + } finally { + connection.commit(); + } } /** @@ -257,9 +289,9 @@ } /** - * Set whether deletion and replacement of existing run information is allowed. - * - * @param allowDeleteExisting <code>true</code> to allow deletion and replacement of existing information + * Set whether replacement of existing rows in the database is allowed. + * + * @param allowDeleteExisting <code>true</code> to allow replacement of existing rows */ void setAllowDeleteExisting(final boolean allowDeleteExisting) { this.allowDeleteExisting = allowDeleteExisting; Added: java/trunk/record-util/src/main/java/org/hps/record/run/AbstractRunDatabaseReader.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/run/AbstractRunDatabaseReader.java (added) +++ java/trunk/record-util/src/main/java/org/hps/record/run/AbstractRunDatabaseReader.java Thu Jul 16 18:47:15 2015 @@ -0,0 +1,88 @@ +package org.hps.record.run; + +import java.sql.Connection; + +/** + * Abstract class for performing conversion of records in the run database into Java objects. + * <p> + * Sub-classes must implement the {@link #read()} method. + * + * @author Jeremy McCormick, SLAC + * @param <T> + */ +public abstract class AbstractRunDatabaseReader<T> { + + /** + * The database connection. + */ + private Connection connection; + + /** + * The object created from the table rows. + */ + private T data; + + /** + * The run number. + */ + private int run = -1; + + /** + * Get the database connection. + * + * @return the database connection + */ + final Connection getConnection() { + return this.connection; + } + + /** + * Get the data created from the {@link #read()} method being called. + * + * @return the data created from database records + */ + final T getData() { + return data; + } + + /** + * Get the run number. + * + * @return the run number + */ + final int getRun() { + return this.run; + } + + /** + * Read data from the database into a Java object accessible from the {@link #getData()} method. + */ + abstract void read(); + + /** + * Set the database connection. + * + * @param connection the database connection + */ + final void setConnection(final Connection connection) { + this.connection = connection; + } + + /** + * Set the object converted from database records. + * + * @param data the object converted from database records + */ + final void setData(final T data) { + this.data = data; + } + + /** + * Set the run number. + * + * @param run the run number + */ + final void setRun(final int run) { + this.run = run; + } +} Added: java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataReader.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataReader.java (added) +++ java/trunk/record-util/src/main/java/org/hps/record/run/EpicsDataReader.java Thu Jul 16 18:47:15 2015 @@ -0,0 +1,59 @@ +package org.hps.record.run; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.hps.record.epics.EpicsData; + +/** + * Convert run database records from the <i>run_epics</i> table in to a {@link EpicsData} object. + * + * @author Jeremy McCormick, SLAC + */ +final class EpicsDataReader extends AbstractRunDatabaseReader<EpicsData> { + + /** + * The SQL SELECT query string. + */ + private final String SELECT_SQL = "SELECT variable_name, value FROM run_epics WHERE run = ?"; + + /** + * Read data from the database and convert to a {@link EpicsData} object. + */ + @Override + void read() { + if (this.getRun() == -1) { + throw new IllegalStateException("run number is invalid: " + this.getRun()); + } + if (this.getConnection() == null) { + throw new IllegalStateException("Connection is not set."); + } + + PreparedStatement statement = null; + try { + statement = this.getConnection().prepareStatement(SELECT_SQL); + statement.setInt(1, this.getRun()); + final ResultSet resultSet = statement.executeQuery(); + + final EpicsData epicsData = new EpicsData(); + + while (resultSet.next()) { + epicsData.setValue(resultSet.getString("variable_name"), resultSet.getDouble("value")); + } + + this.setData(epicsData); + + } catch (final SQLException e) { + throw new RuntimeException(e); + } finally { + if (statement != null) { + try { + statement.close(); + } catch (final SQLException e) { + e.printStackTrace(); + } + } + } + } +} Added: java/trunk/record-util/src/main/java/org/hps/record/run/EvioFileListReader.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/run/EvioFileListReader.java (added) +++ java/trunk/record-util/src/main/java/org/hps/record/run/EvioFileListReader.java Thu Jul 16 18:47:15 2015 @@ -0,0 +1,62 @@ +package org.hps.record.run; + +import java.io.File; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.hps.record.evio.crawler.EvioFileList; + +/** + * Convert run database records from the <i>run_files</i> table into an {@link EvioFileList} object. + * + * @author Jeremy McCormick, SLAC + */ +final class EvioFileListReader extends AbstractRunDatabaseReader<EvioFileList> { + + /** + * The SQL SELECT query string. + */ + private final String SELECT_SQL = "SELECT directory, name FROM run_files WHERE run = ?"; + + /** + * Read data from the database and convert to an {@link EvioFileList} object. + */ + @Override + void read() { + if (this.getRun() == -1) { + throw new IllegalStateException("run number is invalid: " + this.getRun()); + } + if (this.getConnection() == null) { + throw new IllegalStateException("Connection is not set."); + } + + PreparedStatement statement = null; + try { + statement = this.getConnection().prepareStatement(SELECT_SQL); + statement.setInt(1, this.getRun()); + final ResultSet resultSet = statement.executeQuery(); + + final EvioFileList evioFileList = new EvioFileList(); + + while (resultSet.next()) { + evioFileList.add(new File(resultSet.getString("directory") + File.separator + + resultSet.getString("name"))); + } + + this.setData(evioFileList); + + } catch (final SQLException e) { + throw new RuntimeException(e); + } finally { + if (statement != null) { + try { + statement.close(); + } catch (final SQLException e) { + e.printStackTrace(); + } + } + } + } + +} Added: java/trunk/record-util/src/main/java/org/hps/record/run/RunManager.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/run/RunManager.java (added) +++ java/trunk/record-util/src/main/java/org/hps/record/run/RunManager.java Thu Jul 16 18:47:15 2015 @@ -0,0 +1,172 @@ +package org.hps.record.run; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.lcsim.conditions.ConditionsEvent; +import org.lcsim.conditions.ConditionsListener; +import org.lcsim.util.log.DefaultLogFormatter; +import org.lcsim.util.log.LogUtil; + +/** + * Manages access to the run database and creates a {@link RunSummary} object from the data for a specific run. + * <p> + * This class can also convert database records into {@link org.hps.record.epics.EpicsData}, + * {@link org.hps.record.scalers.ScalerData}, and {@link org.hps.record.evio.crawler.EvioFileList} using their + * {@link AbstractRunDatabaseReader} implementation classes. + * + * @author Jeremy McCormick, SLAC + */ +public final class RunManager implements ConditionsListener { + + /** + * The singleton instance of the RunManager. + */ + private static RunManager INSTANCE; + + /** + * The class's logger. + */ + private static Logger LOGGER = LogUtil.create(RunManager.class, new DefaultLogFormatter(), Level.ALL); + + /** + * Get the instance of the {@link RunManager}. + * + * @return the instance of the {@link RunManager}. + */ + public static RunManager getRunManager() { + if (INSTANCE == null) { + INSTANCE = new RunManager(); + } + return INSTANCE; + } + + /** + * The database connection. + */ + private Connection connection; + + /** + * The run number; the -1 value indicates that this has not been set externally yet. + */ + private int run = -1; + + /** + * The {@link RunSummary} for the current run. + */ + private RunSummary runSummary = null; + + @Override + public void conditionsChanged(final ConditionsEvent conditionsEvent) { + final int newRun = conditionsEvent.getConditionsManager().getRun(); + LOGGER.info("initializing for run " + newRun + " ..."); + this.setRun(newRun); + LOGGER.info("done initializing for run " + this.getRun()); + } + + /** + * Get the database connection. + * + * @return the database connection + */ + Connection getConnection() { + return this.connection; + } + + /** + * Get the run number. + * + * @return the run number + */ + public int getRun() { + return run; + } + + /** + * Get the current {@link RunSummary}. + * + * @return the current {@link RunSummary} or <code>null</code> if it is not set + */ + public RunSummary getRunSummary() { + return this.runSummary; + } + + /** + * Read information from the run database and create a {@link RunSummary} from it. + */ + private void readRun() { + // Load main RunSummary object. + final RunSummaryReader runSummaryReader = new RunSummaryReader(); + runSummaryReader.setRun(this.getRun()); + runSummaryReader.setConnection(this.getConnection()); + runSummaryReader.read(); + this.setRunSummary(runSummaryReader.getData()); + + // Set EpicsData on RunSummary. + final EpicsDataReader epicsDataReader = new EpicsDataReader(); + epicsDataReader.setRun(this.getRun()); + epicsDataReader.setConnection(this.getConnection()); + epicsDataReader.read(); + this.getRunSummary().setEpicsData(epicsDataReader.getData()); + + // Set ScalerData on RunSummary. + final ScalerDataReader scalerDataReader = new ScalerDataReader(); + scalerDataReader.setRun(this.getRun()); + scalerDataReader.setConnection(this.getConnection()); + scalerDataReader.read(); + this.getRunSummary().setScalerData(scalerDataReader.getData()); + + // Set ScalerData on RunSummary. + final EvioFileListReader evioFileListReader = new EvioFileListReader(); + evioFileListReader.setRun(this.getRun()); + evioFileListReader.setConnection(this.getConnection()); + evioFileListReader.read(); + this.getRunSummary().setEvioFileList(evioFileListReader.getData()); + } + + /** + * Set the database connection. + * + * @param connection the database connection + */ + public void setConnection(final Connection connection) { + this.connection = connection; + } + + /** + * Set the run number. + * + * @param run the run number + */ + public void setRun(final int run) { + + // Check status of database connection (must be open). + try { + if (this.connection.isClosed()) { + throw new IllegalStateException("The connection is closed."); + } + } catch (final SQLException e) { + throw new RuntimeException(e); + } + + this.run = run; + + try { + // Read the run records from the database and convert into Java objects. + this.readRun(); + } catch (final Exception e) { + LOGGER.log(Level.SEVERE, "Error reading from run database for run: " + run, e); + } + } + + /** + * Set the current {@link RunSummary}. + * + * @param runSummary the current {@link RunSummary} + */ + void setRunSummary(final RunSummary runSummary) { + this.runSummary = runSummary; + } +} Copied: java/trunk/record-util/src/main/java/org/hps/record/run/RunSummary.java (from r3257, java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummary.java) ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/evio/crawler/RunSummary.java (original) +++ java/trunk/record-util/src/main/java/org/hps/record/run/RunSummary.java Thu Jul 16 18:47:15 2015 @@ -1,4 +1,4 @@ -package org.hps.record.evio.crawler; +package org.hps.record.run; import java.io.File; import java.io.PrintStream; @@ -8,11 +8,10 @@ import java.util.GregorianCalendar; import java.util.Map; import java.util.TimeZone; -import java.util.logging.Logger; import org.hps.record.epics.EpicsData; +import org.hps.record.evio.crawler.EvioFileList; import org.hps.record.scalers.ScalerData; -import org.lcsim.util.log.LogUtil; /** * This class models the run summary information which is persisted as a row in the <i>run_log</i> table of the run @@ -37,14 +36,15 @@ * Set up date formatting to display EST (GMT-4). */ private static final DateFormat DATE_DISPLAY = new SimpleDateFormat(); + static { + /** + * Set default time zone to East Coast (JLAB) where data was taken. + */ DATE_DISPLAY.setCalendar(new GregorianCalendar(TimeZone.getTimeZone("America/New_York"))); } - /** - * Setup logger. - */ - private static final Logger LOGGER = LogUtil.create(RunSummary.class); + private Date created; /** * The end date of the run. @@ -69,7 +69,7 @@ /** * The list of EVIO files in the run. */ - private final EvioFileList files = new EvioFileList(); + private EvioFileList evioFileList = new EvioFileList(); /** * The run number. @@ -77,6 +77,11 @@ private final int run; /** + * Flag to indicate run was okay. + */ + private boolean runOkay = true; + + /** * The scaler data from the last physics event in the run. */ private ScalerData scalerData; @@ -92,11 +97,21 @@ private int totalEvents = -1; /** + * The total number of files in the run. + */ + private int totalFiles = 0; + + /** + * Date when the run record was last updated. + */ + private Date updated; + + /** * Create a run summary. * * @param run the run number */ - RunSummary(final int run) { + public RunSummary(final int run) { this.run = run; } @@ -105,11 +120,17 @@ * * @param file the file to add */ - void addFile(final File file) { - this.files.add(file); - - // Total events must be recomputed. - this.totalEvents = -1; + public void addFile(final File file) { + this.evioFileList.add(file); + } + + /** + * Get the creation date of this run record. + * + * @return the creation date of this run record + */ + public Date getCreated() { + return this.created; } /** @@ -124,6 +145,15 @@ } /** + * Return <code>true</code> if END event was found in the data. + * + * @return <code>true</code> if END event was in the data + */ + public boolean getEndOkay() { + return this.endOkay; + } + + /** * Get the EPICS data summary. * <p> * This is computed by taking the mean of each variable for the run. @@ -135,6 +165,19 @@ } /** + * Get the event rate (effectively the trigger rate) which is the total events divided by the number of seconds in + * the run. + * + * @return the event rate + */ + public double getEventRate() { + if (this.getTotalEvents() <= 0) { + throw new RuntimeException("Total events is zero or invalid."); + } + return (double) this.getTotalEvents() / (double) this.getTotalSeconds(); + } + + /** * Get the counts of different event types. * * @return the counts of different event types @@ -149,7 +192,7 @@ * @return the list of EVIO files in this run */ public EvioFileList getEvioFileList() { - return this.files; + return this.evioFileList; } /** @@ -162,8 +205,17 @@ } /** + * Return <code>true</code> if the run was okay (no major errors or data corruption occurred). + * + * @return <code>true</code> if the run was okay + */ + public boolean getRunOkay() { + return this.runOkay; + } + + /** * Get the scaler data of this run (last event only). - * + * * @return the scaler data of this run from the last event */ public ScalerData getScalerData() { @@ -188,13 +240,18 @@ return this.totalEvents; } - void setTotalEvents(int totalEvents) { - this.totalEvents = totalEvents; + /** + * Get the total number of files for this run. + * + * @return the total number of files for this run + */ + public int getTotalFiles() { + return this.totalFiles; } /** * Get the number of seconds in the run which is the difference between the start and end times. - * + * * @return the total seconds in the run */ public long getTotalSeconds() { @@ -204,29 +261,16 @@ if (this.getEndDate() == null) { throw new RuntimeException("missing end date"); } - return (getEndDate().getTime() - getStartDate().getTime()) / 1000; - } - - /** - * Get the event rate (effectively the trigger rate) which is the total events divided by the number of seconds in - * the run. - * - * @return the event rate - */ - public double getEventRate() { - if (this.getTotalEvents() <= 0) { - throw new RuntimeException("Total events is zero or invalid."); - } - return (double) this.getTotalEvents() / (double) this.getTotalSeconds(); - } - - /** - * Return <code>true</code> if END event was found in the data. - * - * @return <code>true</code> if END event was in the data - */ - public boolean isEndOkay() { - return this.endOkay; + return (this.getEndDate().getTime() - this.getStartDate().getTime()) / 1000; + } + + /** + * Get the date when this run record was last updated. + * + * @return the date when this run record was last updated + */ + public Date getUpdated() { + return updated; } /** @@ -234,32 +278,41 @@ * * @param ps the print stream for output */ - public void printRunSummary(final PrintStream ps) { + public void printOut(final PrintStream ps) { ps.println("--------------------------------------------"); ps.println("run: " + this.run); - ps.println("first file: " + this.files.first()); - ps.println("last file: " + this.files.last()); + ps.println("first file: " + this.evioFileList.first()); + ps.println("last file: " + this.evioFileList.last()); ps.println("started: " + DATE_DISPLAY.format(this.getStartDate())); ps.println("ended: " + DATE_DISPLAY.format(this.getEndDate())); ps.println("total events: " + this.getTotalEvents()); - ps.println("end OK: " + this.isEndOkay()); + ps.println("end OK: " + this.getEndOkay()); ps.println("event rate: " + this.getEventRate()); ps.println("event types"); for (final Object key : this.eventTypeCounts.keySet()) { ps.println(" " + key + ": " + this.eventTypeCounts.get(key)); } - ps.println(this.files.size() + " files"); - for (final File file : this.files) { + ps.println(this.evioFileList.size() + " files"); + for (final File file : this.evioFileList) { ps.println(" " + file.getPath()); } } /** + * Set the creation date of the run record. + * + * @param created the creation date of the run record + */ + public void setCreated(final Date created) { + this.created = created; + } + + /** * Set the end date. * * @param endDate the end date */ - void setEndDate(final Date endDate) { + public void setEndDate(final Date endDate) { this.endDate = endDate; } @@ -268,7 +321,7 @@ * * @param endOkay <code>true</code> if end is okay */ - void setEndOkay(final boolean endOkay) { + public void setEndOkay(final boolean endOkay) { this.endOkay = endOkay; } @@ -277,7 +330,7 @@ * * @param epics the EPICS data for the run */ - void setEpicsData(final EpicsData epics) { + public void setEpicsData(final EpicsData epics) { this.epics = epics; } @@ -286,16 +339,34 @@ * * @param eventTypeCounts the event type counts for the run */ - void setEventTypeCounts(final Map<Object, Integer> eventTypeCounts) { + public void setEventTypeCounts(final Map<Object, Integer> eventTypeCounts) { this.eventTypeCounts = eventTypeCounts; } /** + * Set the list of EVIO files for the run. + * + * @param evioFileList the list of EVIO files for the run + */ + public void setEvioFileList(final EvioFileList evioFileList) { + this.evioFileList = evioFileList; + } + + /** + * Set whether the run was "okay" meaning the data is usable for physics analysis. + * + * @param runOkay <code>true</code> if the run is okay + */ + public void setRunOkay(final boolean runOkay) { + this.runOkay = runOkay; + } + + /** * Set the scaler data of the run. - * + * * @param scalerData the scaler data */ - void setScalerData(final ScalerData scalerData) { + public void setScalerData(final ScalerData scalerData) { this.scalerData = scalerData; } @@ -304,25 +375,54 @@ * * @param startDate the start date of the run */ - void setStartDate(final Date startDate) { + public void setStartDate(final Date startDate) { this.startDate = startDate; } /** + * Set the total number of physics events in the run. + * + * @param totalEvents the total number of physics events in the run + */ + public void setTotalEvents(final int totalEvents) { + this.totalEvents = totalEvents; + } + + /** + * Set the total number of EVIO files in the run. + * + * @param totalFiles the total number of EVIO files in the run + */ + public void setTotalFiles(final int totalFiles) { + this.totalFiles = totalFiles; + } + + /** + * Set the date when this run record was last updated. + * + * @param updated the date when the run record was last updated + */ + public void setUpdated(final Date updated) { + this.updated = updated; + } + + /** * Sort the files in the run by sequence number in place. */ - void sortFiles() { - this.files.sort(); + public void sortFiles() { + this.evioFileList.sort(); } /** * Convert this object to a string. - * + * * @return this object converted to a string */ @Override public String toString() { - return "RunSummary { run: " + this.run + ", started: " + this.getStartDate() + ", ended: " + this.getEndDate() - + ", events: " + this.getTotalEvents() + ", endOkay: " + endOkay + " }"; + return "RunSummary { run: " + this.getRun() + ", startDate: " + this.getStartDate() + ", endDate: " + + this.getEndDate() + ", totalEvents: " + this.getTotalEvents() + ", totalFiles: " + + this.getTotalFiles() + ", endOkay: " + this.getEndOkay() + ", runOkay: " + this.getRunOkay() + + ", updated: " + this.getUpdated() + ", created: " + this.getCreated() + " }"; } } Added: java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryReader.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryReader.java (added) +++ java/trunk/record-util/src/main/java/org/hps/record/run/RunSummaryReader.java Thu Jul 16 18:47:15 2015 @@ -0,0 +1,68 @@ +package org.hps.record.run; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Convert run database records from the <i>runs</i> table into a {@link RunSummary} object. + * <p> + * This class will not create the sub-objects for the {@link RunSummary} which must be read using their own + * {@link AbstractRunDatabaseReader} implementation classes. Then these objects should be set on the {@link RunSummary} + * e.g. using {@link RunSummary#setEpicsData(org.hps.record.epics.EpicsData)}, etc. + * + * @author Jeremy McCormick, SLAC + */ +public class RunSummaryReader extends AbstractRunDatabaseReader<RunSummary> { + + /** + * The SQL SELECT query string. + */ + private final String SELECT_SQL = "SELECT run, start_date, end_date, nevents, nfiles, end_ok, run_ok, updated, created FROM runs WHERE run = ?"; + + /** + * Read data from the database and convert to a {@link RunSummary} object. + */ + @Override + void read() { + if (this.getRun() == -1) { + throw new IllegalStateException("run number is invalid: " + this.getRun()); + } + if (this.getConnection() == null) { + throw new IllegalStateException("Connection is not set."); + } + + PreparedStatement statement = null; + try { + statement = this.getConnection().prepareStatement(SELECT_SQL); + statement.setInt(1, this.getRun()); + final ResultSet resultSet = statement.executeQuery(); + if (!resultSet.next()) { + throw new RuntimeException("No record exists for run " + this.getRun() + " in database."); + } + + final RunSummary runSummary = new RunSummary(this.getRun()); + runSummary.setStartDate(resultSet.getTimestamp("start_date")); + runSummary.setEndDate(resultSet.getTimestamp("end_date")); + runSummary.setTotalEvents(resultSet.getInt("nevents")); + runSummary.setTotalFiles(resultSet.getInt("nfiles")); + runSummary.setEndOkay(resultSet.getBoolean("end_ok")); + runSummary.setRunOkay(resultSet.getBoolean("run_ok")); + runSummary.setUpdated(resultSet.getTimestamp("updated")); + runSummary.setCreated(resultSet.getTimestamp("created")); + + this.setData(runSummary); + + } catch (final SQLException e) { + throw new RuntimeException(e); + } finally { + if (statement != null) { + try { + statement.close(); + } catch (final SQLException e) { + e.printStackTrace(); + } + } + } + } +} Added: java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataReader.java ============================================================================= --- java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataReader.java (added) +++ java/trunk/record-util/src/main/java/org/hps/record/run/ScalerDataReader.java Thu Jul 16 18:47:15 2015 @@ -0,0 +1,58 @@ +package org.hps.record.run; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.hps.record.scalers.ScalerData; + +/** + * + * @author Jeremy McCormick, SLAC + */ +public class ScalerDataReader extends AbstractRunDatabaseReader<ScalerData> { + + private String SELECT_SQL = "SELECT idx, value FROM run_scalers WHERE run = ? ORDER BY idx"; + + @Override + void read() { + if (getRun() == -1) { + throw new IllegalStateException("run number is invalid: " + getRun()); + } + if (getConnection() == null) { + throw new IllegalStateException("Connection is not set."); + } + + PreparedStatement statement = null; + try { + statement = getConnection().prepareStatement(SELECT_SQL); + statement.setInt(1, getRun()); + ResultSet resultSet = statement.executeQuery(); + + List<Integer> scalerValues = new ArrayList<Integer>(); + while (resultSet.next()) { + scalerValues.add(resultSet.getInt("value")); + } + + int[] scalerArray = new int[scalerValues.size()]; + for (int i = 0; i < scalerArray.length; i++) { + scalerArray[i] = scalerValues.get(i); + } + + setData(new ScalerData(scalerArray)); + + } catch (SQLException e) { + throw new RuntimeException(e); + } finally { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + } +}