Author: [log in to unmask]
Date: Tue Jan 5 14:45:34 2016
New Revision: 4082
Log:
Minor changes to datacat crawler and rundb tools.
Modified:
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/AidaMetadataReader.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerFileVisitor.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatHelper.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileUtilities.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/LcioReconMetadataReader.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RootDqmMetadataReader.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RootDstMetadataReader.java
java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RunFilter.java
java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunDatabaseBuilder.java
java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunDatabaseCommandLine.java
java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunManager.java
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/AidaMetadataReader.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/AidaMetadataReader.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/AidaMetadataReader.java Tue Jan 5 14:45:34 2016
@@ -22,7 +22,7 @@
@Override
public Map<String, Object> getMetadata(final File file) throws IOException {
final Map<String, Object> metadata = new HashMap<String, Object>();
- final int run = FileUtilities.getRunFromFileName(file);
+ final Long run = FileUtilities.getRunFromFileName(file);
metadata.put("runMin", run);
metadata.put("runMax", run);
return metadata;
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerFileVisitor.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerFileVisitor.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/CrawlerFileVisitor.java Tue Jan 5 14:45:34 2016
@@ -8,6 +8,7 @@
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
+import java.util.logging.Logger;
/**
* Visitor which creates a list of files from walking a directory tree.
@@ -19,6 +20,8 @@
*/
final class CrawlerFileVisitor extends SimpleFileVisitor<Path> {
+ private static Logger LOGGER = Logger.getLogger(CrawlerFileVisitor.class.getPackage().getName());
+
/**
* The list of files found from crawling.
*/
@@ -75,6 +78,7 @@
final File file = path.toFile();
if (this.accept(file)) {
files.add(file);
+ LOGGER.info("accepted file " + file.getPath());
}
return FileVisitResult.CONTINUE;
}
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatCrawler.java Tue Jan 5 14:45:34 2016
@@ -10,7 +10,6 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.cli.CommandLine;
@@ -182,6 +181,11 @@
LOGGER.config("adding format " + format.name());
this.config.addFileFormat(format);
}
+ } else {
+ for (FileFormat format : FileFormat.values()) {
+ this.config.addFileFormat(format);
+ LOGGER.config("adding default format " + format);
+ }
}
// Enable the default set of file formats.
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatHelper.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatHelper.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/DatacatHelper.java Tue Jan 5 14:45:34 2016
@@ -10,6 +10,8 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import org.srs.datacat.client.Client;
import org.srs.datacat.client.ClientBuilder;
@@ -24,6 +26,8 @@
* @author Jeremy McCormick, SLAC
*/
class DatacatHelper {
+
+ private static final Logger LOGGER = Logger.getLogger(DatacatHelper.class.getPackage().getName());
/*
* Default base URL for datacat.
@@ -60,6 +64,7 @@
* @return the metadata for the file
*/
static Map<String, Object> createMetadata(final File file) {
+ LOGGER.fine("creating metadata for " + file.getPath());
File actualFile = file;
if (FileUtilities.isMssFile(file)) {
actualFile = FileUtilities.getCachedFile(file);
@@ -184,6 +189,8 @@
String dataType,
String fileFormat) {
+ LOGGER.info("creating dataset for " + file.getPath());
+
Provider provider = new Provider();
Dataset.Builder datasetBuilder = provider.getDatasetBuilder();
@@ -240,8 +247,8 @@
FileFormat fileFormat = DatacatHelper.getFileFormat(file);
DatasetModel dataset = DatacatHelper.createDataset(
file,
- metadata,
- folder,
+ metadata,
+ folder,
site,
dataType.toString(),
fileFormat.toString());
@@ -265,7 +272,11 @@
throw new RuntimeException("Bad datacat URL.", e);
}
for (DatasetModel dataset : datasets) {
- client.createDataset(folder, dataset);
+ try {
+ client.createDataset(folder, dataset);
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, e.getMessage(), e);
+ }
}
}
}
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/EvioMetadataReader.java Tue Jan 5 14:45:34 2016
@@ -26,8 +26,6 @@
/**
* Reads metadata from EVIO files, including the event count, run min and run max expected by the datacat, as well as
* many custom field values applicable to HPS EVIO raw data.
- * <p>
- * The size of the data file is set externally to this reader using the datacat client.
*
* @author Jeremy McCormick, SLAC
*/
@@ -56,8 +54,6 @@
*/
@Override
public Map<String, Object> getMetadata(final File file) throws IOException {
-
- LOGGER.info("creating metadata for " + file.getPath());
long events = 0;
int badEvents = 0;
@@ -114,7 +110,7 @@
// End of file.
if (evioEvent == null) {
- LOGGER.info("EOF after " + events + " events");
+ LOGGER.fine("EOF after " + events + " events");
break fileLoop;
}
@@ -140,7 +136,7 @@
// First header timestamp.
if (firstHeadTimestamp == null) {
firstHeadTimestamp = thisTimestamp;
- LOGGER.info("first head timestamp " + firstHeadTimestamp + " from event "
+ LOGGER.finer("first head timestamp " + firstHeadTimestamp + " from event "
+ evioEvent.getEventNumber());
}
@@ -152,7 +148,7 @@
if (run == null) {
if (headBankData[1] != 0) {
run = (long) headBankData[1];
- LOGGER.info("run " + run + " from event " + evioEvent.getEventNumber());
+ LOGGER.finer("run " + run + " from event " + evioEvent.getEventNumber());
}
}
}
@@ -189,7 +185,7 @@
// Set the first physics event.
if (firstPhysicsEvent == null) {
firstPhysicsEvent = eventIdData[0];
- LOGGER.info("set first physics event " + firstPhysicsEvent);
+ LOGGER.finer("set first physics event " + firstPhysicsEvent);
}
}
}
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileUtilities.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileUtilities.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/FileUtilities.java Tue Jan 5 14:45:34 2016
@@ -20,9 +20,8 @@
* @param file the file
* @return the run number
*/
- static int getRunFromFileName(final File file) {
- final String name = file.getName();
- return Integer.parseInt(name.substring(4, 10));
+ static Long getRunFromFileName(final File file) {
+ return Long.parseLong(file.getName().substring(5, 10));
}
/**
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/LcioReconMetadataReader.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/LcioReconMetadataReader.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/LcioReconMetadataReader.java Tue Jan 5 14:45:34 2016
@@ -45,10 +45,10 @@
@Override
public Map<String, Object> getMetadata(File file) throws IOException {
- Set<String> collectionNames = new HashSet<String>();
+ //Set<String> collectionNames = new HashSet<String>();
String detectorName = null;
- int eventCount = 0;
- Integer run = null;
+ long eventCount = 0;
+ Long run = null;
LCIOReader reader = null;
try {
reader = new LCIOReader(file);
@@ -56,39 +56,42 @@
try {
while((eventHeader = reader.read()) != null) {
if (run == null) {
- run = eventHeader.getRunNumber();
+ run = (long) eventHeader.getRunNumber();
}
if (detectorName == null) {
detectorName = eventHeader.getDetectorName();
}
- for (List<?> list : eventHeader.getLists()) {
- LCMetaData metadata = eventHeader.getMetaData(list);
- collectionNames.add(metadata.getName());
- }
+ //for (List<?> list : eventHeader.getLists()) {
+ // LCMetaData metadata = eventHeader.getMetaData(list);
+ // collectionNames.add(metadata.getName());
+ //}
eventCount++;
}
} catch (EOFException e) {
- e.printStackTrace();
}
} finally {
if (reader != null) {
- reader.close();
+ try {
+ reader.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
}
// Build collection names string.
- StringBuffer sb = new StringBuffer();
- for (String collectionName : collectionNames) {
- sb.append(collectionName + ",");
- }
- sb.setLength(sb.length() - 1);
+ //StringBuffer sb = new StringBuffer();
+ //for (String collectionName : collectionNames) {
+ // sb.append(collectionName + ",");
+ //}
+ //sb.setLength(sb.length() - 1);
Map<String, Object> metadata = new HashMap<String, Object>();
metadata.put("eventCount", eventCount);
metadata.put("runMin", run);
metadata.put("runMax", run);
metadata.put("DETECTOR", detectorName);
- metadata.put("COLLECTIONS", sb.toString());
+ //metadata.put("COLLECTIONS", sb.toString());
return metadata;
}
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RootDqmMetadataReader.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RootDqmMetadataReader.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RootDqmMetadataReader.java Tue Jan 5 14:45:34 2016
@@ -22,7 +22,7 @@
@Override
public Map<String, Object> getMetadata(final File file) throws IOException {
final Map<String, Object> metadata = new HashMap<String, Object>();
- final int run = FileUtilities.getRunFromFileName(file);
+ final Long run = FileUtilities.getRunFromFileName(file);
metadata.put("runMin", run);
metadata.put("runMax", run);
return metadata;
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RootDstMetadataReader.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RootDstMetadataReader.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RootDstMetadataReader.java Tue Jan 5 14:45:34 2016
@@ -28,6 +28,10 @@
@Override
public Map<String, Object> getMetadata(final File file) throws IOException {
final Map<String, Object> metadata = new HashMap<String, Object>();
+ Long run = FileUtilities.getRunFromFileName(file);
+ metadata.put("runMin", run);
+ metadata.put("runMax", run);
+ /*
RootFileReader rootReader = null;
long eventCount = 0;
int runMin = 0;
@@ -60,6 +64,7 @@
metadata.put("runMin", runMin);
metadata.put("runMax", runMax);
metadata.put("size", size);
+ */
return metadata;
}
}
Modified: java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RunFilter.java
=============================================================================
--- java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RunFilter.java (original)
+++ java/branches/jeremy-dev/crawler/src/main/java/org/hps/crawler/RunFilter.java Tue Jan 5 14:45:34 2016
@@ -3,8 +3,6 @@
import java.io.File;
import java.io.FileFilter;
import java.util.Set;
-
-import org.hps.record.evio.EvioFileUtilities;
/**
* A filter which rejects files with run numbers not in a specified set.
@@ -25,7 +23,7 @@
*/
RunFilter(final Set<Integer> acceptRuns) {
if (acceptRuns.isEmpty()) {
- throw new IllegalArgumentException("the acceptRuns collection is empty");
+ throw new IllegalArgumentException("The acceptRuns collection is empty.");
}
this.acceptRuns = acceptRuns;
}
@@ -39,9 +37,9 @@
@Override
public boolean accept(final File file) {
try {
- int run = EvioFileUtilities.getRunFromName(file);
+ int run = Integer.parseInt(file.getName().substring(5, 10));
return this.acceptRuns.contains(run);
- } catch (NumberFormatException e) {
+ } catch (Exception e) {
return false;
}
}
Modified: java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunDatabaseBuilder.java
=============================================================================
--- java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunDatabaseBuilder.java (original)
+++ java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunDatabaseBuilder.java Tue Jan 5 14:45:34 2016
@@ -140,6 +140,11 @@
* Datacat site to use.
*/
private String site;
+
+ /**
+ * Default folder for file search.
+ */
+ private String folder;
/**
* Reload state for the current run number for testing.
@@ -186,13 +191,14 @@
private void createEvioFileList() {
this.evioFiles = new ArrayList<File>();
- for (DatasetModel dataset : this.evioDatasets) {
+ for (DatasetModel dataset : this.evioDatasets) {
String resource =
((DatasetWithViewModel) dataset).getViewInfo().getLocations().iterator().next().getResource();
File file = new File(resource);
if (file.getPath().startsWith("/mss")) {
file = new File("/cache" + resource);
}
+ LOGGER.info("adding EVIO file " + file.getPath() + " from dataset " + dataset.getName());
this.evioFiles.add(file);
}
EvioFileUtilities.sortBySequence(this.evioFiles);
@@ -203,26 +209,33 @@
*/
private void findEvioDatasets() {
- LOGGER.info("finding EVIO datasets for run " + getRun());
-
+ LOGGER.info("finding EVIO datasets for run " + getRun() + " in folder " + this.folder + " at site " + this.site);
+
DatasetResultSetModel results = datacatClient.searchForDatasets(
- "/HPS/data/raw",
- "current",
+ this.folder,
+ "current",
this.site,
- "fileFormat eq 'EVIO' AND dataType eq 'RAW' AND runMin eq " + getRun(),
- null,
- null,
- null,
+ "fileFormat eq 'EVIO' AND dataType eq 'RAW' AND runMin eq " + getRun(),
+ null,
+ null,
+ null,
null
);
this.evioDatasets = results.getResults();
- if (!this.evioDatasets.isEmpty()) {
+ if (this.evioDatasets.isEmpty()) {
throw new RuntimeException("No EVIO datasets found in data catalog.");
}
-
- this.evioFiles = new ArrayList<File>();
+ }
+
+ /**
+ * Get the folder for dataset search.
+ *
+ * @return the folder for dataset search
+ */
+ String folder() {
+ return folder;
}
/**
@@ -426,9 +439,6 @@
// Set END timestamp.
updateEndTimestamp();
-
- // Calculate trigger rate.
- updateTriggerRate();
// Run the full EVIO processing job.
if (!this.skipEvioProcessing) {
@@ -437,6 +447,9 @@
LOGGER.info("EVIO file processing is skipped.");
}
+ // Calculate trigger rate.
+ updateTriggerRate();
+
// Get extra info from the spreadsheet.
if (this.spreadsheetFile != null) {
updateFromSpreadsheet();
@@ -511,6 +524,14 @@
RunDatabaseBuilder setDryRun(boolean dryRun) {
this.dryRun = dryRun;
LOGGER.config("dryRun = " + this.dryRun);
+ return this;
+ }
+
+ /**
+ * Default folder for file search.
+ */
+ RunDatabaseBuilder setFolder(String folder) {
+ this.folder = folder;
return this;
}
Modified: java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunDatabaseCommandLine.java
=============================================================================
--- java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunDatabaseCommandLine.java (original)
+++ java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunDatabaseCommandLine.java Tue Jan 5 14:45:34 2016
@@ -39,6 +39,7 @@
OPTIONS.addOption("L", "load", false, "load back run information after inserting (for debugging)");
OPTIONS.addOption("u", "url", true, "data catalog URL");
OPTIONS.addOption("S", "site", true, "data catalog site (e.g. SLAC or JLAB)");
+ OPTIONS.addOption("f", "folder", true, "folder in datacat for dataset search");
}
/**
@@ -104,6 +105,11 @@
* Data catalog URL.
*/
private String url = "http://hpsweb.jlab.org/datacat/r";
+
+ /**
+ * Default folder for file search.
+ */
+ private String folder = "/HPS/data/raw";
/**
* Parse command line options and return reference to <code>this</code> object.
@@ -186,6 +192,11 @@
site = cl.getOptionValue("S");
}
+ // Set folder for dataset search.
+ if (cl.hasOption("f")) {
+ folder = cl.getOptionValue("f");
+ }
+
// Initialize the data catalog client.
try {
datacatClient = new ClientBuilder().setUrl(url).build();
@@ -206,6 +217,7 @@
private void run() {
new RunDatabaseBuilder()
.createRunSummary(run)
+ .setFolder(folder)
.setDetectorName(detectorName)
.setConnectionParameters(connectionParameters)
.setDatacatClient(datacatClient)
Modified: java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunManager.java
=============================================================================
--- java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunManager.java (original)
+++ java/branches/jeremy-dev/run-database/src/main/java/org/hps/run/database/RunManager.java Tue Jan 5 14:45:34 2016
@@ -128,7 +128,7 @@
void deleteRun() {
factory.getEpicsDataDao().deleteEpicsData(EpicsType.EPICS_2S, run);
factory.getEpicsDataDao().deleteEpicsData(EpicsType.EPICS_20S, run);
- factory.getScalerDataDao().deleteScalerData(run);
+ factory.getScalerDataDao().deleteScalerData(run);
factory.getSvtConfigDao().deleteSvtConfigs(run);
factory.getTriggerConfigDao().deleteTriggerConfig(run);
factory.getRunSummaryDao().deleteRunSummary(run);
|
