6 added files
java/sandbox/data-cat
--- java/sandbox/data-cat/pom.xml (rev 0)
+++ java/sandbox/data-cat/pom.xml 2014-04-10 22:04:57 UTC (rev 472)
@@ -0,0 +1,58 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.hps</groupId>
+ <artifactId>hps-datacat-scratch</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ <name>datacat-scratch</name>
+
+ <repositories>
+<!--
+ <repository>
+ <id>freehep-repo-public</id>
+ <name>FreeHEP Maven Public</name>
+ <url>http://srs.slac.stanford.edu/nexus/content/groups/freehep-maven2-public/</url>
+ </repository>
+-->
+ <repository>
+ <id>srs-maven2-public</id>
+ <name>SRS Maven 2 central repository</name>
+ <url>http://srs.slac.stanford.edu/nexus/content/groups/srs-maven2-public/</url>
+ </repository>
+ <repository>
+ <id>lcsim-repo-public</id>
+ <name>LCSIM Public Maven Repository</name>
+ <url>http://srs.slac.stanford.edu/nexus/content/groups/lcsim-maven2-public/</url>
+ </repository>
+ <repository>
+ <id>maven-central-repo</id>
+ <name>Maven repository</name>
+ <url>http://repo1.maven.org/maven2/</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.lcsim</groupId>
+ <artifactId>lcsim-distribution</artifactId>
+ <version>3.0.4-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.lcsim</groupId>
+ <artifactId>lcio</artifactId>
+ <version>2.4.4-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>srs</groupId>
+ <artifactId>org-srs-datacat-server</artifactId>
+ <version>1.7.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+</project>
java/sandbox/data-cat/scripts
--- java/sandbox/data-cat/scripts/hps_datacat_find.py (rev 0)
+++ java/sandbox/data-cat/scripts/hps_datacat_find.py 2014-04-10 22:04:57 UTC (rev 472)
@@ -0,0 +1,86 @@
+#!/usr/bin/env python2.7
+
+"""
+Script wrapper for updating HPS data catalog via SSH connection to SLAC.
+
+author: Jeremy McCormick <[log in to unmask]>
+"""
+
+# TODO: set properly Run Min, Run Max, and Events by dataset.
+
+import argparse, os.path, subprocess
+
+# path to script at SLAC
+script_cmd = '~srs/datacat/prod/datacat find'
+
+# default options for search command
+script_options = '--search-groups --recurse'
+
+# default SLAC account for SSH connection
+connection = [log in to unmask]
+
+# site
+site = 'SLAC'
+
+parser = argparse.ArgumentParser(description='Search for files in HPS data catalog')
+parser.add_argument('-p', '--path', help='root path for search')
+parser.add_argument('-s', '--site', help='dataset site')
+parser.add_argument('-c', '--connection', help='SSH connection')
+parser.add_argument('-o', '--output', help='save results to output file')
+parser.add_argument('-q', '--query', help='data query for filtering results')
+
+args = vars(parser.parse_args())
+
+if args['path'] != None:
+ path = args['path']
+else:
+ raise Exception('path is required!')
+
+if args['site'] != None:
+ site = args['site']
+ if site != 'SLAC' and site != 'JLAB':
+ raise Exception("Unrecognized site!")
+
+if args['connection'] != None:
+ connection = args['connection']
+
+output = None
+if args['output'] != None:
+ output = args['output']
+ if os.path.isfile(output):
+ raise Exception('Output file already exists!')
+
+query = ''
+if args['query'] != None:
+ #print args['query']
+ query = '--filter \'%s\'' % args['query']
+ print query
+
+cmd = 'ssh %s %s %s --site %s %s %s' % (connection, script_cmd, script_options, site, query, path)
+
+print cmd
+
+if output != None:
+ output_file = open(output, 'w')
+
+process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+errors = []
+lines = []
+for line in process.stdout.readlines():
+ lines.append(line)
+ if 'Exception' in line:
+ errors.append(line)
+
+retval = process.wait()
+
+if (retval == 0 and len(errors) == 0):
+ if output != None:
+ for line in lines:
+ output_file.write(line)
+ output_file.close()
+ else:
+ for line in lines:
+ print line,
+else:
+ print 'The find command failed with errors!'
+ print errors
\ No newline at end of file
java/sandbox/data-cat/scripts
--- java/sandbox/data-cat/scripts/hps_datacat_register_file.py (rev 0)
+++ java/sandbox/data-cat/scripts/hps_datacat_register_file.py 2014-04-10 22:04:57 UTC (rev 472)
@@ -0,0 +1,110 @@
+#!/usr/bin/env python2.7
+
+"""
+Script wrapper for updating HPS data catalog via SSH connection to SLAC.
+
+author: Jeremy McCormick <[log in to unmask]>
+"""
+
+# TODO: set properly Run Min, Run Max, and Events by dataset.
+
+import argparse, os.path, subprocess
+
+# path to script at SLAC
+script_cmd = '~srs/datacat/prod/datacat registerDataset'
+
+# default SLAC account for SSH connection
+connection = [log in to unmask]
+
+# lowest level node in directory hierarchy
+group = 'HPS'
+
+# site
+site = 'SLAC'
+
+parser = argparse.ArgumentParser(description='Register file in HPS data catalog')
+parser.add_argument('-p', '--path', help='destination path in data catalog')
+parser.add_argument('-d', '--dataset', help='input physical dataset')
+parser.add_argument('-m', '--metadata', nargs='*', help='define meta data')
+parser.add_argument('-g', '--group', help='dataset group')
+parser.add_argument('-s', '--site', help='dataset site')
+parser.add_argument('-c', '--connection', help='SSH connection')
+parser.add_argument('-v', '--verbose', help='turn verbose mode on', action='store_true')
+parser.add_argument('-D', '--dry-run', help='dry run only', action='store_true')
+args = vars(parser.parse_args())
+
+verbose = False
+if args['verbose'] != None:
+ verbose = True
+
+if verbose:
+ print args
+
+if args['path'] != None:
+ path = args['path']
+else:
+ raise Exception('path is required!')
+
+if args['dataset'] != None:
+ dataset = args['dataset']
+else:
+ raise Exception('dataset is required!')
+
+ext = os.path.splitext(dataset)[1][1:]
+
+if args['group'] != None:
+ group = args['group']
+
+metadata = ''
+if args['metadata'] != None:
+ print args['metadata']
+ for var in args['metadata']:
+ equals = var.find('=')
+ if (len(var) < 3 or equals < 0):
+ raise Exception("Bad meta data variable format!")
+ metadata += '--define %s ' % (var)
+
+if args['site'] != None:
+ site = args['site']
+ if site != 'SLAC' and site != 'JLAB':
+ raise Exception("Unrecognized site!")
+
+if args['connection'] != None:
+ connection = args['connection']
+
+dry_run = False
+if args['dry_run']:
+ dry_run = True
+
+cmd = 'ssh %s %s --group %s --site %s %s %s %s %s' % (connection, script_cmd, group, site, metadata, ext, path, dataset)
+
+if verbose:
+ print cmd
+
+if dry_run:
+ print 'Configured for dry run so command will not be executed.'
+else:
+ process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ errors = []
+ for line in process.stdout.readlines():
+ if verbose:
+ print line,
+ if 'Exception' in line:
+ errors.append(line)
+
+ retval = process.wait()
+
+ if len(errors) != 0 or retval != 0:
+ print 'The registration failed with errors!'
+ print errors
+ else:
+ print 'Added data to catalog ...'
+ print ' dataset: %s' % dataset
+ print ' path: %s' % path
+ print ' group: %s' % group
+ #print ' metadata: ' % str(args['metadata'])
+ print ' site: ' % site
+
+ print 'return value: %d' % retval
+
+print 'Done!'
\ No newline at end of file
java/sandbox/data-cat/src/main/java/org/srs/datacat/server
--- java/sandbox/data-cat/src/main/java/org/srs/datacat/server/LcioContentChecker.java (rev 0)
+++ java/sandbox/data-cat/src/main/java/org/srs/datacat/server/LcioContentChecker.java 2014-04-10 22:04:57 UTC (rev 472)
@@ -0,0 +1,94 @@
+package org.srs.datacat.server;
+
+import hep.lcio.event.LCEvent;
+import hep.lcio.implementation.io.LCFactory;
+import hep.lcio.io.ILCFactory;
+import hep.lcio.io.LCReader;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Map;
+
+public class LcioContentChecker implements ContentChecker {
+
+ private long minRunNumber = Integer.MAX_VALUE;
+ private long maxRunNumber = Integer.MIN_VALUE;
+ private long nevents = 0;
+ private String status;
+ private LinkedHashSet<String> collections;
+ Map<String, Object> metaData;
+
+ @Override
+ public void setLocation(long datasetVersion, URL url) throws IOException {
+ status = "OK";
+ collections = new LinkedHashSet<String>();
+ metaData = new HashMap<String, Object>();
+ if (!url.getProtocol().equals("file")) {
+ throw new IOException("Only file protocol is supported.");
+ }
+ LCReader reader = LCFactory.getInstance().createLCReader();
+ reader.open(url.getPath());
+ LCEvent event = reader.readNextEvent();
+ minRunNumber = Integer.MAX_VALUE;
+ maxRunNumber = Integer.MIN_VALUE;
+ while (event != null) {
+
+ int thisRunNumber = event.getRunNumber();
+
+ if (thisRunNumber < minRunNumber)
+ minRunNumber = thisRunNumber;
+ if (thisRunNumber > maxRunNumber)
+ maxRunNumber = thisRunNumber;
+
+ collections.addAll(Arrays.asList(event.getCollectionNames()));
+
+ nevents++;
+
+ event = reader.readNextEvent();
+ }
+
+ StringBuffer buff = new StringBuffer();
+ for (String collectionName : collections) {
+ buff.append(collectionName);
+ buff.append(',');
+ }
+ buff.setLength(buff.length() - 1);
+
+ metaData.put("sCollections", buff.toString());
+
+ reader.close();
+ }
+
+ @Override
+ public long getEventCount() throws IOException {
+ return nevents;
+ }
+
+ @Override
+ public long getRunMin() throws IOException {
+ return minRunNumber;
+ }
+
+ @Override
+ public long getRunMax() throws IOException {
+ return maxRunNumber;
+ }
+
+ @Override
+ public String getStatus() throws IOException {
+ return status;
+ }
+
+ @Override
+ public Map<String,Object> getMetaData() throws IOException {
+ return metaData;
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+}
\ No newline at end of file
java/sandbox/data-cat/src/main/java/org/srs/datacat/server
--- java/sandbox/data-cat/src/main/java/org/srs/datacat/server/LcioContentCheckerCreator.java (rev 0)
+++ java/sandbox/data-cat/src/main/java/org/srs/datacat/server/LcioContentCheckerCreator.java 2014-04-10 22:04:57 UTC (rev 472)
@@ -0,0 +1,14 @@
+package org.srs.datacat.server;
+
+
+public class LcioContentCheckerCreator implements ContentCheckerCreator {
+
+ public ContentChecker create() {
+ return new LcioContentChecker();
+ }
+
+ public void free(ContentChecker cc) {
+ return;
+ }
+
+}
java/sandbox/data-cat/src/test/java/org/srs/datacat/server
--- java/sandbox/data-cat/src/test/java/org/srs/datacat/server/LcioContentCheckerTest.java (rev 0)
+++ java/sandbox/data-cat/src/test/java/org/srs/datacat/server/LcioContentCheckerTest.java 2014-04-10 22:04:57 UTC (rev 472)
@@ -0,0 +1,33 @@
+package org.srs.datacat.server;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Map;
+
+import org.lcsim.util.cache.FileCache;
+
+
+public class LcioContentCheckerTest {
+
+ private static final String TEST_FILE_URL = "http://www.lcsim.org/test/hps/conditions_test.slcio";
+
+ public void testLcioContentChecker() throws IOException {
+
+ FileCache cache = new FileCache();
+ File file = cache.getCachedFile(new URL(TEST_FILE_URL));
+
+ ContentChecker checker = new LcioContentCheckerCreator().create();
+ checker.setLocation(-1, file.toURL());
+ System.out.println("event count: " + checker.getEventCount());
+ System.out.println("run max: " + checker.getRunMax());
+ System.out.println("run min: " + checker.getRunMin());
+ System.out.println("status: " + checker.getStatus());
+ System.out.println("meta data ...");
+ Map<String, Object> metaData = checker.getMetaData();
+ for (String key : metaData.keySet()) {
+ System.out.println(" " + key + ": " + metaData.get(key));
+ }
+ }
+
+}
SVNspam 0.1
