Author: [log in to unmask]
Date: Wed Sep 16 17:31:20 2015
New Revision: 3620
Log:
Updates to datacat client.
Modified:
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClient.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientFactory.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatConstants.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/Dataset.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetDataType.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetFileFormat.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetImpl.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetLocation.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetLocationImpl.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetMetadata.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetSite.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetUtilities.java
java/trunk/datacat-client/src/main/java/org/hps/datacat/client/HttpUtilities.java
java/trunk/datacat-client/src/test/java/org/hps/datacat/client/DatacatClientTest.java
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClient.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClient.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClient.java Wed Sep 16 17:31:20 2015
@@ -6,89 +6,102 @@
/**
* Interface to the SLAC SRS datacat system via HTTP/REST calls.
- *
+ *
* @author Jeremy McCormick, SLAC
*/
public interface DatacatClient {
/**
* Add a dataset to the data catalog.
- *
+ *
* @param folder the folder which must already exist
* @param dataType the data type
* @param resource the resource (path)
- * @param site the site of the file
+ * @param site the site of the file
* @param fileFormat the file format
* @param name the name of the dataset
* @param metadata metadata to assign to the dataset
* @return the HTTP status code from the request
*/
- int addDataset(String folder,
- DatasetDataType dataType,
- String resource,
- DatasetSite site,
- DatasetFileFormat fileFormat,
- String name,
- Map<String, Object> metadata);
-
- /**
- * Create a folder in the data catalog.
- *
- * @param folder the folder's path
- * @return the HTTP status code from the request
- */
- int makeFolder(String folder);
-
+ int addDataset(String folder, DatasetDataType dataType, String resource, DatasetSite site,
+ DatasetFileFormat fileFormat, String name, Map<String, Object> metadata);
+
/**
* Add metadata to an existing dataset.
- *
+ *
* @param folder the folder
* @param datasetName the name of the dataset
- * @param metaData the map of metadata where values can be <code>String</code>, <code>Integer</code> or <code>Float</code>
+ * @param metaData the map of metadata where values can be <code>String</code>, <code>Integer</code> or
+ * <code>Float</code>
* @return the HTTP status code from the request
*/
int addMetadata(String folder, String datasetName, Map<String, Object> metaData);
-
- /**
- * Remove a folder from the catalog.
- * <p>
- * It must be empty or an error will occur.
- *
- * @param folder the folder path
- * @return the HHTP status code from the request
- */
- int removeFolder(String folder);
-
+
/**
* Delete a dataset from the catalog.
* <p>
* This has no affect on the underlying resource (file).
- *
+ *
* @param path the path of the dataset
* @return the HTTP status code from the reqest
*/
int deleteDataset(String path);
-
+
+ /**
+ * Return <code>true</code> if the path exists in the datacat.
+ *
+ * @param path the path in the datacat
+ * @return <code>true</code> if the path exists
+ */
+ boolean exists(String path);
+
/**
* Find datasets in the catalog.
* <p>
* See <a href="http://docs.datacatalog.apiary.io/#search">Search Doc</a> for more details.
- *
+ *
* @param folder the folder path
* @param query the query to execute
* @return the HTTP status code from the request
*/
List<Dataset> findDatasets(String folder, String query, Set<String> showMetadata);
-
- // TODO: method to get dataset from path
- // to get all metadata need the site in query
- // http://localhost:8080/datacat-v0.4-SNAPSHOT/r/path.json/HPS/derp/herp01;s=SLAC
- // use HTTP GET
- // Dataset getDataSet(String path);
-
- // TODO: method to determine if folder or dataset exists
- // http://localhost:8080/datacat-v0.4-SNAPSHOT/r/path.json/HPS/derp/derp
- // will return
- // {"message":"File doesn't exist","type":"NoSuchFileException","cause":"Unable to resolve /HPS/derp/derp in parent Name: derp\tPath: /HPS/derp\t"}
- // boolean exists(String path);
+
+ /**
+ * Get a dataset from its path.
+ * <p>
+ * Example URL:
+ *
+ * <pre>
+ * http://localhost:8080/datacat-v0.4-SNAPSHOT/r/path.json/HPS/data/hps_005772.evio.0;s=SLAC
+ * </pre>
+ *
+ * @param path the path in the data catalog
+ * @return the dataset
+ */
+ Dataset getDataSet(String path, DatasetSite site);
+
+ /**
+ * Return <code>true</code> if path is a folder in the data catalog.
+ *
+ * @return <code>true</code> if path is a folder in the data catalog
+ */
+ boolean isFolder(String path);
+
+ /**
+ * Create a folder in the data catalog.
+ *
+ * @param folder the folder's path
+ * @return the HTTP status code from the request
+ */
+ int makeFolder(String folder);
+
+ /**
+ * Remove a folder from the catalog.
+ * <p>
+ * It must be empty or an error will occur.
+ *
+ * @param folder the folder path
+ * @return the HHTP status code from the request
+ */
+ int removeFolder(String folder);
}
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientFactory.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientFactory.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientFactory.java Wed Sep 16 17:31:20 2015
@@ -1,11 +1,11 @@
package org.hps.datacat.client;
/**
- * Factory class for providing user access to interfaces with protected implementation classes.
+ * Factory class for providing user access to interfaces that have protected implementations.
*
* @author Jeremy McCormick, SLAC
*/
-public class DatacatClientFactory {
+public final class DatacatClientFactory {
/**
* Create a datacat client.
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatClientImpl.java Wed Sep 16 17:31:20 2015
@@ -13,54 +13,54 @@
import java.util.logging.Logger;
import org.json.JSONObject;
+import org.lcsim.util.log.DefaultLogFormatter;
import org.lcsim.util.log.LogUtil;
-import org.lcsim.util.log.MessageOnlyLogFormatter;
/**
* Implementation of {@link DatacatClient} interface for working with SRS datacat REST API.
- *
+ *
* @author Jeremy McCormick, SLAC
*/
final class DatacatClientImpl implements DatacatClient {
-
+
/**
* Setup class logging.
*/
- private static Logger LOGGER = LogUtil.create(DatacatClientImpl.class, new MessageOnlyLogFormatter(), Level.ALL);
+ private static Logger LOGGER = LogUtil.create(DatacatClientImpl.class, new DefaultLogFormatter(), Level.ALL);
+
+ /**
+ * The root directory (e.g. should be 'HPS').
+ */
+ private final String rootDir;
+
+ /**
+ * The site (SLAC or JLAB).
+ */
+ private final DatasetSite site;
/**
* The base URL of the datacat server.
*/
private URL url;
-
- /**
- * The site (SLAC or JLAB).
- */
- private DatasetSite site;
-
- /**
- * The root directory (e.g. should be 'HPS').
- */
- private String rootDir;
-
+
/**
* Create client with default parameters.
*/
- DatacatClientImpl() {
+ DatacatClientImpl() {
this(DatacatConstants.BASE_URL, DatasetSite.SLAC, DatacatConstants.ROOT_DIR);
}
-
+
/**
* Create client.
- *
- * @param baseUrl
- * @param site
- * @param rootDir
- */
- DatacatClientImpl(String url, DatasetSite site, String rootDir) {
+ *
+ * @param baseUrl the base URL of the data catalog application
+ * @param site the default site where physical files are located
+ * @param rootDir the root directory in the data catalog
+ */
+ DatacatClientImpl(final String url, final DatasetSite site, final String rootDir) {
try {
this.url = new URL(url);
- } catch (MalformedURLException e) {
+ } catch (final MalformedURLException e) {
throw new IllegalArgumentException("The URL is bad.", e);
}
if (site == null) {
@@ -71,147 +71,227 @@
throw new IllegalArgumentException("The root dir argument is null.");
}
this.rootDir = rootDir;
- LOGGER.config("url: " + url);
- LOGGER.config("site: " + site);
- LOGGER.config("rootDir: " + rootDir);
- }
-
- /**
- * Remove a folder from the catalog.
- * <p>
- * It must be empty or an error will occur.
- *
- * @param folder the folder path
- * @return the HHTP status code from the request
- */
- @Override
- public int removeFolder(String folder) {
- String fullUrl = url.toString() + "/r/folders.json/" + this.rootDir + folder;
- LOGGER.info("removing folder: " + fullUrl);
- return HttpUtilities.doDelete(fullUrl);
+ LOGGER.config("url: " + url + "; site: " + site + "rootDir: " + rootDir);
+ }
+
+ /**
+ * Add a dataset to the data catalog.
+ *
+ * @param folder the logical folder in the datacat, which must already exist
+ * @param dataType the data type
+ * @param resource the resource (path on the file system)
+ * @param site the site of the file
+ * @param fileFormat the file format
+ * @param name the name of the dataset
+ * @param metadata metadata to assign to the dataset
+ * @return the HTTP status code from the request
+ */
+ @Override
+ public int addDataset(final String folder, final DatasetDataType dataType, final String resource,
+ final DatasetSite site, final DatasetFileFormat fileFormat, final String name,
+ final Map<String, Object> metadata) {
+ final Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put("dataType", dataType.toString());
+ parameters.put("resource", resource);
+ parameters.put("site", DatasetSite.SLAC.name());
+ parameters.put("fileFormat", fileFormat.toString());
+ parameters.put("name", name);
+ final JSONObject jsonDataset = JSONUtilities.createJSONDataset(parameters, metadata);
+ final String urlLocation = url + "/r/datasets.json/" + this.rootDir + "/" + folder;
+ LOGGER.info("addDataset: " + urlLocation);
+ LOGGER.info("dataset JSON: " + jsonDataset.toString());
+ return HttpUtilities.doPost(urlLocation, jsonDataset.toString());
+ }
+
+ /**
+ * Add metadata to an existing dataset.
+ *
+ * @param folder the folder
+ * @param datasetName the name of the dataset
+ * @param metaData the map of metadata where values can be <code>String</code>, <code>Integer</code> or
+ * <code>Float</code>
+ * @return the HTTP status code from the request
+ */
+ @Override
+ public int addMetadata(final String folder, final String name, final Map<String, Object> metaData) {
+ final JSONObject object = new JSONObject();
+ object.put("versionMetadata", JSONUtilities.createJSONMetadataArray(metaData));
+ final String patchUrl = this.url.toString() + "/r/datasets.json/" + this.rootDir + "/" + folder + "/" + name
+ + ";v=current;s=" + this.site;
+ LOGGER.info("addMetadata: " + patchUrl);
+ return HttpUtilities.doPatch(patchUrl, object.toString());
}
/**
* Delete a dataset from the catalog.
* <p>
- * This has no affect on the underlying resource (file).
- *
+ * This has no affect on the underlying resource (file on disk).
+ *
* @param path the path of the dataset
* @return the HTTP status code from the reqest
*/
@Override
- public int deleteDataset(String path) {
- String urlLocation = url.toString() + "/r/datasets.json/" + this.rootDir + path;
+ public int deleteDataset(final String path) {
+ final String urlLocation = url.toString() + "/r/datasets.json/" + this.rootDir + path;
LOGGER.info("deleting dataset: " + urlLocation);
return HttpUtilities.doDelete(urlLocation);
}
-
+
+ /**
+ * Return <code>true</code> if the path exists.
+ *
+ * @param path the path in the data catalog
+ */
+ @Override
+ public boolean exists(final String path) {
+ if (path == null) {
+ throw new IllegalArgumentException("The path is null.");
+ }
+ if (path.length() == 0) {
+ throw new IllegalArgumentException("The path is a blank string.");
+ }
+ final String urlLocation = this.url + "/r/path.json/" + this.rootDir + "/" + path;
+ final StringBuffer output = new StringBuffer();
+ final int status = HttpUtilities.doGet(urlLocation, output);
+ if (status > 400) {
+ throw new RuntimeException("HTTP GET returned error status: " + status);
+ }
+ final JSONObject jsonObject = new JSONObject(output.toString());
+ return jsonObject.has("_type");
+ }
+
+ /**
+ * Find datasets in the catalog.
+ * <p>
+ * See <a href="http://docs.datacatalog.apiary.io/#search">Search Doc</a> for more details
+ * on search syntax.
+ *
+ * @param folder the folder path
+ * @param query the query statement to execute
+ * @return the HTTP status code from the request
+ */
+ @Override
+ public List<Dataset> findDatasets(final String directory, final String query, final Set<String> showMetadata) {
+
+ String urlLocation = this.url.toString() + "/r/search.json/" + this.rootDir + "/";
+ if (directory != null) {
+ urlLocation += directory;
+ }
+ urlLocation += ";s=" + this.site.name() + "?";
+
+ // Encode query string so it will be a valid URL.
+ if (query != null) {
+ String encoded = null;
+ try {
+ encoded = URLEncoder.encode(query, "UTF-8");
+ } catch (final UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ urlLocation += "filter=" + encoded;
+ }
+
+ // Metadata fields to show.
+ if (showMetadata != null) {
+ for (final String metadataField : showMetadata) {
+ urlLocation += "&show=" + metadataField;
+ }
+ }
+
+ LOGGER.info("findDatasets: " + urlLocation);
+ final StringBuffer outputBuffer = new StringBuffer();
+ final int response = HttpUtilities.doGet(urlLocation, outputBuffer);
+ if (response >= 400) {
+ throw new RuntimeException("HTTP GET returned error code: " + response);
+ }
+
+ // Build and return dataset list
+ final JSONObject searchResults = new JSONObject(outputBuffer.toString());
+ LOGGER.info("returning search results: " + searchResults.toString());
+ return DatasetUtilities.getDatasetsFromSearch(searchResults);
+ }
+
+ /**
+ * Get a dataset from its path.
+ *
+ * @param path the path in the data catalog
+ * @return the dataset
+ */
+ @Override
+ public Dataset getDataSet(final String path, final DatasetSite site) {
+ if (path == null) {
+ throw new IllegalArgumentException("The path is null.");
+ }
+ if (path.length() == 0) {
+ throw new IllegalArgumentException("The path is a blank string.");
+ }
+ if (site == null) {
+ throw new IllegalArgumentException("The site is null.");
+ }
+ String urlLocation = this.url + "/r/path.json/" + this.rootDir;
+ if (!path.startsWith("/")) {
+ urlLocation += "/";
+ }
+ urlLocation += path + ";s=" + site.name();
+ final StringBuffer output = new StringBuffer();
+ HttpUtilities.doGet(urlLocation, output);
+ return new DatasetImpl(new JSONObject(output.toString()));
+ }
+
+ /**
+ * Return <code>true</code> if path is a folder in the data catalog.
+ *
+ * @return <code>true</code> if path is a folder in the data catalog
+ */
+ @Override
+ public boolean isFolder(final String path) {
+ if (path == null) {
+ throw new IllegalArgumentException("The path is null.");
+ }
+ if (path.length() == 0) {
+ throw new IllegalArgumentException("The path is a blank string.");
+ }
+ final String urlLocation = this.url + "/r/path.json/" + this.rootDir + "/" + path;
+ final StringBuffer output = new StringBuffer();
+ final int status = HttpUtilities.doGet(urlLocation, output);
+ if (status > 400) {
+ throw new RuntimeException("HTTP GET query returned error status: " + status);
+ }
+ final JSONObject jsonObject = new JSONObject(output.toString());
+ return jsonObject.has("_type") ? "folder".equals(jsonObject.getString("_type")) : false;
+ }
+
/**
* Create a folder in the data catalog.
- *
+ *
* @param folder the folder's path
* @return the HTTP status code from the request
*/
@Override
- public int makeFolder(String path) {
- Map<String, Object> parameters = new HashMap<String, Object>();
+ public int makeFolder(final String path) {
+ final Map<String, Object> parameters = new HashMap<String, Object>();
parameters.put("path", "/" + DatacatConstants.ROOT_DIR + "/" + path);
- String name = new File(path).getName();
+ final String name = new File(path).getName();
parameters.put("name", name);
parameters.put("_type", "folder");
- JSONObject object = JSONUtilities.createJSONFromMap(parameters);
- String urlLocation = url + "/r/folders.json/" + this.rootDir;
+ final JSONObject object = JSONUtilities.createJSONFromMap(parameters);
+ final String urlLocation = url + "/r/folders.json/" + this.rootDir;
LOGGER.info("making folder: " + urlLocation);
LOGGER.info("folder JSON: " + object.toString());
return HttpUtilities.doPost(urlLocation, object.toString());
}
-
- /**
- * Add metadata to an existing dataset.
- *
- * @param folder the folder
- * @param datasetName the name of the dataset
- * @param metaData the map of metadata where values can be <code>String</code>, <code>Integer</code> or <code>Float</code>
- * @return the HTTP status code from the request
- */
- @Override
- public int addMetadata(String folder, String name, Map<String, Object> metaData) {
- JSONObject object = new JSONObject();
- object.put("versionMetadata", JSONUtilities.createJSONMetadataArray(metaData));
- String patchUrl = this.url.toString() + "/r/datasets.json/" + this.rootDir + "/" + folder + "/" + name + ";v=current;s=" + this.site;
- LOGGER.info("addMetadata: " + patchUrl);
- return HttpUtilities.doPatch(patchUrl, object.toString());
- }
-
- /**
- * Find datasets in the catalog.
+
+ /**
+ * Remove a folder from the catalog.
* <p>
- * See <a href="http://docs.datacatalog.apiary.io/#search">Search Doc</a> for more details.
- *
+ * It must be empty or an error will occur.
+ *
* @param folder the folder path
- * @param query the query to execute
- * @return the HTTP status code from the request
- */
- @Override
- public List<Dataset> findDatasets(String directory, String query, Set<String> showMetadata) {
-
- String urlLocation = this.url.toString() + "/r/search.json/" + this.rootDir + "/";
- if (directory != null) {
- urlLocation += directory;
- }
- urlLocation += ";s=" + this.site.name() + "?";
- if (query != null) {
- String encoded = null;
- try {
- encoded = URLEncoder.encode(query, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- urlLocation += "filter=" + encoded;
- }
- if (showMetadata != null) {
- for (String metadataField : showMetadata) {
- urlLocation += "&show=" + metadataField;
- }
- }
- LOGGER.info("findDatasets: " + urlLocation);
- StringBuffer outputBuffer = new StringBuffer();
- int response = HttpUtilities.doGet(urlLocation, outputBuffer);
- if (response >= 400) {
- throw new RuntimeException("HTTP GET failed with code: " + response);
- }
-
- // Build and return dataset list
- JSONObject searchResults = new JSONObject(outputBuffer.toString());
- LOGGER.info("returning search results: " + searchResults.toString());
- return DatasetUtilities.getDatasetsFromSearch(searchResults);
- }
-
- /**
- * Add a dataset to the data catalog.
- *
- * @param folder the folder which must already exist
- * @param dataType the data type
- * @param resource the resource (path)
- * @param site the site of the file
- * @param fileFormat the file format
- * @param name the name of the dataset
- * @param metadata metadata to assign to the dataset
- * @return the HTTP status code from the request
- */
- @Override
- public int addDataset(String folder, DatasetDataType dataType, String resource, DatasetSite site,
- DatasetFileFormat fileFormat, String name, Map<String, Object> metadata) {
- Map<String, Object> parameters = new HashMap<String, Object>();
- parameters.put("dataType", dataType.toString());
- parameters.put("resource", resource);
- parameters.put("site", DatasetSite.SLAC.name());
- parameters.put("fileFormat", fileFormat.toString());
- parameters.put("name", name);
- JSONObject jsonDataset = JSONUtilities.createJSONDataset(parameters, metadata);
- String urlLocation = url + "/r/datasets.json/" + this.rootDir + "/" + folder;
- LOGGER.info("addDataset: " + urlLocation);
- LOGGER.info("dataset JSON: " + jsonDataset.toString());
- return HttpUtilities.doPost(urlLocation, jsonDataset.toString());
+ * @return the HHTP status code from the request
+ */
+ @Override
+ public int removeFolder(final String folder) {
+ final String fullUrl = url.toString() + "/r/folders.json/" + this.rootDir + folder;
+ LOGGER.info("removing folder: " + fullUrl);
+ return HttpUtilities.doDelete(fullUrl);
}
}
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatConstants.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatConstants.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatacatConstants.java Wed Sep 16 17:31:20 2015
@@ -15,6 +15,6 @@
/**
* The base URL of the datacat server.
*/
- // FIXME: This needs to be more easily configurable and not hard-coded.
+ // FIXME: Development server location.
public static final String BASE_URL = "http://localhost:8080/datacat-v0.4-SNAPSHOT";
}
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/Dataset.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/Dataset.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/Dataset.java Wed Sep 16 17:31:20 2015
@@ -4,9 +4,10 @@
import java.util.List;
/**
+ * Representation of a dataset in the data catalog.
+ *
*
* @author Jeremy McCormick, SLAC
- *
*/
public interface Dataset {
@@ -18,7 +19,7 @@
String getName();
/**
- * Get the path of the dataset e.g. "/HPS/folder/dataset01".
+ * Get the logical path of the dataset e.g. "/HPS/folder/dataset01".
*
* @return the path of the dataset
*/
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetDataType.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetDataType.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetDataType.java Wed Sep 16 17:31:20 2015
@@ -1,14 +1,29 @@
package org.hps.datacat.client;
/**
+ * Dataset types for HPS.
*
* @author Jeremy McCormick, SLAC
- *
*/
public enum DatasetDataType {
+ /**
+ * Data quality management plots.
+ */
DQM,
+ /**
+ * Raw data (EVIO).
+ */
RAW,
+ /**
+ * Reconstructed data (usually LCIO).
+ */
RECON,
+ /**
+ * Digital Summary Tape files (ROOT).
+ */
DST,
+ /**
+ * Test type (don't use in production).
+ */
TEST;
}
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetFileFormat.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetFileFormat.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetFileFormat.java Wed Sep 16 17:31:20 2015
@@ -1,14 +1,61 @@
package org.hps.datacat.client;
+
/**
+ * Dataset file formats for HPS.
*
* @author Jeremy McCormick, SLAC
- *
*/
public enum DatasetFileFormat {
- EVIO,
- LCIO,
- ROOT,
- AIDA,
- TEST;
+
+ /**
+ * EVIO data format.
+ */
+ EVIO(),
+ /**
+ * LCIO data format (note custom file extension).
+ */
+ LCIO("slcio"),
+ /**
+ * ROOT files.
+ */
+ ROOT(),
+ /**
+ * AIDA files.
+ */
+ AIDA(),
+ /**
+ * Testing only (do not use in production).
+ */
+ TEST(null);
+
+ /**
+ * The file extension of the format.
+ */
+ private String extension;
+
+ /**
+ * Create a file format with an extension.
+ *
+ * @param extension the file's extension
+ */
+ private DatasetFileFormat(String extension) {
+ this.extension = extension;
+ }
+
+ /**
+ * Create a file format with default extension (lower case of enum name).
+ */
+ private DatasetFileFormat() {
+ this.extension = this.name().toLowerCase();
+ }
+
+ /**
+ * Get the format's file extension.
+ *
+ * @return the format file extension
+ */
+ public String extension() {
+ return extension;
+ }
}
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetImpl.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetImpl.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetImpl.java Wed Sep 16 17:31:20 2015
@@ -11,27 +11,106 @@
import org.json.JSONObject;
/**
+ * Implementation of the {@link Dataset} interface.
*
* @author Jeremy McCormick, SLAC
- *
*/
final class DatasetImpl implements Dataset {
+ /**
+ * The name of the dataset.
+ */
private String name;
+
+ /**
+ * The path in the datacatalog which is folder + name.
+ */
private String path;
+
+ /**
+ * The data type of the file.
+ */
private DatasetDataType dataType;
+
+ /**
+ * The format of the file.
+ */
private DatasetFileFormat fileFormat;
+
+ /**
+ * The list of file locations.
+ */
private List<DatasetLocation> locations = new ArrayList<DatasetLocation>();
+
+ /**
+ * The creation date.
+ */
private Date created;
+
+ /**
+ * The dataset's metadata.
+ */
private DatasetMetadata metadata;
+ /**
+ * Parser for reading in dates from JSON.
+ */
private static final SimpleDateFormat DATE_PARSER = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+ /**
+ * Create a new dataset from JSON.
+ *
+ * @param jsonObject the JSON data
+ */
DatasetImpl(JSONObject jsonObject) {
- parse(jsonObject);
- }
+ if (!jsonObject.has("_type")) {
+ throw new IllegalArgumentException("JSON object is missing _type field.");
+ }
+ if (!jsonObject.getString("_type").startsWith("dataset")) {
+ throw new IllegalArgumentException("JSON _type field is not a dataset: " + jsonObject.getString("_type"));
+ }
- private void parse(JSONObject jsonObject) {
+ if (jsonObject.getString("_type").equals("dataset#flat")) {
+ parseFlat(jsonObject);
+ } else if (jsonObject.getString("_type").equals("dataset#full")) {
+ parseFull(jsonObject);
+ } else {
+ throw new IllegalArgumentException("Unknown dataset type: " + jsonObject.getString("_type"));
+ }
+ }
+
+ /**
+ * Parse the flat JSON representation.
+ *
+ * @param jsonObject the JSON object
+ */
+ private void parseFlat(JSONObject jsonObject) {
+ name = jsonObject.getString("name");
+ path = jsonObject.getString("path");
+ DatasetLocationImpl location = new DatasetLocationImpl(
+ DatasetSite.valueOf(jsonObject.getString("site")),
+ jsonObject.getString("resource"),
+ jsonObject.getInt("size"),
+ ScanStatus.valueOf(jsonObject.getString("scanStatus")));
+ location.setEventCount(jsonObject.getInt("eventCount"));
+ location.setRunMin(jsonObject.getInt("runMin"));
+ location.setRunMax(jsonObject.getInt("runMax"));
+ locations.add(location);
+ dataType = DatasetDataType.valueOf(jsonObject.getString("dataType"));
+ fileFormat = DatasetFileFormat.valueOf(jsonObject.getString("fileFormat"));
+ try {
+ created = DATE_PARSER.parse(jsonObject.getString("created"));
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Bad created value: " + jsonObject.getString("created"), e);
+ }
+ }
+
+ /**
+ * Parse the full JSON representation.
+ *
+ * @param jsonObject the JSON object
+ */
+ private void parseFull(JSONObject jsonObject) {
if (!jsonObject.getString("_type").equals("dataset#full")) {
throw new IllegalArgumentException("Wrong _type in JSON data: " + jsonObject.getString("_type"));
}
@@ -53,41 +132,81 @@
}
}
+ /**
+ * Get the name of the dataset without the path component e.g. "dataset01".
+ *
+ * @return the name of the dataset
+ */
@Override
public String getName() {
return this.name;
}
+ /**
+ * Get the logical path of the dataset e.g. "/HPS/folder/dataset01".
+ *
+ * @return the path of the dataset
+ */
@Override
public String getPath() {
return this.path;
}
+ /**
+ * Get the dataset locations.
+ *
+ * @return the dataset locations
+ */
@Override
public List<DatasetLocation> getLocations() {
return Collections.unmodifiableList(this.locations);
}
+ /**
+ * Get the file format e.g. EVIO, LCIO, etc.
+ *
+ * @return the dataset file format
+ */
@Override
public DatasetFileFormat getFileFormat() {
return this.fileFormat;
}
+ /**
+ * Get the data type e.g. RAW, RECON, etc.
+ *
+ * @return the data type
+ */
@Override
public DatasetDataType getDataType() {
return this.dataType;
}
+ /**
+ * Get the creation date.
+ *
+ * @return the creation date
+ */
@Override
public Date getCreated() {
return this.created;
}
+ /**
+ * Get the dataset's metadata.
+ *
+ * @return the dataset's metadata
+ */
@Override
public DatasetMetadata getMetadata() {
return this.metadata;
}
+ /**
+ * Convert this object to a string.
+ *
+ * @return this object converted to a string
+ */
public String toString() {
return "Dataset { name: " + name + ", path:" + path + ", " + "dataType: " + dataType.name() + "fileFormat: " + fileFormat.name() + ", created: " + created + " }";
}
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetLocation.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetLocation.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetLocation.java Wed Sep 16 17:31:20 2015
@@ -1,23 +1,61 @@
package org.hps.datacat.client;
/**
+ * Representation of a dataset location in the data catalog.
*
* @author Jeremy McCormick, SLAC
- *
*/
public interface DatasetLocation {
+ /**
+ * Get the site of the dataset (JLAB or SLAC).
+ *
+ * @return the dataset site
+ */
DatasetSite getSite();
+ /**
+ * Get the resource of the dataset location (file system path).
+ *
+ * @return the resource of the dataset location
+ */
String getResource();
+ /**
+ * Get the scan status of the dataset location.
+ *
+ * @return the scan status
+ */
ScanStatus getScanStatus();
+ /**
+ * The size of the file in bytes.
+ *
+ * @return the size of the file in bytes
+ */
long getSize();
+ /**
+ * Get the minimum run number.
+ *
+ * @return the minimum run number
+ */
+ // FIXME: Belongs in dataset metadata.
int getRunMin();
+ /**
+ * Get the maximum run number.
+ *
+ * @return the maximum run number
+ */
+ // FIXME: Belongs in dataset metadata.
int getRunMax();
+ /**
+ * Get the event count.
+ *
+ * @return the event count
+ */
+ // FIXME: Belongs in dataset metadata.
int getEventCount();
}
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetLocationImpl.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetLocationImpl.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetLocationImpl.java Wed Sep 16 17:31:20 2015
@@ -3,23 +3,76 @@
import org.json.JSONObject;
/**
+ * Implementation of {@link DatasetLocation} interface.
*
* @author Jeremy McCormick, SLAC
*/
final class DatasetLocationImpl implements DatasetLocation {
+ /**
+ * The dataset's site.
+ */
private DatasetSite site;
+
+ /**
+ * The resource on the file system.
+ */
private String resource;
+
+ /**
+ * The scan status.
+ */
private ScanStatus scanStatus = ScanStatus.UNKNOWN;
+
+ /**
+ * The size of the file in bytes.
+ */
private long size;
+
+ /**
+ * The minimum run number.
+ */
private int runMin;
+
+ /**
+ * The maximum run number.
+ */
private int runMax;
+
+ /**
+ * The event count.
+ */
private int eventCount;
+ /**
+ * Create a dataset location.
+ *
+ * @param site the site of the dataset location
+ * @param resource the source on disk
+ * @param size the size of the file
+ * @param scanStatus the scan status
+ */
+ DatasetLocationImpl(DatasetSite site, String resource, long size, ScanStatus scanStatus) {
+ this.site = site;
+ this.resource = resource;
+ this.scanStatus = scanStatus;
+ this.size = size;
+ }
+
+ /**
+ * Create a dataset location from JSON.
+ *
+ * @param jsonObject the JSON object
+ */
DatasetLocationImpl(JSONObject jsonObject) {
parse(jsonObject);
}
+ /**
+ * Parse JSON data into this object.
+ *
+ * @param jsonObject the JSON data
+ */
private void parse(JSONObject jsonObject) {
if (!jsonObject.getString("_type").equals("location")) {
throw new IllegalArgumentException("Wrong _type in JSON data: " + jsonObject.getString("_type"));
@@ -35,39 +88,100 @@
this.eventCount = jsonObject.getInt("eventCount");
}
+ /**
+ * Get the site of the dataset (JLAB or SLAC).
+ *
+ * @return the dataset site
+ */
@Override
public DatasetSite getSite() {
return this.site;
}
+ /**
+ * Get the resource of the dataset location (file system path).
+ *
+ * @return the resource of the dataset location
+ */
@Override
public String getResource() {
return resource;
}
+ /**
+ * Get the scan status of the dataset location.
+ *
+ * @return the scan status
+ */
@Override
public ScanStatus getScanStatus() {
return scanStatus;
}
+ /**
+ * The size of the file in bytes.
+ *
+ * @return the size of the file in bytes
+ */
@Override
public long getSize() {
return this.size;
}
+ /**
+ * Get the minimum run number.
+ *
+ * @return the minimum run number
+ */
@Override
public int getRunMin() {
return this.runMin;
}
+ /**
+ * Get the maximum run number.
+ *
+ * @return the maximum run number
+ */
@Override
public int getRunMax() {
return this.runMax;
}
-
+
+ /**
+ * Get the event count.
+ *
+ * @return the event count
+ */
@Override
public int getEventCount() {
return this.eventCount;
}
-
+
+ /**
+ * Set the minimum run number.
+ *
+ * @param runMin the minimum run number
+ */
+ void setRunMin(int runMin) {
+ this.runMin = runMin;
+ }
+
+ /**
+ * Set the maximum run number.
+ *
+ * @param runMax the maximum run number
+ */
+ void setRunMax(int runMax) {
+ this.runMax = runMax;
+ }
+
+ /**
+ * Set the event count.
+ *
+ * @param eventCount the event count
+ */
+ void setEventCount(int eventCount) {
+ this.eventCount = eventCount;
+ }
}
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetMetadata.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetMetadata.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetMetadata.java Wed Sep 16 17:31:20 2015
@@ -1,8 +1,7 @@
package org.hps.datacat.client;
-
/**
- * Dataset metadata which is keys and values that are double, integer or string.
+ * Dataset metadata which is string keys and values that are double, integer or string.
*
* @author Jeremy McCormick, SLAC
*/
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetSite.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetSite.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetSite.java Wed Sep 16 17:31:20 2015
@@ -1,9 +1,9 @@
package org.hps.datacat.client;
/**
+ * Site of a dataset (SLAC or JLAB).
*
* @author Jeremy McCormick, SLAC
- *
*/
public enum DatasetSite {
/**
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetUtilities.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetUtilities.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/DatasetUtilities.java Wed Sep 16 17:31:20 2015
@@ -7,7 +7,7 @@
import org.json.JSONObject;
/**
- * Dataset utilities.
+ * Dataset utilities for the crawler.
*
* @author Jeremy McCormick, SLAC
*/
Modified: java/trunk/datacat-client/src/main/java/org/hps/datacat/client/HttpUtilities.java
=============================================================================
--- java/trunk/datacat-client/src/main/java/org/hps/datacat/client/HttpUtilities.java (original)
+++ java/trunk/datacat-client/src/main/java/org/hps/datacat/client/HttpUtilities.java Wed Sep 16 17:31:20 2015
@@ -4,7 +4,6 @@
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -19,50 +18,39 @@
import org.apache.http.util.EntityUtils;
/**
- *
* @author Jeremy McCormick, SLAC
*/
final class HttpUtilities {
/**
- * Do an HTTP POST.
- *
+ * Do an HTTP DELETE.
+ *
* @param urlLocation the URL location
- * @param data the data to stream to the server
* @return the HTTP response code
*/
- static int doPost(String urlLocation, String data) {
+ static int doDelete(final String urlLocation) {
int responseCode = 0;
try {
- URL url = new URL(urlLocation);
- HttpURLConnection connection = (HttpURLConnection) url.openConnection();
- connection.setRequestMethod("POST");
- connection.setRequestProperty("Content-Type", "application/json");
+ final URL url = new URL(urlLocation);
+ final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true);
- if (data != null) {
- OutputStreamWriter out = new OutputStreamWriter(connection.getOutputStream());
- out.write(data);
- out.close();
- }
- System.out.println("url: " + urlLocation);
- System.out.println("data: " + data);
- System.out.println("response: " + connection.getResponseCode());
- System.out.println("message: " + connection.getResponseMessage());
+ connection.setRequestMethod("DELETE");
+ connection.connect();
responseCode = connection.getResponseCode();
- } catch (IOException e) {
+ } catch (final IOException e) {
throw new RuntimeException(e);
}
return responseCode;
}
-
+
/**
* Do an HTTP get and return the output from the server in a <code>StringBuffer</code>.
- *
+ *
* @param urlLocation the URL location
* @param stringBuffer the string buffer with the server output
* @return the HTTP response
*/
- static int doGet(String urlLocation, StringBuffer stringBuffer) {
+ static int doGet(final String urlLocation, final StringBuffer stringBuffer) {
HttpURLConnection connection = null;
int response = 0;
try {
@@ -73,94 +61,83 @@
connection.setDoInput(true);
connection.connect();
if (stringBuffer != null) {
- String output = IOUtils.toString(connection.getInputStream(), "UTF-8");
+ final String output = IOUtils.toString(connection.getInputStream(), "UTF-8");
stringBuffer.append(output);
}
response = connection.getResponseCode();
- } catch (IOException e) {
+ } catch (final IOException e) {
throw new RuntimeException(e);
} finally {
connection.disconnect();
}
return response;
- }
-
+ }
+
/**
* Do an HTTP patch.
- *
+ *
* @param urlLocation the URL location
* @param data the data to stream to the server
* @return the HTTP response code
*/
- static int doPatch(String urlLocation, String data) {
- int responseCode = 0;
- CloseableHttpClient httpClient = HttpClients.createDefault();
+ static int doPatch(final String urlLocation, final String data) {
+ int responseCode = 0;
+ final CloseableHttpClient httpClient = HttpClients.createDefault();
HttpPatch httpPatch = null;
try {
- httpPatch = new HttpPatch(new URI(urlLocation));
- InputStreamEntity entity =
- new InputStreamEntity(
- new ByteArrayInputStream(
- data.getBytes("UTF-8")),
- -1,
- ContentType.APPLICATION_JSON);
- httpPatch.setEntity(entity);
- CloseableHttpResponse response = httpClient.execute(httpPatch);
+ httpPatch = new HttpPatch(new URI(urlLocation));
+ final InputStreamEntity entity = new InputStreamEntity(new ByteArrayInputStream(data.getBytes("UTF-8")),
+ -1, ContentType.APPLICATION_JSON);
+ httpPatch.setEntity(entity);
+ final CloseableHttpResponse response = httpClient.execute(httpPatch);
try {
EntityUtils.consume(response.getEntity());
} finally {
response.close();
}
responseCode = response.getStatusLine().getStatusCode();
- } catch (URISyntaxException e) {
+ } catch (final URISyntaxException e) {
throw new IllegalArgumentException(e);
- } catch(IOException e) {
+ } catch (final IOException e) {
throw new RuntimeException(e);
} finally {
try {
httpClient.close();
- } catch (IOException e) {
+ } catch (final IOException e) {
throw new RuntimeException(e);
}
- }
+ }
return responseCode;
- }
-
+ }
+
/**
- * Do an HTTP DELETE.
- *
+ * Do an HTTP POST.
+ *
* @param urlLocation the URL location
+ * @param data the data to stream to the server
* @return the HTTP response code
*/
- static int doDelete(String urlLocation) {
+ static int doPost(final String urlLocation, final String data) {
int responseCode = 0;
try {
- URL url = new URL(urlLocation);
- HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ final URL url = new URL(urlLocation);
+ final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
connection.setDoOutput(true);
- connection.setRequestMethod("DELETE");
- connection.connect();
+ if (data != null) {
+ final OutputStreamWriter out = new OutputStreamWriter(connection.getOutputStream());
+ out.write(data);
+ out.close();
+ }
+ System.out.println("url: " + urlLocation);
+ System.out.println("data: " + data);
+ System.out.println("response: " + connection.getResponseCode());
+ System.out.println("message: " + connection.getResponseMessage());
responseCode = connection.getResponseCode();
- } catch (IOException e) {
+ } catch (final IOException e) {
throw new RuntimeException(e);
}
return responseCode;
- }
-
- /*
- static URL createURL(String... chunks) {
- if (chunks.length == 0) {
- throw new IllegalArgumentException("No arguments provided.");
- }
- String urlString = "";
- for (String chunk : chunks) {
- urlString += chunk;
- }
- try {
- return new URL(urlString);
- } catch (MalformedURLException e) {
- throw new IllegalArgumentException("Bad URL string: " + urlString);
- }
- }
- */
+ }
}
Modified: java/trunk/datacat-client/src/test/java/org/hps/datacat/client/DatacatClientTest.java
=============================================================================
--- java/trunk/datacat-client/src/test/java/org/hps/datacat/client/DatacatClientTest.java (original)
+++ java/trunk/datacat-client/src/test/java/org/hps/datacat/client/DatacatClientTest.java Wed Sep 16 17:31:20 2015
@@ -9,68 +9,68 @@
import junit.framework.TestCase;
/**
- *
* @author Jeremy McCormick, SLAC
- *
*/
public class DatacatClientTest extends TestCase {
-
+
private static final String DATASET_NAME = "dummyDataset";
private static final String FOLDER = "dummyFolder";
private static final String RESOURCE = "/path/to/dummyDataset.ds";
-
+
public void testDatacat() throws Exception {
// Datacat client with default parameters.
- DatacatClient client = new DatacatClientImpl();
-
+ final DatacatClient client = new DatacatClientImpl();
+
// Stores response from HTTP operations
int response = -1;
-
+
// Create dummy folder.
response = client.makeFolder("dummyFolder");
-
+
+ // TODO: check that folder exists
+ assertTrue(client.isFolder("dummyFolder"));
+
// Add dummy dataset.
- Map<String, Object> dsMetadata = new HashMap<String, Object>();
+ final Map<String, Object> dsMetadata = new HashMap<String, Object>();
dsMetadata.put("testInt", 1);
dsMetadata.put("testFloat", 1.1f);
dsMetadata.put("testDouble", 1.2d);
dsMetadata.put("testString", "herpderp");
- response = client.addDataset(FOLDER, DatasetDataType.TEST, RESOURCE, DatasetSite.SLAC, DatasetFileFormat.TEST, DATASET_NAME, dsMetadata);
-
+ response = client.addDataset(FOLDER, DatasetDataType.TEST, RESOURCE, DatasetSite.SLAC, DatasetFileFormat.TEST,
+ DATASET_NAME, dsMetadata);
+
// Patch the dataset with some meta data.
- Map<String, Object> metaData = new HashMap<String, Object>();
+ final Map<String, Object> metaData = new HashMap<String, Object>();
metaData.put("testInt2", 1234);
response = client.addMetadata(FOLDER, DATASET_NAME, metaData);
-
- // TODO: check that folder exists
-
- // TODO: check that dataset exists
-
+
// TODO: get the full folder info
-
- // TODO: get the full dataset info
-
+
+ // Get the dataset info back.
+ final Dataset gimmeDataset = client.getDataSet(FOLDER + "/" + DATASET_NAME, DatasetSite.SLAC);
+ System.out.println("got dataset " + gimmeDataset.getName());
+
// Find the dataset with a simple query.
- Set<String> metadataFields = new HashSet<String>();
+ final Set<String> metadataFields = new HashSet<String>();
metadataFields.add("testInt");
- metadataFields.add("testFloat");
+ metadataFields.add("testFloat");
metadataFields.add("testDouble");
metadataFields.add("testString");
- List<Dataset> datasets = client.findDatasets(FOLDER, "testInt == 1", metadataFields);
- for (Dataset dataset : datasets) {
+ final List<Dataset> datasets = client.findDatasets(FOLDER, "testInt == 1", metadataFields);
+ for (final Dataset dataset : datasets) {
System.out.println("found dataset: " + dataset.getName());
System.out.println("metadata: " + dataset.getMetadata());
}
-
+
// Delete the dataset.
response = client.deleteDataset("/" + FOLDER + "/" + DATASET_NAME);
System.out.println("deleteDataset: " + response);
System.out.println();
-
+
// Remove the folder.
client.removeFolder("/" + FOLDER);
System.out.println("removeFolder: " + response);
System.out.println();
- }
+ }
}
