Merge pull request #181 from intel-cloud/0.4.0.0

merge commits from 0.4.0.0

Author: Yaguang Wang

CHANGELOG

@@ -1,3 +1,37 @@
+  COSBench 0.4.0.b2 (14-ww17.1)
+------------------------------------
+. #57: time synchronization
+. #96: Creating Web UI facility for generating different workload configuration files
+. #155: Add support for "acceptable failure rate, failure %" to all stages
+. #4: xml validator
+. #77: contextual parameter list
+. #167: storage policy supporting for openstack swift
+. #37: support CDMI interface
+. #139: NullPointerException when a mission is aborted
+
+
+  COSBench 0.4.0.b1 (14-ww11.4)
+------------------------------------
+. #166: show driver alive state on controller index page
+. #158: Extend "histogram" selector to support "open ranges"
+. #165: enable Open range for object size in uniform and histogram
+. #136: 1 GB Object Reads Fail due to IllegalArgumentException
+. #99 (#152): measure and report the time for first and last byte in or out
+. #159: Extend "histogram" selector to support "floating point" weights/percentages
+. #143: at loading archived workloads, even there are 100 workloads, the UI shows only 98.
+. #140: duplicated workload id generated 
+. #151: avoid being challenged for each request
+
+
+  COSBench 0.4.0.a1 (14-ww1.4)
+------------------------------------
+. #137: when there are large # of archive jobs, exception will raise from archive list on web portal.
+. #139: NullPointerException when a mission is aborted
+. support http BASIC and DIGEST authentication
+. add cdmi base and util bundle
+. add cdmi adapter for swift+cdmi middleware
+
+
   COSBench 0.3.3.0 (13-ww44.5)
 ------------------------------------
 . #20: multiple same operations
@@ -56,81 +90,4 @@ COSBench 0.3.0.1 (13-ww17.5)
 
   COSBench 0.3.0.0 (13-ww15.5)
 ------------------------------------
-1. rebase version to 0.3.0 for open source
-
-======================================================================
-
-  COSBench 2.2.0.GA (13-ww15.1)
-------------------------------------
-1. fix a bug related to socket connections
-2. add a console option "-Dcosbench.controller.history" to help set the max number of workloads cosbench will hold in RAM
-3. add a console option "-Dcosbench.driver.history" to help set the max number of missions cosbench will hold in RAM
-4. add a config parameter "retry" to help user define the number of retries cosbench will attempt should previous efforts fail
-5. improve log printing to help better debuging
-6. miscellanous minor bug fixing
-
-  COSBench 2.2.0.beta2 (13-ww08.2)
-------------------------------------
-1. fix a bug which will cause OOM after a few days' intensive testing.
-2. support ceph 
-3. support to forcily terminate load to avoid long tail due to high load.
-4. add one parameter "tenant_name" for keystone configuration.
-5. add retry logic to avoid one failed auth thread to terminate whole workload.
-6. change parameters: 
-	a) for swift, change parameter "url" to "auth_url"; 
-	b) for keystone, add parameter "tenant_name".
-	
-
-  COSBench 2.2.0.beta1 (12-ww48.5)
-------------------------------------
-1. internal code refactoring
-2. add one new bundle to wrap http related utilities and abstract Context class
-3. formalize adaptor interfaces
-4. provide one sample project for adaptor development
-5. provide one development guide for adaptor
-
-  COSBench 2.1.0.GA (12-ww44.5)
-------------------------------------
-1. enhance start/stop scripts to handle unexpected cases.
-2. add step by step installation manual in user guide document.
-3. 2.1 release preparation:
-	- generate branch for 2.1 release preparation and switch to work on the branch for dev/release folder.
-	- change version to 2.1.0.GA
-	- remove version information in "README"
-4. add storage line at dispose stage in ampli-config-sample.xml to support v2.5
-
-  COSBench 2.0.5.GA (12-ww43.4)
-------------------------------------
-1. support object integrity check
-2. finalize config editing page
-3. unify sample config files
-
-  COSBench 2.0.4.GA (12-ww42.3)
-------------------------------------
-1. fix one bug which causes queuing workload can't be cancelled.
-2. fix one bug which causes 100% read and 0% write operation mix can't work as expecteed.
-3. unify mission and task on web page
-4. draft version of config editing page
-
-  COSBench 2.0.2.GA (12-ww38.3)
-------------------------------------
-1. fix one bug in swift-config-sample.xml, which causes failure at dispose stage.
-2. fix one bug for amplistor which can't get policy id defined in storage.
-3. fix imbalanced work allocation between driver.
-4. fix one bug which causes http connections not released after testing.
-
-
-  COSBench 2.0.1.GA (12-ww34.5)
-------------------------------------
-1. enhance the content based on internal and external users� feedback.
-2. make policy parameter optional for object related operation
-3. support timeout for amplistor
-4. fix a bug on "reset" button at matrix page
-
-  COSBench 2.0.0.GA (12-ww31.5)
-------------------------------------
-1. each workload stage now has a rampdown phase.
-2. support latency histogram.
-3. cosbench now has its own logging system.
-4. resolved serval issues related to performance metrics calculation.
-5. resolved serval issues related to HTTP client.
+1. rebase version to 0.3.0 for open source

COSBench-User-Guide.odt

@@ -41,7 +41,7 @@ public class AmpliClient {
 
     private HttpClient client = null;
     /* current operation */
-    private volatile HttpUriRequest request;
+    private volatile HttpUriRequest method;
     private int port;
     private String host;
     private String nsRoot;
@@ -54,20 +54,20 @@ public AmpliClient(HttpClient client, String host, int port, String nsRoot) {
     }
 
     public void dispose() {
-    	request = null;
+    	method = null;
         HttpClientUtil.disposeHttpClient(client);
     }
 
     public void abort() {
-        if (request != null)
-            request.abort();
-        request = null;
+        if (method != null)
+            method.abort();
+        method = null;
     }
 
     public boolean login() throws IOException, HttpException {
         String storageUrl = "http://" + this.host + ":" + this.port;
 
-        HttpHead method = HttpClientUtil.makeHttpHead(storageUrl);
+        method = HttpClientUtil.makeHttpHead(storageUrl);
         HttpResponse response = null;
         try {
             response = client.execute(method);
@@ -89,15 +89,14 @@ public String StoreObject(String sourceFilename, String ampliNamespace,
             AmpliException {
         File file = new File(sourceFilename);
 
-        HttpPut method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
                     + nsRoot;
             method = HttpClientUtil.makeHttpPut(storageUrl + "/" + HttpClientUtil.encodeURL(ampliNamespace)
                     + "/" + HttpClientUtil.encodeURL(ampliFilename));
 
-            method.setEntity(new FileEntity(file, "application/octet-stream"));
+            ((HttpPut)method).setEntity(new FileEntity(file, "application/octet-stream"));
 
             response = client.execute(method);
 
@@ -127,7 +126,6 @@ public String StoreObject(String sourceFilename, String ampliNamespace,
     public String StoreStreamedObject(InputStream stream, long length,
             String ampliNamespace, String ampliFilename) throws IOException,
             HttpException, AmpliException {
-        HttpPut method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -143,7 +141,7 @@ public String StoreStreamedObject(InputStream stream, long length,
                 entity.setChunked(false);
             }
 
-            method.setEntity(entity);
+            ((HttpPut)method).setEntity(entity);
 
             response = client.execute(method);
 
@@ -191,7 +189,6 @@ public String StoreObject(byte[] data, String ampliNamespace,
             String ampliFilename) throws IOException, HttpException,
             AmpliException {
         // int len = data.length;
-        HttpPut method = null;
         String storageUrl = "http://" + this.host + ":" + this.port + nsRoot;
         method = HttpClientUtil.makeHttpPut(storageUrl + "/" + HttpClientUtil.encodeURL(ampliNamespace) + "/"
                 + HttpClientUtil.encodeURL(ampliFilename));
@@ -200,7 +197,7 @@ public String StoreObject(byte[] data, String ampliNamespace,
 
         HttpResponse response = null;
         try {
-            method.setEntity(new ByteArrayEntity(data));
+            ((HttpPut)method).setEntity(new ByteArrayEntity(data));
             response = client.execute(method);
             if (response.getStatusLine().getStatusCode() == HttpStatus.SC_CREATED) {
                 return response.getFirstHeader("ETag").getValue();
@@ -222,7 +219,7 @@ public byte[] getObject(String namespace, String objName)
             throws IOException, HttpException, AmpliException {
         String storageUrl = "http://" + this.host + ":" + this.port + nsRoot;
 
-        HttpGet method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
+        method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
                 + "/" + HttpClientUtil.encodeURL(objName));
 
         HttpResponse response = null;
@@ -249,7 +246,7 @@ public InputStream getObjectAsStream(String namespace, String objName)
             throws IOException, HttpException, AmpliException {
         String storageUrl = "http://" + this.host + ":" + this.port + nsRoot;
 
-        HttpGet method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
+        method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
                 + "/" + HttpClientUtil.encodeURL(objName));
 
         HttpResponse response = null;
@@ -272,7 +269,6 @@ public InputStream getObjectAsStream(String namespace, String objName)
     public boolean deleteObject(String ampliNamespace, String name)
             throws HttpException, IOException, AmpliException {
 
-        HttpDelete method = null;
         HttpResponse response = null;
 
         try {
@@ -302,7 +298,6 @@ public boolean deleteObject(String ampliNamespace, String name)
     public AmpliPolicy createPolicy(AmpliPolicy policy) throws HttpException,
             IOException, AmpliException {
 
-        HttpPut method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -336,7 +331,6 @@ public AmpliPolicy createPolicy(AmpliPolicy policy) throws HttpException,
     public AmpliPolicy getPolicy(String policyId) throws HttpException,
             IOException, AmpliException {
 
-        HttpGet method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -366,13 +360,13 @@ public AmpliPolicy getPolicy(String policyId) throws HttpException,
 
     public AmpliNamespace createNamespace(AmpliNamespace namespace)
             throws HttpException, IOException, AmpliException {
-        HttpPut method = null;
-        HttpResponse response = null;
+
+    	HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
                     + nsRoot;
 
-            method = new HttpPut(storageUrl);
+            method = HttpClientUtil.makeHttpGet(storageUrl);
 
             method.setHeader("Content-Type", "text/plain");
 
@@ -405,7 +399,7 @@ public String createNamespace(String namespace, String policy_id)
             String storageUrl = "http://" + this.host + ":" + this.port
                     + nsRoot;
 
-            method = new HttpPut(storageUrl);
+            method = HttpClientUtil.makeHttpPut(storageUrl);
 
             method.setHeader("Content-Type", "text/plain");
 
@@ -438,7 +432,6 @@ public String createNamespace(String namespace, String policy_id)
     public AmpliNamespace getNamespace(String name) throws HttpException,
             IOException, AmpliException {
 
-        HttpGet method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -469,7 +462,6 @@ public AmpliNamespace getNamespace(String name) throws HttpException,
     public boolean isNamespaceExisted(String name) throws HttpException,
             IOException, AmpliException {
 
-        HttpHead method = null;
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -497,7 +489,7 @@ public boolean isNamespaceExisted(String name) throws HttpException,
 
     public boolean deleteNamespace(String namespace) throws HttpException,
             IOException, AmpliException {
-        HttpDelete method = null;
+
         HttpResponse response = null;
         try {
             String storageUrl = "http://" + this.host + ":" + this.port
@@ -525,7 +517,7 @@ public Map<String, String> getObjectMetadata(String namespace,
             String objName) throws IOException, HttpException, AmpliException {
         String storageUrl = "http://" + this.host + ":" + this.port + nsRoot;
 
-        HttpGet method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
+        method = HttpClientUtil.makeHttpGet(storageUrl + "/" + HttpClientUtil.encodeURL(namespace)
                 + "/" + HttpClientUtil.encodeURL(objName) + "?meta=http");
 
         HttpResponse response = null;

COSBenchUserGuide.pdf

@@ -8,20 +8,25 @@ Bundle-RequiredExecutionEnvironment: JavaSE-1.6
 Import-Package: com.intel.cosbench.api.auth,
  com.intel.cosbench.api.context,
  com.intel.cosbench.api.storage,
+ com.intel.cosbench.client.cdmi.util,
  com.intel.cosbench.client.http,
  com.intel.cosbench.config,
  com.intel.cosbench.log,
- com.intel.cosbench.client.cdmi.util,
  org.apache.commons.codec;version="[1.3.0,2.0.0)",
  org.apache.commons.codec.binary;version="1.3.0",
  org.apache.commons.codec.net;version="[1.3.0,2.0.0)",
  org.apache.commons.io.input;version="[1.3.0,2.0.0)",
  org.apache.commons.lang.math;version="2.5.0",
  org.apache.http;version="[4.1.4,5.0.0)",
+ org.apache.http.auth.params;version="[4.1.3,5.0.0)",
  org.apache.http.client;version="[4.1.3,5.0.0)",
  org.apache.http.client.methods;version="[4.1.3,5.0.0)",
+ org.apache.http.client.params;version="[4.1.3,5.0.0)",
+ org.apache.http.client.protocol;version="[4.1.3,5.0.0)",
  org.apache.http.conn;version="[4.1.3,5.0.0)",
  org.apache.http.entity;version="[4.1.4,5.0.0)",
+ org.apache.http.impl.client;version="[4.1.3,5.0.0)",
  org.apache.http.message;version="[4.1.4,5.0.0)",
  org.apache.http.params;version="[4.1.4,5.0.0)",
+ org.apache.http.protocol;version="[4.1.4,5.0.0)",
  org.apache.http.util;version="[4.1.4,5.0.0)"

dev/cosbench-ampli/src/com/intel/cosbench/client/amplistor/AmpliClient.java

@@ -23,31 +23,34 @@ public class CDMIStorage extends NoneStorage {
     // below parameters expect to get from configuration file.
     private int timeout;
     private String rootPath;
+    private String type; 
     private String headers;
     private boolean flag;
 
     // local variables
-    private CdmiClient client;
+    private BaseCdmiClient client;
     private String[] header_list;
 
     @Override
     public void init(Config config, Logger logger) {
         super.init(config, logger);
         initParms(config);
 
-        client = new CdmiClient(false);
+        client = CdmiClientFactory.getClient(type);
     }
 
     private void initParms(Config config) {
     	rootPath = config.get(ROOT_PATH_KEY, ROOT_PATH_DEFAULT);
         timeout = config.getInt(TIMEOUT_KEY, TIMEOUT_DEFAULT);
         headers = config.get(CUSTOM_HEADERS_KEY, CUSTOM_HEADERS_DEFAULT);
         flag = config.getBoolean(RAISE_DELETE_ERRORS_KEY, RAISE_DELETE_ERRORS_DEFAULT);
+        type = config.get(CDMI_CONTENT_TYPE_KEY, CDMI_CONTENT_TYPE_DEFAULT);
         header_list = headers.split(",");
 
         parms.put(ROOT_PATH_KEY, rootPath);
     	parms.put(TIMEOUT_KEY, timeout);
     	parms.put(RAISE_DELETE_ERRORS_KEY, flag);    	
+    	parms.put(CDMI_CONTENT_TYPE_KEY, type);
     }
 
     @Override
@@ -70,7 +73,7 @@ public void setAuthContext(AuthContext info) {
             }
 
             logger.debug("httpclient =" + httpClient + ", url = " + url);
-            client.init(httpClient, url, headerKV);
+            client.init(httpClient, url, headerKV, false);
         } catch (Exception e) {
             throw new StorageException(e);
         }