add gdas storage, modify list operation

sine-io · Apr 12, 2022 · 3f47cd6 · 3f47cd6
1 parent 2bc3ae9
commit 3f47cd6
Show file tree

Hide file tree

Showing 45 changed files with 1,554 additions and 55 deletions.
diff --git a/CHANGELOG-ehualu b/CHANGELOG-ehualu
@@ -1,4 +1,29 @@
 
+ COSBench 0.4.7.7(ehualu 2022.04.12)
+------------------------------------
+- #302: modify list operation of gdas
+		List operation: change usePrefix to useMarker where in the bucket listing begins. SDK default max-keys is 1000.
+- #301: add gdas storage.
+		e.g: <storage type="gdas" config="endpoint=xxx" />
+
+		Notice:
+			gdas use aws-sdk-java(version is 1.12.158 now)
+
+		gdas storage Features: 
+			no_verify_ssl: <storage type="gdas" config="no_verify_ssl=true;...other config..."/> 
+			Restore object: <operation type="restore" ...>
+					 <storage type="gdas" config="restore_days=1;...path_style_access=true;timeout=100000"/>
+			Multipart upload: <operation type="mwrite" ...>
+							  <storage type="s3" config="part_size=5242880;...path_style_access=true;timeout=100000"/>
+			Support GiB，MiB，KiB(2^n，not 10^n): 
+				<operation type="write" ratio="100" config="cprefix=xx;containers=c(1);oprefix=xx;objects=s(1,10);sizes=c(4)KiB" />
+			Head object: <operation type="head" ...>
+			mprepare, multipart upload object at prepare stage.
+			mfilewrite, multipart upload object at filewrite stage.
+			aws_region: <storage type="gdas" config="endpoint=xxx;aws_region=us-east-1" />
+						default is us-east-1
+
+
  COSBench 0.4.7.6(ehualu 2022.02.14)
 ------------------------------------
 - #300: Refactor: s3 and s3v2 will not be supported, so s3v2 was deleted, add ess and essv2.
@@ -27,7 +52,7 @@
 		ess Features special:
 			prefetch and range read: please read s3-config-prefetch-sample.xml and s3-config-range-sample.xml in the config dir.
 
-		essv2 Features special::
+		essv2 Features special:
 			aws_region: <storage type="essv2" config="endpoint=xxx;aws_region=us-east-1" />
 						default is us-east-1
 

diff --git a/README.md b/README.md
@@ -5,11 +5,25 @@ COSBench is a benchmarking tool to measure the performance of Cloud Object Stora
 emerging technology that is different from traditional file systems (e.g., NFS) or block device systems (e.g., iSCSI).
 Amazon S3 and Openstack* swift are well-known object storage solutions.
 
-COSBench now supports OpenStack* Swift, Amazon* S3, OpenIO*, Amplidata v2.3, 2.5 and 3.1, Scality*, Ceph, CDMI, Google* Cloud Storage, Aliyun OSS as well as custom adaptors.
+COSBench now supports GDAS, E-SuperStor, OpenStack* Swift, Amazon* S3, OpenIO*, Amplidata v2.3, 2.5 and 3.1, Scality*, Ceph, CDMI, Google* Cloud Storage, Aliyun OSS as well as custom adaptors.
 
 
-New features for ess and essv2(Usage, please refer to conf/ehualu-config-sample.xml, thanks.)
+New storage: gdas(Usage, please refer to conf/ehualu-config-sample.xml, thanks.)
 ----------------------------------------
+- GDAS is S3 storage and hardware is Blu-ray Disc.
+- mprepare, multipart upload object at prepare stage.
+- mfilewrite, multipart upload object at filewrite stage.
+- aws_region parameter for gdas.
+- Head object
+- GiB, MiB, KiB: Now GB is 10^n, GiB is 2^n.  
+- Multipart upload: Add Multipart upload method and part_size parameter: You can set it now. Default is 5MiB.  
+- Restore Object: Add Restore method and restore_days parameter: restore_days. You can set it now. Default is 1.  
+- HTTPS: If want to disable verify SSL, please set no_verify_ssl to true. Default is false.
+
+
+New storage: ess and essv2(Usage, please refer to conf/ehualu-config-sample.xml, thanks.)
+----------------------------------------
+- ESS is e-SuperStor and hardware can be 'fullstack' (NVMe, SSD, HDD, Tape, Disc, etc.). 
 - mprepare, multipart upload object at prepare stage.
 - mfilewrite, multipart upload object at filewrite stage.
 - aws_region parameter for essv2.
@@ -22,11 +36,11 @@ New features for ess and essv2(Usage, please refer to conf/ehualu-config-sample.
 - HTTPS: If want to disable verify SSL, please set no_verify_ssl to true. Default is false.
 
 
-eHualu - Notice
+Notice
 ----------------------------------------
 - s3v2 was deleted(essv2 instead), s3 is no longer actively maintained.
 - if need new features, please use ess/essv2, thanks.
-- ess means e-SuperStor(s3)
+- gdas use aws-sdk-java(now, version is 1.12.158)
 - ess use aws-sdk-java(now, version is 1.12.158)
 - essv2 use aws-sdk-java-v2(now, version is 2.17.129)
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.4.7.6
+0.4.7.7
diff --git a/dev/cosbench-ampli/META-INF/MANIFEST.MF b/dev/cosbench-ampli/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: Cosbench Ampli Client Bundle
 Bundle-SymbolicName: cosbench-ampli
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
 Import-Package: com.intel.cosbench.api.context,

diff --git a/dev/cosbench-api/META-INF/MANIFEST.MF b/dev/cosbench-api/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: COSBench API Bundle
 Bundle-SymbolicName: cosbench-api
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8,
  JavaSE-1.6

diff --git a/dev/cosbench-castor/META-INF/MANIFEST.MF b/dev/cosbench-castor/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: COSBench Castor Configuration Bundle
 Bundle-SymbolicName: cosbench-castor
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Fragment-Host: com.springsource.org.castor;bundle-version="[1.2.0,2.0.0)"
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6

diff --git a/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF b/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: CDMI Base Client Bundle
 Bundle-SymbolicName: cosbench-cdmi-base
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
 Import-Package: com.intel.cosbench.api.auth,

diff --git a/dev/cosbench-cdmi-swift/META-INF/MANIFEST.MF b/dev/cosbench-cdmi-swift/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: CDMI Client for Swift Bundle
 Bundle-SymbolicName: cosbench-cdmi-swift
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
 Import-Package: com.intel.cosbench.api.auth,

diff --git a/dev/cosbench-cdmi-util/META-INF/MANIFEST.MF b/dev/cosbench-cdmi-util/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: CDMI Utility Bundle
 Bundle-SymbolicName: cosbench-cdmi-util
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
 Export-Package: com.intel.cosbench.client.cdmi.util

diff --git a/dev/cosbench-config/META-INF/MANIFEST.MF b/dev/cosbench-config/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: COSBench Config Bundle
 Bundle-SymbolicName: cosbench-config
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8,
  JavaSE-1.6

diff --git a/dev/cosbench-controller-web/META-INF/MANIFEST.MF b/dev/cosbench-controller-web/META-INF/MANIFEST.MF
@@ -3,7 +3,7 @@ Web-ContextPath: controller
 Bundle-ManifestVersion: 2
 Bundle-Name: COSBench Controller Web Bundle
 Bundle-SymbolicName: cosbench-controller-web
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8,
  JavaSE-1.6

diff --git a/dev/cosbench-controller/META-INF/MANIFEST.MF b/dev/cosbench-controller/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: COSBench Controller Bundler
 Bundle-SymbolicName: cosbench-controller
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8,
  JavaSE-1.6

diff --git a/dev/cosbench-core-web/META-INF/MANIFEST.MF b/dev/cosbench-core-web/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: COSBench Web Core Bundle
 Bundle-SymbolicName: cosbench-core-web
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8,
  JavaSE-1.6

diff --git a/dev/cosbench-core/META-INF/MANIFEST.MF b/dev/cosbench-core/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: COSBench Core Bundle
 Bundle-SymbolicName: cosbench-core
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8,
  JavaSE-1.6

diff --git a/dev/cosbench-driver-web/META-INF/MANIFEST.MF b/dev/cosbench-driver-web/META-INF/MANIFEST.MF
@@ -3,7 +3,7 @@ Web-ContextPath: driver
 Bundle-ManifestVersion: 2
 Bundle-Name: COSBench Driver Web Bundle
 Bundle-SymbolicName: cosbench-driver-web
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8,
  JavaSE-1.6

diff --git a/dev/cosbench-driver/META-INF/MANIFEST.MF b/dev/cosbench-driver/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: COSBench Driver Bundle
 Bundle-SymbolicName: cosbench-driver
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: intel
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8,
  JavaSE-1.6

diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java
@@ -39,8 +39,6 @@ public class Lister extends AbstractOperator {
 
     private ObjectPicker objPicker = new ObjectPicker();
 
-//    private byte buffer[] = new byte[1024*1024];
-
     public Lister() {
         /* empty */
     }
@@ -75,7 +73,6 @@ private Sample doList(OutputStream out, String conName, String objName,
 
         InputStream in = null;
         CountingOutputStream cout = new CountingOutputStream(out);
-        doLogWarn(session.getLogger(), "listerrr: "+ conName + "/" + objName); // TODO: ????
         long start = System.nanoTime();
         long xferTime = 0L;
         try {
@@ -92,7 +89,7 @@ private Sample doList(OutputStream out, String conName, String objName,
 			doLogWarn(session.getLogger(), msg);
 
 			return new Sample(new Date(), getId(), getOpType(), getSampleType(), getName(), false);
-		}catch (Exception e) {
+		}catch (Exception e) { // TODO: catch IOException, need to improve.
             isUnauthorizedException(e, session);
             errorStatisticsHandle(e, session, conName + "/" + objName);
 

diff --git a/dev/cosbench-ecs/META-INF/MANIFEST.MF b/dev/cosbench-ecs/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: cosbench-ecs
 Bundle-SymbolicName: cosbench-ecs
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: emc
 Import-Package: com.intel.cosbench.api.auth,
  com.intel.cosbench.api.context,

diff --git a/dev/cosbench-ehualu/.classpath b/dev/cosbench-ehualu/.classpath
@@ -9,7 +9,7 @@
 	<classpathentry exported="true" kind="lib" path="aspectjweaver.jar"/>
 	<classpathentry exported="true" kind="lib" path="auth-2.17.129.jar"/>
 	<classpathentry exported="true" kind="lib" path="auth-2.17.129-sources.jar"/>
-	<classpathentry exported="true" kind="lib" path="aws-core-2.17.129.jar"/>
+	<classpathentry exported="true" kind="lib" path="aws-core-2.17.129.jar" sourcepath="aws-core-2.17.129-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="aws-core-2.17.129-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="aws-java-sdk-core-1.12.158.jar" sourcepath="aws-java-sdk-core-1.12.158-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="aws-java-sdk-core-1.12.158-sources.jar"/>
@@ -69,7 +69,7 @@
 	<classpathentry exported="true" kind="lib" path="netty-common-4.1.72.Final-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-handler-4.1.72.Final.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-handler-4.1.72.Final-sources.jar"/>
-	<classpathentry exported="true" kind="lib" path="netty-nio-client-2.17.129.jar"/>
+	<classpathentry exported="true" kind="lib" path="netty-nio-client-2.17.129.jar" sourcepath="netty-nio-client-2.17.129-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-nio-client-2.17.129-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-reactive-streams-2.0.5.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-reactive-streams-2.0.5-sources.jar"/>
@@ -79,7 +79,7 @@
 	<classpathentry exported="true" kind="lib" path="netty-resolver-4.1.72.Final-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-tcnative-classes-2.0.46.Final.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-tcnative-classes-2.0.46.Final-sources.jar"/>
-	<classpathentry exported="true" kind="lib" path="netty-transport-4.1.72.Final.jar"/>
+	<classpathentry exported="true" kind="lib" path="netty-transport-4.1.72.Final.jar" sourcepath="netty-transport-4.1.72.Final-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-transport-4.1.72.Final-sources.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-transport-classes-epoll-4.1.72.Final.jar"/>
 	<classpathentry exported="true" kind="lib" path="netty-transport-classes-epoll-4.1.72.Final-sources.jar"/>

diff --git a/dev/cosbench-ehualu/META-INF/MANIFEST.MF b/dev/cosbench-ehualu/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: eHualu S3 Storage Client Bundle
 Bundle-SymbolicName: cosbench-ehualu
-Bundle-Version: 0.4.7.6
+Bundle-Version: 0.4.7.7
 Bundle-Vendor: sine
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8,
  JavaSE-1.6

diff --git a/dev/cosbench-ehualu/META-INF/spring/plugin-context.xml b/dev/cosbench-ehualu/META-INF/spring/plugin-context.xml
@@ -8,15 +8,21 @@
          http://www.springframework.org/schema/osgi
          http://www.springframework.org/schema/osgi/spring-osgi.xsd">
 
-	<bean name="storageFactory" class="com.intel.cosbench.api.ehualu.EssStorageFactory" />
+	<bean name="storageFactoryEss" class="com.intel.cosbench.api.ehualu.EssStorageFactory" />
 
-	<bean name="storageFactoryV2" class="com.intel.cosbench.api.ehualu.EssStorageFactoryV2" />
+	<bean name="storageFactoryEssV2" class="com.intel.cosbench.api.ehualu.EssStorageFactoryV2" />
+
+	<bean name="storageFactoryGdas" class="com.intel.cosbench.api.ehualu.GdasStorageFactory" />
 
-	<osgi:service ref="storageFactory" context-class-loader="service-provider"
+	<osgi:service ref="storageFactoryEss" context-class-loader="service-provider"
+		interface="com.intel.cosbench.api.storage.StorageAPIFactory">
+	</osgi:service>
+
+	<osgi:service ref="storageFactoryEssV2" context-class-loader="service-provider"
 		interface="com.intel.cosbench.api.storage.StorageAPIFactory">
 	</osgi:service>
 
-	<osgi:service ref="storageFactoryV2" context-class-loader="service-provider"
+	<osgi:service ref="storageFactoryGdas" context-class-loader="service-provider"
 		interface="com.intel.cosbench.api.storage.StorageAPIFactory">
 	</osgi:service>
 

diff --git a/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak
@@ -0,0 +1,35 @@
+/**
+
+Copyright 2021-2022 eHualu Corporation, All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+*/
+package com.intel.cosbench.api.ehualu;
+
+import com.intel.cosbench.api.storage.StorageAPI;
+import com.intel.cosbench.api.storage.StorageAPIFactory;
+
+
+public class AsyncEssStorageFactoryV2 implements StorageAPIFactory {
+
+    @Override
+    public String getStorageName() {
+        return "aessv2";
+    }
+
+    @Override
+    public StorageAPI getStorageAPI() {
+        return new AsyncEssStorageV2();
+    }
+}