From 3f47cd6c34a0c84885695da40d4cf344a564d6a4 Mon Sep 17 00:00:00 2001 From: wanghx Date: Tue, 12 Apr 2022 22:13:00 +0800 Subject: [PATCH] add gdas storage, modify list operation --- CHANGELOG-ehualu | 27 +- README.md | 22 +- VERSION | 2 +- dev/cosbench-ampli/META-INF/MANIFEST.MF | 2 +- dev/cosbench-api/META-INF/MANIFEST.MF | 2 +- dev/cosbench-castor/META-INF/MANIFEST.MF | 2 +- dev/cosbench-cdmi-base/META-INF/MANIFEST.MF | 2 +- dev/cosbench-cdmi-swift/META-INF/MANIFEST.MF | 2 +- dev/cosbench-cdmi-util/META-INF/MANIFEST.MF | 2 +- dev/cosbench-config/META-INF/MANIFEST.MF | 2 +- .../META-INF/MANIFEST.MF | 2 +- dev/cosbench-controller/META-INF/MANIFEST.MF | 2 +- dev/cosbench-core-web/META-INF/MANIFEST.MF | 2 +- dev/cosbench-core/META-INF/MANIFEST.MF | 2 +- dev/cosbench-driver-web/META-INF/MANIFEST.MF | 2 +- dev/cosbench-driver/META-INF/MANIFEST.MF | 2 +- .../cosbench/driver/operator/Lister.java | 5 +- dev/cosbench-ecs/META-INF/MANIFEST.MF | 2 +- dev/cosbench-ehualu/.classpath | 6 +- dev/cosbench-ehualu/META-INF/MANIFEST.MF | 2 +- .../META-INF/spring/plugin-context.xml | 14 +- .../ehualu/AsyncEssStorageFactoryV2.java.bak | 35 ++ .../api/ehualu/AsyncEssStorageV2.java.bak | 428 ++++++++++++++++++ .../cosbench/api/ehualu/EssStorageV2.java | 2 +- .../cosbench/api/ehualu/GdasStorage.java | 338 ++++++++++++++ .../api/ehualu/GdasStorageFactory.java | 35 ++ .../cosbench/client/ehualu/GdasConstants.java | 93 ++++ .../ehualu/AsyncEssStorageFactoryV2.java.bak | 35 ++ .../api/ehualu/AsyncEssStorageV2.java.bak | 428 ++++++++++++++++++ dev/cosbench-gcs/META-INF/MANIFEST.MF | 2 +- dev/cosbench-http/META-INF/MANIFEST.MF | 2 +- dev/cosbench-httpauth/META-INF/MANIFEST.MF | 2 +- dev/cosbench-keystone/META-INF/MANIFEST.MF | 2 +- dev/cosbench-librados/META-INF/MANIFEST.MF | 2 +- dev/cosbench-log/META-INF/MANIFEST.MF | 2 +- dev/cosbench-log4j/META-INF/MANIFEST.MF | 2 +- dev/cosbench-mock/META-INF/MANIFEST.MF | 2 +- dev/cosbench-openio/META-INF/MANIFEST.MF | 2 +- dev/cosbench-oss/META-INF/MANIFEST.MF | 2 +- dev/cosbench-s3/META-INF/MANIFEST.MF | 2 +- dev/cosbench-scality/META-INF/MANIFEST.MF | 2 +- dev/cosbench-swauth/META-INF/MANIFEST.MF | 2 +- dev/cosbench-swift/META-INF/MANIFEST.MF | 2 +- dev/cosbench-tomcat/META-INF/MANIFEST.MF | 2 +- release/conf/ehualu-config-sample.xml | 79 +++- 45 files changed, 1554 insertions(+), 55 deletions(-) create mode 100644 dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak create mode 100644 dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageV2.java.bak create mode 100644 dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/GdasStorage.java create mode 100644 dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/GdasStorageFactory.java create mode 100644 dev/cosbench-ehualu/src/com/intel/cosbench/client/ehualu/GdasConstants.java create mode 100644 dev/cosbench-ehualu/target/classes/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak create mode 100644 dev/cosbench-ehualu/target/classes/com/intel/cosbench/api/ehualu/AsyncEssStorageV2.java.bak diff --git a/CHANGELOG-ehualu b/CHANGELOG-ehualu index f3b1da8..e916344 100644 --- a/CHANGELOG-ehualu +++ b/CHANGELOG-ehualu @@ -1,4 +1,29 @@ + COSBench 0.4.7.7(ehualu 2022.04.12) +------------------------------------ +- #302: modify list operation of gdas + List operation: change usePrefix to useMarker where in the bucket listing begins. SDK default max-keys is 1000. +- #301: add gdas storage. + e.g: + + Notice: + gdas use aws-sdk-java(version is 1.12.158 now) + + gdas storage Features: + no_verify_ssl: + Restore object: + + Multipart upload: + + Support GiB，MiB，KiB(2^n，not 10^n): + + Head object: + mprepare, multipart upload object at prepare stage. + mfilewrite, multipart upload object at filewrite stage. + aws_region: + default is us-east-1 + + COSBench 0.4.7.6(ehualu 2022.02.14) ------------------------------------ - #300: Refactor: s3 and s3v2 will not be supported, so s3v2 was deleted, add ess and essv2. @@ -27,7 +52,7 @@ ess Features special: prefetch and range read: please read s3-config-prefetch-sample.xml and s3-config-range-sample.xml in the config dir. - essv2 Features special:: + essv2 Features special: aws_region: default is us-east-1 diff --git a/README.md b/README.md index 41850e0..71448b5 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,25 @@ COSBench is a benchmarking tool to measure the performance of Cloud Object Stora emerging technology that is different from traditional file systems (e.g., NFS) or block device systems (e.g., iSCSI). Amazon S3 and Openstack* swift are well-known object storage solutions. -COSBench now supports OpenStack* Swift, Amazon* S3, OpenIO*, Amplidata v2.3, 2.5 and 3.1, Scality*, Ceph, CDMI, Google* Cloud Storage, Aliyun OSS as well as custom adaptors. +COSBench now supports GDAS, E-SuperStor, OpenStack* Swift, Amazon* S3, OpenIO*, Amplidata v2.3, 2.5 and 3.1, Scality*, Ceph, CDMI, Google* Cloud Storage, Aliyun OSS as well as custom adaptors. -New features for ess and essv2(Usage, please refer to conf/ehualu-config-sample.xml, thanks.) +New storage: gdas(Usage, please refer to conf/ehualu-config-sample.xml, thanks.) ---------------------------------------- +- GDAS is S3 storage and hardware is Blu-ray Disc. +- mprepare, multipart upload object at prepare stage. +- mfilewrite, multipart upload object at filewrite stage. +- aws_region parameter for gdas. +- Head object +- GiB, MiB, KiB: Now GB is 10^n, GiB is 2^n. +- Multipart upload: Add Multipart upload method and part_size parameter: You can set it now. Default is 5MiB. +- Restore Object: Add Restore method and restore_days parameter: restore_days. You can set it now. Default is 1. +- HTTPS: If want to disable verify SSL, please set no_verify_ssl to true. Default is false. + + +New storage: ess and essv2(Usage, please refer to conf/ehualu-config-sample.xml, thanks.) +---------------------------------------- +- ESS is e-SuperStor and hardware can be 'fullstack' (NVMe, SSD, HDD, Tape, Disc, etc.). - mprepare, multipart upload object at prepare stage. - mfilewrite, multipart upload object at filewrite stage. - aws_region parameter for essv2. @@ -22,11 +36,11 @@ New features for ess and essv2(Usage, please refer to conf/ehualu-config-sample. - HTTPS: If want to disable verify SSL, please set no_verify_ssl to true. Default is false. -eHualu - Notice +Notice ---------------------------------------- - s3v2 was deleted(essv2 instead), s3 is no longer actively maintained. - if need new features, please use ess/essv2, thanks. -- ess means e-SuperStor(s3) +- gdas use aws-sdk-java(now, version is 1.12.158) - ess use aws-sdk-java(now, version is 1.12.158) - essv2 use aws-sdk-java-v2(now, version is 2.17.129) diff --git a/VERSION b/VERSION index 26b0a45..d039d4b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.7.6 +0.4.7.7 diff --git a/dev/cosbench-ampli/META-INF/MANIFEST.MF b/dev/cosbench-ampli/META-INF/MANIFEST.MF index c4c2d70..c7816bc 100644 --- a/dev/cosbench-ampli/META-INF/MANIFEST.MF +++ b/dev/cosbench-ampli/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Cosbench Ampli Client Bundle Bundle-SymbolicName: cosbench-ampli -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.context, diff --git a/dev/cosbench-api/META-INF/MANIFEST.MF b/dev/cosbench-api/META-INF/MANIFEST.MF index 31b2d61..81b0b1c 100644 --- a/dev/cosbench-api/META-INF/MANIFEST.MF +++ b/dev/cosbench-api/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench API Bundle Bundle-SymbolicName: cosbench-api -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-castor/META-INF/MANIFEST.MF b/dev/cosbench-castor/META-INF/MANIFEST.MF index b0c1c58..631e606 100644 --- a/dev/cosbench-castor/META-INF/MANIFEST.MF +++ b/dev/cosbench-castor/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Castor Configuration Bundle Bundle-SymbolicName: cosbench-castor -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Fragment-Host: com.springsource.org.castor;bundle-version="[1.2.0,2.0.0)" Bundle-RequiredExecutionEnvironment: JavaSE-1.6 diff --git a/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF b/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF index 3f423d5..40e23f7 100644 --- a/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF +++ b/dev/cosbench-cdmi-base/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: CDMI Base Client Bundle Bundle-SymbolicName: cosbench-cdmi-base -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, diff --git a/dev/cosbench-cdmi-swift/META-INF/MANIFEST.MF b/dev/cosbench-cdmi-swift/META-INF/MANIFEST.MF index a1ad124..0dcaa6d 100644 --- a/dev/cosbench-cdmi-swift/META-INF/MANIFEST.MF +++ b/dev/cosbench-cdmi-swift/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: CDMI Client for Swift Bundle Bundle-SymbolicName: cosbench-cdmi-swift -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, diff --git a/dev/cosbench-cdmi-util/META-INF/MANIFEST.MF b/dev/cosbench-cdmi-util/META-INF/MANIFEST.MF index 0a83b6a..b7502d9 100644 --- a/dev/cosbench-cdmi-util/META-INF/MANIFEST.MF +++ b/dev/cosbench-cdmi-util/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: CDMI Utility Bundle Bundle-SymbolicName: cosbench-cdmi-util -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Export-Package: com.intel.cosbench.client.cdmi.util diff --git a/dev/cosbench-config/META-INF/MANIFEST.MF b/dev/cosbench-config/META-INF/MANIFEST.MF index 7eeb38e..49d40c5 100644 --- a/dev/cosbench-config/META-INF/MANIFEST.MF +++ b/dev/cosbench-config/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Config Bundle Bundle-SymbolicName: cosbench-config -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-controller-web/META-INF/MANIFEST.MF b/dev/cosbench-controller-web/META-INF/MANIFEST.MF index bd885d0..8618d8b 100644 --- a/dev/cosbench-controller-web/META-INF/MANIFEST.MF +++ b/dev/cosbench-controller-web/META-INF/MANIFEST.MF @@ -3,7 +3,7 @@ Web-ContextPath: controller Bundle-ManifestVersion: 2 Bundle-Name: COSBench Controller Web Bundle Bundle-SymbolicName: cosbench-controller-web -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-controller/META-INF/MANIFEST.MF b/dev/cosbench-controller/META-INF/MANIFEST.MF index e80cc08..8d8a15b 100644 --- a/dev/cosbench-controller/META-INF/MANIFEST.MF +++ b/dev/cosbench-controller/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Controller Bundler Bundle-SymbolicName: cosbench-controller -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-core-web/META-INF/MANIFEST.MF b/dev/cosbench-core-web/META-INF/MANIFEST.MF index fefe9fa..fa4299d 100644 --- a/dev/cosbench-core-web/META-INF/MANIFEST.MF +++ b/dev/cosbench-core-web/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Web Core Bundle Bundle-SymbolicName: cosbench-core-web -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-core/META-INF/MANIFEST.MF b/dev/cosbench-core/META-INF/MANIFEST.MF index c4fd955..36ac778 100644 --- a/dev/cosbench-core/META-INF/MANIFEST.MF +++ b/dev/cosbench-core/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Core Bundle Bundle-SymbolicName: cosbench-core -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-driver-web/META-INF/MANIFEST.MF b/dev/cosbench-driver-web/META-INF/MANIFEST.MF index e3876f5..beed179 100644 --- a/dev/cosbench-driver-web/META-INF/MANIFEST.MF +++ b/dev/cosbench-driver-web/META-INF/MANIFEST.MF @@ -3,7 +3,7 @@ Web-ContextPath: driver Bundle-ManifestVersion: 2 Bundle-Name: COSBench Driver Web Bundle Bundle-SymbolicName: cosbench-driver-web -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-driver/META-INF/MANIFEST.MF b/dev/cosbench-driver/META-INF/MANIFEST.MF index c03c2bc..1f1874f 100644 --- a/dev/cosbench-driver/META-INF/MANIFEST.MF +++ b/dev/cosbench-driver/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Driver Bundle Bundle-SymbolicName: cosbench-driver -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java index 62e95de..f847fc0 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java @@ -39,8 +39,6 @@ public class Lister extends AbstractOperator { private ObjectPicker objPicker = new ObjectPicker(); -// private byte buffer[] = new byte[1024*1024]; - public Lister() { /* empty */ } @@ -75,7 +73,6 @@ private Sample doList(OutputStream out, String conName, String objName, InputStream in = null; CountingOutputStream cout = new CountingOutputStream(out); - doLogWarn(session.getLogger(), "listerrr: "+ conName + "/" + objName); // TODO: ???? long start = System.nanoTime(); long xferTime = 0L; try { @@ -92,7 +89,7 @@ private Sample doList(OutputStream out, String conName, String objName, doLogWarn(session.getLogger(), msg); return new Sample(new Date(), getId(), getOpType(), getSampleType(), getName(), false); - }catch (Exception e) { + }catch (Exception e) { // TODO: catch IOException, need to improve. isUnauthorizedException(e, session); errorStatisticsHandle(e, session, conName + "/" + objName); diff --git a/dev/cosbench-ecs/META-INF/MANIFEST.MF b/dev/cosbench-ecs/META-INF/MANIFEST.MF index b22b306..02b65f9 100644 --- a/dev/cosbench-ecs/META-INF/MANIFEST.MF +++ b/dev/cosbench-ecs/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: cosbench-ecs Bundle-SymbolicName: cosbench-ecs -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: emc Import-Package: com.intel.cosbench.api.auth, com.intel.cosbench.api.context, diff --git a/dev/cosbench-ehualu/.classpath b/dev/cosbench-ehualu/.classpath index dec24f9..dbf6faf 100644 --- a/dev/cosbench-ehualu/.classpath +++ b/dev/cosbench-ehualu/.classpath @@ -9,7 +9,7 @@ - + @@ -69,7 +69,7 @@ - + @@ -79,7 +79,7 @@ - + diff --git a/dev/cosbench-ehualu/META-INF/MANIFEST.MF b/dev/cosbench-ehualu/META-INF/MANIFEST.MF index 4cf6704..1d2b984 100644 --- a/dev/cosbench-ehualu/META-INF/MANIFEST.MF +++ b/dev/cosbench-ehualu/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: eHualu S3 Storage Client Bundle Bundle-SymbolicName: cosbench-ehualu -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: sine Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-ehualu/META-INF/spring/plugin-context.xml b/dev/cosbench-ehualu/META-INF/spring/plugin-context.xml index a78247c..8c52faa 100644 --- a/dev/cosbench-ehualu/META-INF/spring/plugin-context.xml +++ b/dev/cosbench-ehualu/META-INF/spring/plugin-context.xml @@ -8,15 +8,21 @@ http://www.springframework.org/schema/osgi http://www.springframework.org/schema/osgi/spring-osgi.xsd"> - + - + + + - + + + - diff --git a/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak new file mode 100644 index 0000000..14f8b5c --- /dev/null +++ b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak @@ -0,0 +1,35 @@ +/** + +Copyright 2021-2022 eHualu Corporation, All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ +package com.intel.cosbench.api.ehualu; + +import com.intel.cosbench.api.storage.StorageAPI; +import com.intel.cosbench.api.storage.StorageAPIFactory; + + +public class AsyncEssStorageFactoryV2 implements StorageAPIFactory { + + @Override + public String getStorageName() { + return "aessv2"; + } + + @Override + public StorageAPI getStorageAPI() { + return new AsyncEssStorageV2(); + } +} diff --git a/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageV2.java.bak b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageV2.java.bak new file mode 100644 index 0000000..046628a --- /dev/null +++ b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/AsyncEssStorageV2.java.bak @@ -0,0 +1,428 @@ +/** + +Copyright 2021-2022 eHualu Corporation, All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.intel.cosbench.api.ehualu; + +import static com.intel.cosbench.client.ehualu.EssConstants.*; + +//import java.io.*; +import java.net.URI; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; + +//import java.util.List; +//import java.util.concurrent.CompletableFuture; +// +//import org.reactivestreams.Publisher; +// +//import java.util.ArrayList; +// +//import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +//import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +//import software.amazon.awssdk.awscore.exception.AwsServiceException; +//import software.amazon.awssdk.core.ResponseBytes; +//import software.amazon.awssdk.core.async.AsyncRequestBody; +//import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; +//import software.amazon.awssdk.core.exception.SdkClientException; +//import software.amazon.awssdk.core.sync.RequestBody; +//import software.amazon.awssdk.http.SdkHttpClient; +//import software.amazon.awssdk.http.SdkHttpConfigurationOption; +//import software.amazon.awssdk.http.TlsTrustManagersProvider; +//import software.amazon.awssdk.http.apache.ApacheHttpClient; +//import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +//import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +//import software.amazon.awssdk.http.nio.netty.ProxyConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.utils.AttributeMap; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.ResponsePublisher; +import software.amazon.awssdk.http.SdkHttpConfigurationOption; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.ProxyConfiguration; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Configuration; + +import com.intel.cosbench.api.storage.*; +import com.amazonaws.services.s3.transfer.internal.future.CompletedFuture; +import com.intel.cosbench.api.context.*; +import com.intel.cosbench.config.Config; +import com.intel.cosbench.log.Logger; + +public class AsyncEssStorageV2 extends NoneStorage { + + private int timeout; + + private String accessKey; + private String secretKey; + private String endpoint; + + private S3AsyncClient client; + + private String storageClass; + private int restoreDays; + private long partSize; // Upload the file parts. + private boolean noVerifySSL; + private String awsRegion; + + private boolean pathStyleAccess; + private int maxConnections; + private String proxyHost; + private String proxyPort; + + @Override + public void init(Config config, Logger logger) { + super.init(config, logger); + + timeout = config.getInt(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT); + endpoint = config.get(ENDPOINT_KEY, ENDPOINT_DEFAULT); + accessKey = config.get(AUTH_USERNAME_KEY, AUTH_USERNAME_DEFAULT); + secretKey = config.get(AUTH_PASSWORD_KEY, AUTH_PASSWORD_DEFAULT); + + pathStyleAccess = config.getBoolean(PATH_STYLE_ACCESS_KEY, PATH_STYLE_ACCESS_DEFAULT); + maxConnections = config.getInt(MAX_CONNECTIONS, MAX_CONNECTIONS_DEFAULT); + + proxyHost = config.get(PROXY_HOST_KEY, ""); + proxyPort = config.get(PROXY_PORT_KEY, ""); + parms.put(PROXY_PORT_KEY, proxyPort); + + // 2021.02.14 + // You can set storage_class to other value in storage part. + storageClass = config.get(STORAGE_CLASS_KEY, STORAGE_CLASS_DEFAULT); + + // 2021.07.11 + // You can set restore_days to other value(int) in storage part. + restoreDays = config.getInt(RESTORE_DAYS_KEY, RESTORE_DAYS_DEFAULT); + + // 2021.08.03 + // You can set part_size to other value in storage part. + partSize = config.getLong(PART_SIZE_KEY, PART_SIZE_DEFAULT); + + // 2020.11.26 + // You can set no_verify_ssl to true in storage part to disable SSL checking. + noVerifySSL = config.getBoolean(NO_VERIFY_SSL_KEY, NO_VERIFY_SSL_DEFAULT); + + // 2022.02.03 + // You can set region now, and default is us-east-1. + awsRegion = config.get(REGION_KEY, REGION_DEFAULT); + + initAsyncClient(); + + } + + private S3AsyncClient initAsyncClient() { + logger.debug("initialize S3 async client with storage config: {}", parms); + + AwsBasicCredentials awsCreds = AwsBasicCredentials.create(accessKey, secretKey); + + S3Configuration s3Config = S3Configuration.builder().pathStyleAccessEnabled(pathStyleAccess).build(); + + ProxyConfiguration.Builder proxyConfig = ProxyConfiguration.builder(); + if ((!proxyHost.equals("")) && (!proxyPort.equals(""))) { + proxyConfig.host(proxyHost).port(parms.getInt(PROXY_PORT_KEY)); + } + + NettyNioAsyncHttpClient.Builder asyncHttpClientBuilder = NettyNioAsyncHttpClient.builder() + // For HTTP/1.1 this is the same as max connections. + .maxConcurrency(maxConnections) + // Set connection timeout for initially establishing a connection. + .connectionTimeout(Duration.ofMillis(timeout)) + // Set socket timeout(write and read) for data to be transferred. + // TODO: need to confirm. + .writeTimeout(Duration.ofMillis(timeout)).readTimeout(Duration.ofMillis(timeout)) + // disable expect continue for HTTP/1.1. + // TODO + .putChannelOption(null, proxyConfig) + // proxy config. + .proxyConfiguration(proxyConfig.build()); + + SdkAsyncHttpClient asyncHttpClient; + if (noVerifySSL) { + asyncHttpClient = asyncHttpClientBuilder.buildWithDefaults(AttributeMap.builder() + .put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, Boolean.TRUE).build()); + } else { + asyncHttpClient = asyncHttpClientBuilder.build(); + } + + client = S3AsyncClient.builder().credentialsProvider(StaticCredentialsProvider.create(awsCreds)) + .endpointOverride(URI.create(endpoint)) + // solve issue: https://github.com/kisscelia/cosbench-ehualu/issues/3 + .region(Region.of(awsRegion)).serviceConfiguration(s3Config).httpClient(asyncHttpClient).build(); + + logger.debug("S3 async client has been initialized"); + + return client; + } + + @Override + public void setAuthContext(AuthContext info) { + super.setAuthContext(info); + } + + @Override + public void dispose() { + super.dispose(); + client = null; + } + + @Override + public InputStream getObject(String container, String object, Config config) { + super.getObject(container, object, config); + InputStream stream = null; + + GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(container).key(object).build(); + CompletableFuture> futureGet = client.getObject(getObjectRequest, + AsyncResponseTransformer.toPublisher()); + futureGet.whenComplete((resp, err) -> { + try { + if (resp != null) { + System.out.println("Object downloaded. Details: " + resp); + } else { + err.printStackTrace(); + + } + } finally { + client.close(); + } + }); + +// try { +// GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(container).key(object).build(); +// CompletedFuture futureGet = client.getObject(getObjectRequest, null); +// +// +//// ResponseBytes objectBytes = client.getObjectAsBytes(getObjectRequest); +// +//// stream = objectBytes.asInputStream(); +// +// } catch (AwsServiceException ase) { +// throw new StorageException(ase); +// } catch (SdkClientException sce) { +// throw new StorageTimeoutException(sce); +// } + + return stream; + } + + @Override + public void restoreObject(String container, String object, Config config) { + super.restoreObject(container, object, config); + try { + // Create and submit a request to restore an object from Glacier/Deep Archive + // for some days. + RestoreRequest restoreRequest = RestoreRequest.builder().days(restoreDays).build(); + + RestoreObjectRequest restoreObjectRequest = RestoreObjectRequest.builder().bucket(container).key(object) + .restoreRequest(restoreRequest).build(); + + client.restoreObject(restoreObjectRequest); + + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void createContainer(String container, Config config) { + super.createContainer(container, config); + try { + container = container.split("/")[0]; + + CreateBucketRequest createBucketRequest = CreateBucketRequest.builder().bucket(container).build(); + client.createBucket(createBucketRequest); + + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void createObject(String container, String object, InputStream data, long length, Config config) { + super.createObject(container, object, data, length, config); + + PutObjectRequest putObjectRequest = PutObjectRequest.builder().contentLength(length) + .contentType("application/octet-stream").storageClass(storageClass).bucket(container).key(object) + .build(); + + // https://github.com/aws/aws-sdk-java-v2/issues/1177 + // V2 async uses NIO APIs and supports non-blocking streaming operations. + // With v2 async client, you need to provide a Publisher instead of InputStream + // (blocking IO) via AsyncRequestBody#fromPublisher. + AsyncRequestBody aRequestBody = AsyncRequestBody.fromPublisher(null); + CompletableFuture future = client.putObject(putObjectRequest, aRequestBody); + + future.whenComplete((resp, err) -> { + try { + if (resp != null) { + // normal flow. + + } else { + // error handler. + + } + } finally { + client.close(); + } + }); + + future.join(); + +// try { +// +// } catch (AwsServiceException ase) { +// throw new StorageException(ase); +// } catch (SdkClientException sce) { +// throw new StorageTimeoutException(sce); +// } + } + + @Override + public void createMultipartObject(String container, String object, InputStream data, long length, Config config) { + super.createMultipartObject(container, object, data, length, config); + +// Map metadata = new HashMap<>(); +// metadata.put("Content-Length", Long.toString(length)); + + CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder() + .contentType("application/octet-stream").storageClass(storageClass).bucket(container).key(object) +// .overrideConfiguration(AwsRequestOverrideConfiguration.builder().putHeader("Content-Length", Long.toString(length)).build()) +// .metadata(metadata) + .build(); + + // Create a list of ETag objects. You retrieve ETags for each object part + // uploaded, + // then, after each individual part has been uploaded, pass the list of ETags to + // the request to complete the upload. + List partETags = new ArrayList(); + + try { + CreateMultipartUploadResponse response = client.createMultipartUpload(createMultipartUploadRequest); + String uploadId = response.uploadId(); + + long position = 0; + + for (int i = 1; position < length; i++) { + // Because the last part could be less than 5 MiB, adjust the part size as + // needed. + partSize = Math.min(partSize, (length - position)); + + // Create the request to upload a part. + UploadPartRequest uploadRequest = UploadPartRequest.builder().contentLength(partSize) // TODO: needed? + .bucket(container).key(object).uploadId(uploadId).partNumber(i).build(); + + // Upload the part and add the response's ETag to our list. + RequestBody requestBody = RequestBody.fromInputStream(data, partSize); + UploadPartResponse uploadPartResponse = client.uploadPart(uploadRequest, requestBody); + + CompletedPart tempPart = CompletedPart.builder().partNumber(i).eTag(uploadPartResponse.eTag()).build(); + partETags.add(tempPart); + + position += partSize; + } + + CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(partETags) + .build(); + + // Complete the multipart upload. + CompleteMultipartUploadRequest compRequest = CompleteMultipartUploadRequest.builder().bucket(container) + .key(object).uploadId(uploadId).multipartUpload(completedMultipartUpload).build(); + client.completeMultipartUpload(compRequest); + + } catch (AwsServiceException ase) { + throw new StorageException(ase); + } catch (SdkClientException sce) { + throw new StorageTimeoutException(sce); + } + } + + @Override + public void deleteContainer(String container, Config config) { + super.deleteContainer(container, config); + try { + container = container.split("/")[0]; + + DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder().bucket(container).build(); + + client.deleteBucket(deleteBucketRequest); + + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void deleteObject(String container, String object, Config config) { + super.deleteObject(container, object, config); + try { + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(container).key(object) + .build(); + + client.deleteObject(deleteObjectRequest); + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public InputStream getList(String container, String object, Config config) { + super.getList(container, object, config); + InputStream stream = null; + try { + + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(container).prefix(object) + .build(); + + CompletableFuture future = client.listObjectsV2(listObjectsV2Request); + + future.whenComplete((resp, err) -> { + try { + if (resp != null) { + + } else { + + } + } finally { + client.close(); + } + }); + + future.join(); + +// stream = new ByteArrayInputStream(result.toString().getBytes()); + } catch (Exception e) { + throw new StorageException(e); + } + + return stream; + } + + @Override + public void headObject(String container, String object, Config config) { + super.headObject(container, object, config); + + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(container).key(object).build(); + + try { + client.headObject(headObjectRequest); + + } catch (Exception e) { + throw new StorageException(e); + } + } +} diff --git a/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/EssStorageV2.java b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/EssStorageV2.java index 1169ec4..613137c 100644 --- a/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/EssStorageV2.java +++ b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/EssStorageV2.java @@ -136,7 +136,7 @@ private S3Client initClient() { } else { httpClient = httpClientBuilder.build(); } - + client = S3Client.builder().credentialsProvider(StaticCredentialsProvider.create(awsCreds)) .endpointOverride(URI.create(endpoint)) // solve issue: https://github.com/kisscelia/cosbench-ehualu/issues/3 diff --git a/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/GdasStorage.java b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/GdasStorage.java new file mode 100644 index 0000000..3e7698a --- /dev/null +++ b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/GdasStorage.java @@ -0,0 +1,338 @@ +/** + +Copyright 2021-2022 eHualu Corporation, All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.intel.cosbench.api.ehualu; + +import static com.intel.cosbench.client.ehualu.GdasConstants.*; + +import java.io.*; + +import java.util.List; +import java.util.ArrayList; + +import com.amazonaws.*; +import com.amazonaws.auth.*; +import com.amazonaws.services.s3.*; +import com.amazonaws.services.s3.model.*; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; + +import com.intel.cosbench.log.Logger; +import com.intel.cosbench.api.storage.*; +import com.intel.cosbench.api.context.*; +import com.intel.cosbench.config.Config; + + +public class GdasStorage extends NoneStorage { + + private int timeout; + + private String accessKey; + private String secretKey; + private String endpoint; + + private AmazonS3 client; + private AmazonS3 restoreClient; // 2021.7.13, sine. + + private int restoreDays; + private long partSize; // 2021.7.13, sine. Upload the file parts. + private boolean noVerifySSL; + + private boolean pathStyleAccess; + private int maxConnections; + private String proxyHost; + private String proxyPort; + + @Override + public void init(Config config, Logger logger) { + super.init(config, logger); + + timeout = config.getInt(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT); + endpoint = config.get(ENDPOINT_KEY, ENDPOINT_DEFAULT); + accessKey = config.get(AUTH_USERNAME_KEY, AUTH_USERNAME_DEFAULT); + secretKey = config.get(AUTH_PASSWORD_KEY, AUTH_PASSWORD_DEFAULT); + + pathStyleAccess = config.getBoolean(PATH_STYLE_ACCESS_KEY, PATH_STYLE_ACCESS_DEFAULT); + maxConnections = config.getInt(MAX_CONNECTIONS, MAX_CONNECTIONS_DEFAULT); + + proxyHost = config.get(PROXY_HOST_KEY, ""); + proxyPort = config.get(PROXY_PORT_KEY, ""); + parms.put(PROXY_PORT_KEY, proxyPort); // because proxyPort is String, but setProxyPort needs a int. + // You can set restore_days to other value(int) in storage part. + restoreDays = config.getInt(RESTORE_DAYS_KEY, RESTORE_DAYS_DEFAULT); + // You can set part_size to other value in storage part. + partSize = config.getLong(PART_SIZE_KEY, PART_SIZE_DEFAULT); + // You can set no_verify_ssl to true in storage part to disable SSL checking. + noVerifySSL = config.getBoolean(NO_VERIFY_SSL_KEY, NO_VERIFY_SSL_DEFAULT); + if (noVerifySSL) { + // This property is meant to be used as a flag + // (i.e. -Dcom.amazonaws.sdk.disableCertChecking) rather then taking a value + // (-Dcom.amazonaws.sdk.disableCertChecking=true). + System.setProperty(SDKGlobalConfiguration.DISABLE_CERT_CHECKING_SYSTEM_PROPERTY, "true"); + } + + initClient(); + initRestoreClient(); + } + + // You can set different singType to get different client(common client type vs + // restore client type). 2021.7.14 sine + private ClientConfiguration getDefaultClientConfiguration(String signType) { + + ClientConfiguration defaultClientConfiguration = new ClientConfiguration(); + // Set connection timeout for initially establishing a connection. + defaultClientConfiguration.setConnectionTimeout(timeout); + // Set socket timeout for data to be transferred. + defaultClientConfiguration.setSocketTimeout(timeout); + // Set max connections. + defaultClientConfiguration.setMaxConnections(maxConnections); + // use expect continue HTTP/1.1 header. + defaultClientConfiguration.withUseExpectContinue(false); + // Set Signer type. + defaultClientConfiguration.withSignerOverride(signType); + + // noqa + if ((!proxyHost.equals("")) && (!proxyPort.equals(""))) { + defaultClientConfiguration.setProxyHost(proxyHost); + defaultClientConfiguration.setProxyPort(parms.getInt(PROXY_PORT_KEY)); + } + + return defaultClientConfiguration; + } + + private AmazonS3 initClient() { + logger.debug("initialize S3 client with storage config: {}", parms); + + ClientConfiguration clientConf = getDefaultClientConfiguration("S3SignerType"); + + AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey); + + EndpointConfiguration myEndpoint = new EndpointConfiguration(endpoint, ""); + + client = AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(myCredentials)) + .withClientConfiguration(clientConf).withEndpointConfiguration(myEndpoint) + .withPathStyleAccessEnabled(pathStyleAccess).build(); + + logger.debug("S3 client has been initialized"); + + return client; + } + + private AmazonS3 initRestoreClient() { + + logger.debug("initialize S3 client with storage config: {}", parms); + + ClientConfiguration clientConf = getDefaultClientConfiguration("AWSS3V4SignerType"); + + AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey); + + EndpointConfiguration myendpoint = new EndpointConfiguration(endpoint, ""); + restoreClient = AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(myCredentials)).withClientConfiguration(clientConf) + .withEndpointConfiguration(myendpoint) + .withPathStyleAccessEnabled(pathStyleAccess).build(); + + logger.debug("S3 client has been initialized"); + + return restoreClient; + } + + @Override + public void setAuthContext(AuthContext info) { + super.setAuthContext(info); + } + + @Override + public void dispose() { + super.dispose(); + client = null; + } + + @Override + public InputStream getObject(String container, String object, Config config) { + super.getObject(container, object, config); + InputStream stream = null; + try { + S3Object s3Obj = client.getObject(container, object); + stream = s3Obj.getObjectContent(); + } catch (AmazonServiceException ase) { + throw new StorageException(ase); + } catch (SdkClientException sce) { + throw new StorageTimeoutException(sce); + } + return stream; + } + + @Override + public void restoreObject(String container, String object, Config config) { + super.restoreObject(container, object, config); + try { + // Create and submit a request to restore an object from Glacier/Deep Archive + // for some days. + RestoreObjectRequest request = new RestoreObjectRequest(container, object, restoreDays); + restoreClient.restoreObjectV2(request); + + ObjectMetadata response = restoreClient.getObjectMetadata(container, object); + Boolean restoreFlag = response.getOngoingRestore(); + logger.info(object + " at bucket -> " + container + " | Restore days: " + restoreDays + + ", and ongoing-request status is: " + restoreFlag); + + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void createContainer(String container, Config config) { + super.createContainer(container, config); + try { + client.createBucket(container); + } catch (AmazonS3Exception bae) { + if (bae.getErrorCode().contentEquals("BucketAlreadyExists")) { + logger.info("Bucket " + container + " already exists, create-bucket step skipped"); + } else { + throw new StorageException(bae); + } + }catch (AmazonServiceException ase) { + throw new StorageException(ase); + }catch (SdkClientException sce) { + throw new StorageTimeoutException(sce); + } + } + + @Override + public void createObject(String container, String object, InputStream data, long length, Config config) { + super.createObject(container, object, data, length, config); + + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(length); + metadata.setContentType("application/octet-stream"); + + // https://github.com/awsdocs/aws-java-developer-guide/blob/master/doc_source/best-practices.rst + PutObjectRequest request = new PutObjectRequest(container, object, data, metadata); + + request.getRequestClientOptions().setReadLimit((int)length + 1); // set limit to object length+1 + + try { + // client.putObject(container, object, data, metadata); + + client.putObject(request); + + } catch (AmazonServiceException ase) { + throw new StorageException(ase); + } catch (SdkClientException sce) { + throw new StorageTimeoutException(sce); + } + } + + // 2021.8.3 updated, sine. + @Override + public void createMultipartObject(String container, String object, InputStream data, long length, Config config) { + super.createMultipartObject(container, object, data, length, config); + + // Set Metadata. + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(length); + metadata.setContentType("application/octet-stream"); + + // Create a list of ETag objects. You retrieve ETags for each object part uploaded, + // then, after each individual part has been uploaded, pass the list of ETags to + // the request to complete the upload. + List partETags = new ArrayList(); + + try { + // Initiate the multipart upload. + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(container, object, + metadata); + InitiateMultipartUploadResult initResponse = client.initiateMultipartUpload(initRequest); + + String uploadID = initResponse.getUploadId(); + + long position = 0; + + for (int i = 1; position < length; i++) { + // Because the last part could be less than 5 MiB, adjust the part size as + // needed. + partSize = Math.min(partSize, (length - position)); + + // Create the request to upload a part. + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(container) + .withKey(object) + .withUploadId(uploadID) + .withPartNumber(i) + .withInputStream(data) + .withPartSize(partSize); + + uploadRequest.getRequestClientOptions().setReadLimit((int)length+1); // length+1 + + // Upload the part and add the response's ETag to our list. + UploadPartResult uploadResult = client.uploadPart(uploadRequest); + partETags.add(uploadResult.getPartETag()); + + position += partSize; + } + + // Complete the multipart upload. + CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(container, object, + uploadID, partETags); + client.completeMultipartUpload(compRequest); + + } catch (AmazonServiceException ase) { + throw new StorageException(ase); + } catch (SdkClientException sce) { + throw new StorageTimeoutException(sce); + } + } + + @Override + public void deleteContainer(String container, Config config) { + super.deleteContainer(container, config); + try { + client.deleteBucket(container); + }catch (AmazonServiceException ase) { + throw new StorageException(ase); + }catch (SdkClientException sce) { + throw new StorageTimeoutException(sce); + } + } + + @Override + public void deleteObject(String container, String object, Config config) { + super.deleteObject(container, object, config); + try { + client.deleteObject(container, object); + }catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public InputStream getList(String container, String object, Config config) { + // List objects with marker and with max-keys 1000. + super.getList(container, object, config); + InputStream stream = null; + try { + ListObjectsRequest req = new ListObjectsRequest().withBucketName(container).withMarker(object); + ObjectListing result = client.listObjects(req); + + stream = new ByteArrayInputStream(result.getObjectSummaries().toString().getBytes()); + } catch (Exception e) { + throw new StorageException(e); + } + + return stream; + } +} diff --git a/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/GdasStorageFactory.java b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/GdasStorageFactory.java new file mode 100644 index 0000000..00e7522 --- /dev/null +++ b/dev/cosbench-ehualu/src/com/intel/cosbench/api/ehualu/GdasStorageFactory.java @@ -0,0 +1,35 @@ +/** + +Copyright 2021-2022 eHualu Corporation, All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ +package com.intel.cosbench.api.ehualu; + +import com.intel.cosbench.api.storage.StorageAPI; +import com.intel.cosbench.api.storage.StorageAPIFactory; + + +public class GdasStorageFactory implements StorageAPIFactory { + + @Override + public String getStorageName() { + return "gdas"; + } + + @Override + public StorageAPI getStorageAPI() { + return new GdasStorage(); + } +} diff --git a/dev/cosbench-ehualu/src/com/intel/cosbench/client/ehualu/GdasConstants.java b/dev/cosbench-ehualu/src/com/intel/cosbench/client/ehualu/GdasConstants.java new file mode 100644 index 0000000..9995f8d --- /dev/null +++ b/dev/cosbench-ehualu/src/com/intel/cosbench/client/ehualu/GdasConstants.java @@ -0,0 +1,93 @@ +/** + +Copyright 2021-2022 eHualu Corporation, All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ +package com.intel.cosbench.client.ehualu; + + +public interface GdasConstants { + // -------------------------------------------------------------------------- + // CONNECTION + // -------------------------------------------------------------------------- + String CONN_TIMEOUT_KEY = "timeout"; + int CONN_TIMEOUT_DEFAULT = 30000; + + // -------------------------------------------------------------------------- + // ENDPOINT + // -------------------------------------------------------------------------- + String ENDPOINT_KEY = "endpoint"; + String ENDPOINT_DEFAULT = "http://s3.amazonaws.com"; + + // -------------------------------------------------------------------------- + // AUTHENTICATION + // -------------------------------------------------------------------------- + String AUTH_USERNAME_KEY = "accesskey"; + String AUTH_USERNAME_DEFAULT = ""; + + String AUTH_PASSWORD_KEY = "secretkey"; + String AUTH_PASSWORD_DEFAULT = ""; + + // -------------------------------------------------------------------------- + // CLIENT CONFIGURATION + // -------------------------------------------------------------------------- + String PROXY_HOST_KEY = "proxyhost"; + String PROXY_PORT_KEY = "proxyport"; + + // MAX CONNECTIONS DEFAULT + // -------------------------------------------------------------------------- + String MAX_CONNECTIONS = "max_connections"; + int MAX_CONNECTIONS_DEFAULT = 50; + + // -------------------------------------------------------------------------- + // PATH STYLE ACCESS + // -------------------------------------------------------------------------- + String PATH_STYLE_ACCESS_KEY = "path_style_access"; + boolean PATH_STYLE_ACCESS_DEFAULT = false; + + // -------------------------------------------------------------------------- + // CONTEXT NEEDS FROM AUTH MODULE + // -------------------------------------------------------------------------- + String S3CLIENT_KEY = "s3client"; + + // -------------------------------------------------------------------------- + // 2020.11.26, if true, will verify ssl. + // NO VERIFY SSL + // -------------------------------------------------------------------------- + String NO_VERIFY_SSL_KEY = "no_verify_ssl"; + boolean NO_VERIFY_SSL_DEFAULT = false; + + // -------------------------------------------------------------------------- + // 2021.7.11 + // RestoreDays + // -------------------------------------------------------------------------- + String RESTORE_DAYS_KEY = "restore_days"; + int RESTORE_DAYS_DEFAULT = 1; + + // -------------------------------------------------------------------------- + // 2021.8.3, default: 5MiB + // PartSize for Multipart upload. + // -------------------------------------------------------------------------- + String PART_SIZE_KEY = "part_size"; + long PART_SIZE_DEFAULT = 5 * 1024 * 1024; // 5MiB + + // -------------------------------------------------------------------------- + // 2022.02.03, add region + // Default region is US-EAST-1. + // -------------------------------------------------------------------------- + String REGION_KEY = "aws_region"; + String REGION_DEFAULT = "us-east-1"; + +} diff --git a/dev/cosbench-ehualu/target/classes/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak b/dev/cosbench-ehualu/target/classes/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak new file mode 100644 index 0000000..14f8b5c --- /dev/null +++ b/dev/cosbench-ehualu/target/classes/com/intel/cosbench/api/ehualu/AsyncEssStorageFactoryV2.java.bak @@ -0,0 +1,35 @@ +/** + +Copyright 2021-2022 eHualu Corporation, All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ +package com.intel.cosbench.api.ehualu; + +import com.intel.cosbench.api.storage.StorageAPI; +import com.intel.cosbench.api.storage.StorageAPIFactory; + + +public class AsyncEssStorageFactoryV2 implements StorageAPIFactory { + + @Override + public String getStorageName() { + return "aessv2"; + } + + @Override + public StorageAPI getStorageAPI() { + return new AsyncEssStorageV2(); + } +} diff --git a/dev/cosbench-ehualu/target/classes/com/intel/cosbench/api/ehualu/AsyncEssStorageV2.java.bak b/dev/cosbench-ehualu/target/classes/com/intel/cosbench/api/ehualu/AsyncEssStorageV2.java.bak new file mode 100644 index 0000000..046628a --- /dev/null +++ b/dev/cosbench-ehualu/target/classes/com/intel/cosbench/api/ehualu/AsyncEssStorageV2.java.bak @@ -0,0 +1,428 @@ +/** + +Copyright 2021-2022 eHualu Corporation, All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.intel.cosbench.api.ehualu; + +import static com.intel.cosbench.client.ehualu.EssConstants.*; + +//import java.io.*; +import java.net.URI; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; + +//import java.util.List; +//import java.util.concurrent.CompletableFuture; +// +//import org.reactivestreams.Publisher; +// +//import java.util.ArrayList; +// +//import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +//import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +//import software.amazon.awssdk.awscore.exception.AwsServiceException; +//import software.amazon.awssdk.core.ResponseBytes; +//import software.amazon.awssdk.core.async.AsyncRequestBody; +//import software.amazon.awssdk.core.client.config.ClientAsyncConfiguration; +//import software.amazon.awssdk.core.exception.SdkClientException; +//import software.amazon.awssdk.core.sync.RequestBody; +//import software.amazon.awssdk.http.SdkHttpClient; +//import software.amazon.awssdk.http.SdkHttpConfigurationOption; +//import software.amazon.awssdk.http.TlsTrustManagersProvider; +//import software.amazon.awssdk.http.apache.ApacheHttpClient; +//import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +//import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +//import software.amazon.awssdk.http.nio.netty.ProxyConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.utils.AttributeMap; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.async.ResponsePublisher; +import software.amazon.awssdk.http.SdkHttpConfigurationOption; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.ProxyConfiguration; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Configuration; + +import com.intel.cosbench.api.storage.*; +import com.amazonaws.services.s3.transfer.internal.future.CompletedFuture; +import com.intel.cosbench.api.context.*; +import com.intel.cosbench.config.Config; +import com.intel.cosbench.log.Logger; + +public class AsyncEssStorageV2 extends NoneStorage { + + private int timeout; + + private String accessKey; + private String secretKey; + private String endpoint; + + private S3AsyncClient client; + + private String storageClass; + private int restoreDays; + private long partSize; // Upload the file parts. + private boolean noVerifySSL; + private String awsRegion; + + private boolean pathStyleAccess; + private int maxConnections; + private String proxyHost; + private String proxyPort; + + @Override + public void init(Config config, Logger logger) { + super.init(config, logger); + + timeout = config.getInt(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT); + endpoint = config.get(ENDPOINT_KEY, ENDPOINT_DEFAULT); + accessKey = config.get(AUTH_USERNAME_KEY, AUTH_USERNAME_DEFAULT); + secretKey = config.get(AUTH_PASSWORD_KEY, AUTH_PASSWORD_DEFAULT); + + pathStyleAccess = config.getBoolean(PATH_STYLE_ACCESS_KEY, PATH_STYLE_ACCESS_DEFAULT); + maxConnections = config.getInt(MAX_CONNECTIONS, MAX_CONNECTIONS_DEFAULT); + + proxyHost = config.get(PROXY_HOST_KEY, ""); + proxyPort = config.get(PROXY_PORT_KEY, ""); + parms.put(PROXY_PORT_KEY, proxyPort); + + // 2021.02.14 + // You can set storage_class to other value in storage part. + storageClass = config.get(STORAGE_CLASS_KEY, STORAGE_CLASS_DEFAULT); + + // 2021.07.11 + // You can set restore_days to other value(int) in storage part. + restoreDays = config.getInt(RESTORE_DAYS_KEY, RESTORE_DAYS_DEFAULT); + + // 2021.08.03 + // You can set part_size to other value in storage part. + partSize = config.getLong(PART_SIZE_KEY, PART_SIZE_DEFAULT); + + // 2020.11.26 + // You can set no_verify_ssl to true in storage part to disable SSL checking. + noVerifySSL = config.getBoolean(NO_VERIFY_SSL_KEY, NO_VERIFY_SSL_DEFAULT); + + // 2022.02.03 + // You can set region now, and default is us-east-1. + awsRegion = config.get(REGION_KEY, REGION_DEFAULT); + + initAsyncClient(); + + } + + private S3AsyncClient initAsyncClient() { + logger.debug("initialize S3 async client with storage config: {}", parms); + + AwsBasicCredentials awsCreds = AwsBasicCredentials.create(accessKey, secretKey); + + S3Configuration s3Config = S3Configuration.builder().pathStyleAccessEnabled(pathStyleAccess).build(); + + ProxyConfiguration.Builder proxyConfig = ProxyConfiguration.builder(); + if ((!proxyHost.equals("")) && (!proxyPort.equals(""))) { + proxyConfig.host(proxyHost).port(parms.getInt(PROXY_PORT_KEY)); + } + + NettyNioAsyncHttpClient.Builder asyncHttpClientBuilder = NettyNioAsyncHttpClient.builder() + // For HTTP/1.1 this is the same as max connections. + .maxConcurrency(maxConnections) + // Set connection timeout for initially establishing a connection. + .connectionTimeout(Duration.ofMillis(timeout)) + // Set socket timeout(write and read) for data to be transferred. + // TODO: need to confirm. + .writeTimeout(Duration.ofMillis(timeout)).readTimeout(Duration.ofMillis(timeout)) + // disable expect continue for HTTP/1.1. + // TODO + .putChannelOption(null, proxyConfig) + // proxy config. + .proxyConfiguration(proxyConfig.build()); + + SdkAsyncHttpClient asyncHttpClient; + if (noVerifySSL) { + asyncHttpClient = asyncHttpClientBuilder.buildWithDefaults(AttributeMap.builder() + .put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, Boolean.TRUE).build()); + } else { + asyncHttpClient = asyncHttpClientBuilder.build(); + } + + client = S3AsyncClient.builder().credentialsProvider(StaticCredentialsProvider.create(awsCreds)) + .endpointOverride(URI.create(endpoint)) + // solve issue: https://github.com/kisscelia/cosbench-ehualu/issues/3 + .region(Region.of(awsRegion)).serviceConfiguration(s3Config).httpClient(asyncHttpClient).build(); + + logger.debug("S3 async client has been initialized"); + + return client; + } + + @Override + public void setAuthContext(AuthContext info) { + super.setAuthContext(info); + } + + @Override + public void dispose() { + super.dispose(); + client = null; + } + + @Override + public InputStream getObject(String container, String object, Config config) { + super.getObject(container, object, config); + InputStream stream = null; + + GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(container).key(object).build(); + CompletableFuture> futureGet = client.getObject(getObjectRequest, + AsyncResponseTransformer.toPublisher()); + futureGet.whenComplete((resp, err) -> { + try { + if (resp != null) { + System.out.println("Object downloaded. Details: " + resp); + } else { + err.printStackTrace(); + + } + } finally { + client.close(); + } + }); + +// try { +// GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(container).key(object).build(); +// CompletedFuture futureGet = client.getObject(getObjectRequest, null); +// +// +//// ResponseBytes objectBytes = client.getObjectAsBytes(getObjectRequest); +// +//// stream = objectBytes.asInputStream(); +// +// } catch (AwsServiceException ase) { +// throw new StorageException(ase); +// } catch (SdkClientException sce) { +// throw new StorageTimeoutException(sce); +// } + + return stream; + } + + @Override + public void restoreObject(String container, String object, Config config) { + super.restoreObject(container, object, config); + try { + // Create and submit a request to restore an object from Glacier/Deep Archive + // for some days. + RestoreRequest restoreRequest = RestoreRequest.builder().days(restoreDays).build(); + + RestoreObjectRequest restoreObjectRequest = RestoreObjectRequest.builder().bucket(container).key(object) + .restoreRequest(restoreRequest).build(); + + client.restoreObject(restoreObjectRequest); + + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void createContainer(String container, Config config) { + super.createContainer(container, config); + try { + container = container.split("/")[0]; + + CreateBucketRequest createBucketRequest = CreateBucketRequest.builder().bucket(container).build(); + client.createBucket(createBucketRequest); + + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void createObject(String container, String object, InputStream data, long length, Config config) { + super.createObject(container, object, data, length, config); + + PutObjectRequest putObjectRequest = PutObjectRequest.builder().contentLength(length) + .contentType("application/octet-stream").storageClass(storageClass).bucket(container).key(object) + .build(); + + // https://github.com/aws/aws-sdk-java-v2/issues/1177 + // V2 async uses NIO APIs and supports non-blocking streaming operations. + // With v2 async client, you need to provide a Publisher instead of InputStream + // (blocking IO) via AsyncRequestBody#fromPublisher. + AsyncRequestBody aRequestBody = AsyncRequestBody.fromPublisher(null); + CompletableFuture future = client.putObject(putObjectRequest, aRequestBody); + + future.whenComplete((resp, err) -> { + try { + if (resp != null) { + // normal flow. + + } else { + // error handler. + + } + } finally { + client.close(); + } + }); + + future.join(); + +// try { +// +// } catch (AwsServiceException ase) { +// throw new StorageException(ase); +// } catch (SdkClientException sce) { +// throw new StorageTimeoutException(sce); +// } + } + + @Override + public void createMultipartObject(String container, String object, InputStream data, long length, Config config) { + super.createMultipartObject(container, object, data, length, config); + +// Map metadata = new HashMap<>(); +// metadata.put("Content-Length", Long.toString(length)); + + CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder() + .contentType("application/octet-stream").storageClass(storageClass).bucket(container).key(object) +// .overrideConfiguration(AwsRequestOverrideConfiguration.builder().putHeader("Content-Length", Long.toString(length)).build()) +// .metadata(metadata) + .build(); + + // Create a list of ETag objects. You retrieve ETags for each object part + // uploaded, + // then, after each individual part has been uploaded, pass the list of ETags to + // the request to complete the upload. + List partETags = new ArrayList(); + + try { + CreateMultipartUploadResponse response = client.createMultipartUpload(createMultipartUploadRequest); + String uploadId = response.uploadId(); + + long position = 0; + + for (int i = 1; position < length; i++) { + // Because the last part could be less than 5 MiB, adjust the part size as + // needed. + partSize = Math.min(partSize, (length - position)); + + // Create the request to upload a part. + UploadPartRequest uploadRequest = UploadPartRequest.builder().contentLength(partSize) // TODO: needed? + .bucket(container).key(object).uploadId(uploadId).partNumber(i).build(); + + // Upload the part and add the response's ETag to our list. + RequestBody requestBody = RequestBody.fromInputStream(data, partSize); + UploadPartResponse uploadPartResponse = client.uploadPart(uploadRequest, requestBody); + + CompletedPart tempPart = CompletedPart.builder().partNumber(i).eTag(uploadPartResponse.eTag()).build(); + partETags.add(tempPart); + + position += partSize; + } + + CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(partETags) + .build(); + + // Complete the multipart upload. + CompleteMultipartUploadRequest compRequest = CompleteMultipartUploadRequest.builder().bucket(container) + .key(object).uploadId(uploadId).multipartUpload(completedMultipartUpload).build(); + client.completeMultipartUpload(compRequest); + + } catch (AwsServiceException ase) { + throw new StorageException(ase); + } catch (SdkClientException sce) { + throw new StorageTimeoutException(sce); + } + } + + @Override + public void deleteContainer(String container, Config config) { + super.deleteContainer(container, config); + try { + container = container.split("/")[0]; + + DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder().bucket(container).build(); + + client.deleteBucket(deleteBucketRequest); + + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void deleteObject(String container, String object, Config config) { + super.deleteObject(container, object, config); + try { + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(container).key(object) + .build(); + + client.deleteObject(deleteObjectRequest); + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public InputStream getList(String container, String object, Config config) { + super.getList(container, object, config); + InputStream stream = null; + try { + + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(container).prefix(object) + .build(); + + CompletableFuture future = client.listObjectsV2(listObjectsV2Request); + + future.whenComplete((resp, err) -> { + try { + if (resp != null) { + + } else { + + } + } finally { + client.close(); + } + }); + + future.join(); + +// stream = new ByteArrayInputStream(result.toString().getBytes()); + } catch (Exception e) { + throw new StorageException(e); + } + + return stream; + } + + @Override + public void headObject(String container, String object, Config config) { + super.headObject(container, object, config); + + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(container).key(object).build(); + + try { + client.headObject(headObjectRequest); + + } catch (Exception e) { + throw new StorageException(e); + } + } +} diff --git a/dev/cosbench-gcs/META-INF/MANIFEST.MF b/dev/cosbench-gcs/META-INF/MANIFEST.MF index 991c1ee..43a9349 100644 --- a/dev/cosbench-gcs/META-INF/MANIFEST.MF +++ b/dev/cosbench-gcs/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Google Cloud Storage Client Bundle Bundle-SymbolicName: cosbench-gcs -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: Intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, diff --git a/dev/cosbench-http/META-INF/MANIFEST.MF b/dev/cosbench-http/META-INF/MANIFEST.MF index d08efcb..dabd207 100644 --- a/dev/cosbench-http/META-INF/MANIFEST.MF +++ b/dev/cosbench-http/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Http Client Utility Bundle Bundle-SymbolicName: cosbench-http -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-httpauth/META-INF/MANIFEST.MF b/dev/cosbench-httpauth/META-INF/MANIFEST.MF index beb8fff..0c4c561 100644 --- a/dev/cosbench-httpauth/META-INF/MANIFEST.MF +++ b/dev/cosbench-httpauth/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Http Authentication Bundle Bundle-SymbolicName: cosbench-httpauth -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 diff --git a/dev/cosbench-keystone/META-INF/MANIFEST.MF b/dev/cosbench-keystone/META-INF/MANIFEST.MF index d9007a8..10f5f6d 100644 --- a/dev/cosbench-keystone/META-INF/MANIFEST.MF +++ b/dev/cosbench-keystone/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Keystone Client Bundle Bundle-SymbolicName: cosbench-keystone -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, diff --git a/dev/cosbench-librados/META-INF/MANIFEST.MF b/dev/cosbench-librados/META-INF/MANIFEST.MF index 2d858cb..b510224 100644 --- a/dev/cosbench-librados/META-INF/MANIFEST.MF +++ b/dev/cosbench-librados/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Librados Storage Client Bundle Bundle-SymbolicName: cosbench-librados -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Bundle-ClassPath: ., diff --git a/dev/cosbench-log/META-INF/MANIFEST.MF b/dev/cosbench-log/META-INF/MANIFEST.MF index 54a58ab..07e6d5c 100644 --- a/dev/cosbench-log/META-INF/MANIFEST.MF +++ b/dev/cosbench-log/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Log Bundle Bundle-SymbolicName: cosbench-log -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Activator: com.intel.cosbench.log.osgi.Activator Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, diff --git a/dev/cosbench-log4j/META-INF/MANIFEST.MF b/dev/cosbench-log4j/META-INF/MANIFEST.MF index dc76e8c..924f70a 100644 --- a/dev/cosbench-log4j/META-INF/MANIFEST.MF +++ b/dev/cosbench-log4j/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Log4j Configuration Bundle Bundle-SymbolicName: cosbench-log4j -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Fragment-Host: com.springsource.org.apache.log4j;bundle-version="[1.0.0,2.0.0)" Bundle-RequiredExecutionEnvironment: JavaSE-1.8, diff --git a/dev/cosbench-mock/META-INF/MANIFEST.MF b/dev/cosbench-mock/META-INF/MANIFEST.MF index 82e1653..82b2577 100644 --- a/dev/cosbench-mock/META-INF/MANIFEST.MF +++ b/dev/cosbench-mock/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Mock Client Bundle Bundle-SymbolicName: cosbench-mock -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, diff --git a/dev/cosbench-openio/META-INF/MANIFEST.MF b/dev/cosbench-openio/META-INF/MANIFEST.MF index 853a572..353e388 100644 --- a/dev/cosbench-openio/META-INF/MANIFEST.MF +++ b/dev/cosbench-openio/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: OpenIO Storage Client Bundle Bundle-SymbolicName: cosbench-openio -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: Intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, diff --git a/dev/cosbench-oss/META-INF/MANIFEST.MF b/dev/cosbench-oss/META-INF/MANIFEST.MF index c4f6613..373eca5 100644 --- a/dev/cosbench-oss/META-INF/MANIFEST.MF +++ b/dev/cosbench-oss/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Cosbench-oss Bundle-SymbolicName: cosbench-oss -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Bundle-ClassPath: ., diff --git a/dev/cosbench-s3/META-INF/MANIFEST.MF b/dev/cosbench-s3/META-INF/MANIFEST.MF index 5fe333b..93f0b1d 100644 --- a/dev/cosbench-s3/META-INF/MANIFEST.MF +++ b/dev/cosbench-s3/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: S3 Storage Client Bundle Bundle-SymbolicName: cosbench-s3 -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.8, JavaSE-1.6 Bundle-ClassPath: ., diff --git a/dev/cosbench-scality/META-INF/MANIFEST.MF b/dev/cosbench-scality/META-INF/MANIFEST.MF index 178146a..db00fb9 100644 --- a/dev/cosbench-scality/META-INF/MANIFEST.MF +++ b/dev/cosbench-scality/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Scality COSBench Client Bundle Bundle-SymbolicName: cosbench-scality -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: Scality Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, diff --git a/dev/cosbench-swauth/META-INF/MANIFEST.MF b/dev/cosbench-swauth/META-INF/MANIFEST.MF index 1ecab03..84565cd 100644 --- a/dev/cosbench-swauth/META-INF/MANIFEST.MF +++ b/dev/cosbench-swauth/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Swauth Client Bundle Bundle-SymbolicName: cosbench-swauth -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, diff --git a/dev/cosbench-swift/META-INF/MANIFEST.MF b/dev/cosbench-swift/META-INF/MANIFEST.MF index 24b9305..9ede726 100644 --- a/dev/cosbench-swift/META-INF/MANIFEST.MF +++ b/dev/cosbench-swift/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Swift Client Bundle Bundle-SymbolicName: cosbench-swift -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Import-Package: com.intel.cosbench.api.auth, diff --git a/dev/cosbench-tomcat/META-INF/MANIFEST.MF b/dev/cosbench-tomcat/META-INF/MANIFEST.MF index aae4855..2a0e491 100644 --- a/dev/cosbench-tomcat/META-INF/MANIFEST.MF +++ b/dev/cosbench-tomcat/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: COSBench Tomcat Bundle Bundle-SymbolicName: cosbench-tomcat -Bundle-Version: 0.4.7.6 +Bundle-Version: 0.4.7.7 Bundle-Activator: com.intel.cosbench.tomcat.osgi.Activator Import-Package: com.intel.cosbench.log, org.apache.catalina;version="[6.0.16,7.0.0)", diff --git a/release/conf/ehualu-config-sample.xml b/release/conf/ehualu-config-sample.xml index d580596..d515d19 100644 --- a/release/conf/ehualu-config-sample.xml +++ b/release/conf/ehualu-config-sample.xml @@ -1,9 +1,74 @@ - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -54,7 +119,7 @@ - + @@ -65,7 +130,7 @@ - + @@ -78,9 +143,9 @@ - +