Mercurial > hg > orthanc-object-storage
changeset 113:78c075412ab4
more config for S3 transfer mode
author | Alain Mazy <am@osimis.io> |
---|---|
date | Tue, 17 Oct 2023 13:01:04 +0200 |
parents | f2c242a6f8ab |
children | 752ab0c59950 ca68456d789a |
files | Aws/AwsS3StoragePlugin.cpp NEWS |
diffstat | 2 files changed, 31 insertions(+), 14 deletions(-) [+] |
line wrap: on
line diff
--- a/Aws/AwsS3StoragePlugin.cpp Wed Oct 11 09:56:22 2023 +0200 +++ b/Aws/AwsS3StoragePlugin.cpp Tue Oct 17 13:01:04 2023 +0200 @@ -55,7 +55,7 @@ public: - AwsS3StoragePlugin(const std::string& nameForLogs, std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager); + AwsS3StoragePlugin(const std::string& nameForLogs, std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager, unsigned int transferThreadPoolSize, unsigned int transferBufferSizeMB); virtual ~AwsS3StoragePlugin(); @@ -454,30 +454,34 @@ configuration.caFile = caFile; } - bool useTransferManager = true; // new in v 2.3.0 + bool useTransferManager = false; // new in v 2.3.0 + unsigned int transferPoolSize = 10; + unsigned int transferBufferSizeMB = 5; + pluginSection.LookupBooleanValue(useTransferManager, "UseTransferManager"); + pluginSection.LookupUnsignedIntegerValue(transferPoolSize, "TransferPoolSize"); + pluginSection.LookupUnsignedIntegerValue(transferBufferSizeMB, "TransferBufferSize"); + + + std::shared_ptr<Aws::S3::S3Client> client; if (pluginSection.LookupStringValue(accessKey, "AccessKey") && pluginSection.LookupStringValue(secretKey, "SecretKey")) { OrthancPlugins::LogInfo("AWS S3 Storage: using credentials from the configuration file"); Aws::Auth::AWSCredentials credentials(accessKey.c_str(), secretKey.c_str()); - std::shared_ptr<Aws::S3::S3Client> client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing); - - OrthancPlugins::LogInfo("AWS S3 storage initialized"); - - return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager); + client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing); } else { // when using default credentials, credentials are not checked at startup but only the first time you try to access the bucket ! OrthancPlugins::LogInfo("AWS S3 Storage: using default credentials provider"); - std::shared_ptr<Aws::S3::S3Client> client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing); + client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing); + } - OrthancPlugins::LogInfo("AWS S3 storage initialized"); + OrthancPlugins::LogInfo("AWS S3 storage initialized"); - return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager); - } + return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager, transferPoolSize, transferBufferSizeMB); } catch (const std::exception& e) { @@ -496,7 +500,14 @@ } -AwsS3StoragePlugin::AwsS3StoragePlugin(const std::string& nameForLogs, std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager) +AwsS3StoragePlugin::AwsS3StoragePlugin(const std::string& nameForLogs, + std::shared_ptr<Aws::S3::S3Client> client, + const std::string& bucketName, + bool enableLegacyStorageStructure, + bool storageContainsUnknownFiles, + bool useTransferManager, + unsigned int transferThreadPoolSize, + unsigned int transferBufferSizeMB) : BaseStorage(nameForLogs, enableLegacyStorageStructure), bucketName_(bucketName), storageContainsUnknownFiles_(storageContainsUnknownFiles), @@ -505,9 +516,11 @@ { if (useTransferManager_) { - executor_ = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>(ALLOCATION_TAG, 10); + executor_ = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>(ALLOCATION_TAG, transferThreadPoolSize); Aws::Transfer::TransferManagerConfiguration transferConfig(executor_.get()); transferConfig.s3Client = client_; + transferConfig.bufferSize = static_cast<uint64_t>(transferBufferSizeMB) * 1024 * 1024; + transferConfig.transferBufferMaxHeapSize = static_cast<uint64_t>(transferBufferSizeMB) * 1024 * 1024 * transferThreadPoolSize; transferManager_ = Aws::Transfer::TransferManager::Create(transferConfig); }
--- a/NEWS Wed Oct 11 09:56:22 2023 +0200 +++ b/NEWS Tue Oct 17 13:01:04 2023 +0200 @@ -2,11 +2,15 @@ =============================== * AWS plugin: - * Added a new configuration "UseTransferMode" (true by default). + * Added a new configuration "UseTransferMode" (false by default). When set to true, the Transfer Manager mode is used to upload/download whole files to/from the bucket. If set to false, the default "object" mode is used. The Transfer Manager mode is supposed to be faster, especially for large files. + 2 related configurations have been added too: + - "TransferPoolSize" to define the number of threads to be used by + the Transfer Manager (default = 10) + - "TransferBufferSize" to define the size (in MB) * now using the latest AWS SDK C++ (1.11.178). * in /move-storage: now detecting if file should be moved or not.