# HG changeset patch
# User Alain Mazy <am@osimis.io>
# Date 1697540464 -7200
# Node ID 78c075412ab4f07f1d193e2493d9043a77034374
# Parent  f2c242a6f8ab1240a9fccde15f49ff29732e0af9
more config for S3 transfer mode

diff -r f2c242a6f8ab -r 78c075412ab4 Aws/AwsS3StoragePlugin.cpp
--- a/Aws/AwsS3StoragePlugin.cpp	Wed Oct 11 09:56:22 2023 +0200
+++ b/Aws/AwsS3StoragePlugin.cpp	Tue Oct 17 13:01:04 2023 +0200
@@ -55,7 +55,7 @@
 
 public:
 
-  AwsS3StoragePlugin(const std::string& nameForLogs,  std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager);
+  AwsS3StoragePlugin(const std::string& nameForLogs,  std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager, unsigned int transferThreadPoolSize, unsigned int transferBufferSizeMB);
 
   virtual ~AwsS3StoragePlugin();
 
@@ -454,30 +454,34 @@
       configuration.caFile = caFile;
     }
     
-    bool useTransferManager = true; // new in v 2.3.0
+    bool useTransferManager = false; // new in v 2.3.0
+    unsigned int transferPoolSize = 10;
+    unsigned int transferBufferSizeMB = 5;
+
     pluginSection.LookupBooleanValue(useTransferManager, "UseTransferManager");
+    pluginSection.LookupUnsignedIntegerValue(transferPoolSize, "TransferPoolSize");
+    pluginSection.LookupUnsignedIntegerValue(transferBufferSizeMB, "TransferBufferSize");
+
+
+    std::shared_ptr<Aws::S3::S3Client> client;
 
     if (pluginSection.LookupStringValue(accessKey, "AccessKey") && pluginSection.LookupStringValue(secretKey, "SecretKey"))
     {
       OrthancPlugins::LogInfo("AWS S3 Storage: using credentials from the configuration file");
       Aws::Auth::AWSCredentials credentials(accessKey.c_str(), secretKey.c_str());
       
-      std::shared_ptr<Aws::S3::S3Client> client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing);
-      
-      OrthancPlugins::LogInfo("AWS S3 storage initialized");
-
-      return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager);
+      client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing);
     } 
     else
     {
       // when using default credentials, credentials are not checked at startup but only the first time you try to access the bucket !
       OrthancPlugins::LogInfo("AWS S3 Storage: using default credentials provider");
-      std::shared_ptr<Aws::S3::S3Client> client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing);
+      client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing);
+    }  
 
-      OrthancPlugins::LogInfo("AWS S3 storage initialized");
+    OrthancPlugins::LogInfo("AWS S3 storage initialized");
 
-      return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager);
-    }  
+    return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager, transferPoolSize, transferBufferSizeMB);
   }
   catch (const std::exception& e)
   {
@@ -496,7 +500,14 @@
 }
 
 
-AwsS3StoragePlugin::AwsS3StoragePlugin(const std::string& nameForLogs, std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager)
+AwsS3StoragePlugin::AwsS3StoragePlugin(const std::string& nameForLogs, 
+                                       std::shared_ptr<Aws::S3::S3Client> client, 
+                                       const std::string& bucketName, 
+                                       bool enableLegacyStorageStructure, 
+                                       bool storageContainsUnknownFiles, 
+                                       bool useTransferManager,
+                                       unsigned int transferThreadPoolSize,
+                                       unsigned int transferBufferSizeMB)
   : BaseStorage(nameForLogs, enableLegacyStorageStructure),
     bucketName_(bucketName),
     storageContainsUnknownFiles_(storageContainsUnknownFiles),
@@ -505,9 +516,11 @@
 {
   if (useTransferManager_)
   {
-    executor_ = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>(ALLOCATION_TAG, 10);
+    executor_ = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>(ALLOCATION_TAG, transferThreadPoolSize);
     Aws::Transfer::TransferManagerConfiguration transferConfig(executor_.get());
     transferConfig.s3Client = client_;
+    transferConfig.bufferSize = static_cast<uint64_t>(transferBufferSizeMB) * 1024 * 1024;
+    transferConfig.transferBufferMaxHeapSize = static_cast<uint64_t>(transferBufferSizeMB) * 1024 * 1024 * transferThreadPoolSize;
 
     transferManager_ = Aws::Transfer::TransferManager::Create(transferConfig);
   }
diff -r f2c242a6f8ab -r 78c075412ab4 NEWS
--- a/NEWS	Wed Oct 11 09:56:22 2023 +0200
+++ b/NEWS	Tue Oct 17 13:01:04 2023 +0200
@@ -2,11 +2,15 @@
 ===============================
 
 * AWS plugin:
-  * Added a new configuration "UseTransferMode" (true by default).
+  * Added a new configuration "UseTransferMode" (false by default).
     When set to true, the Transfer Manager mode is used to upload/download
     whole files to/from the bucket.  If set to false, the default "object"
     mode is used.  The Transfer Manager mode is supposed to be faster,
     especially for large files.
+    2 related configurations have been added too:
+    - "TransferPoolSize" to define the number of threads to be used by
+      the Transfer Manager (default = 10)
+    - "TransferBufferSize" to define the size (in MB) 
   * now using the latest AWS SDK C++ (1.11.178).
 
 * in /move-storage: now detecting if file should be moved or not.