Mercurial > hg > orthanc-object-storage

comparison Aws/AwsS3StoragePlugin.cpp @ 113:78c075412ab4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
more config for S3 transfer mode
author Alain Mazy <am@osimis.io>
date Tue, 17 Oct 2023 13:01:04 +0200
parents 407bd022b0cf
children e39aa88ec20e
comparison
equal deleted inserted replaced
112:f2c242a6f8ab 113:78c075412ab4
53 std::shared_ptr<Aws::Utils::Threading::Executor> executor_; 53 std::shared_ptr<Aws::Utils::Threading::Executor> executor_;
54 std::shared_ptr<Aws::Transfer::TransferManager> transferManager_; 54 std::shared_ptr<Aws::Transfer::TransferManager> transferManager_;
55 55
56 public: 56 public:
57 57
58 AwsS3StoragePlugin(const std::string& nameForLogs, std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager); 58 AwsS3StoragePlugin(const std::string& nameForLogs, std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager, unsigned int transferThreadPoolSize, unsigned int transferBufferSizeMB);
59 59
60 virtual ~AwsS3StoragePlugin(); 60 virtual ~AwsS3StoragePlugin();
61 61
62 virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); 62 virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled);
63 virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); 63 virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled);
452 if (!caFile.empty()) 452 if (!caFile.empty())
453 { 453 {
454 configuration.caFile = caFile; 454 configuration.caFile = caFile;
455 } 455 }
456 456
457 bool useTransferManager = true; // new in v 2.3.0 457 bool useTransferManager = false; // new in v 2.3.0
458 unsigned int transferPoolSize = 10;
459 unsigned int transferBufferSizeMB = 5;
460
458 pluginSection.LookupBooleanValue(useTransferManager, "UseTransferManager"); 461 pluginSection.LookupBooleanValue(useTransferManager, "UseTransferManager");
462 pluginSection.LookupUnsignedIntegerValue(transferPoolSize, "TransferPoolSize");
463 pluginSection.LookupUnsignedIntegerValue(transferBufferSizeMB, "TransferBufferSize");
464
465
466 std::shared_ptr<Aws::S3::S3Client> client;
459 467
460 if (pluginSection.LookupStringValue(accessKey, "AccessKey") && pluginSection.LookupStringValue(secretKey, "SecretKey")) 468 if (pluginSection.LookupStringValue(accessKey, "AccessKey") && pluginSection.LookupStringValue(secretKey, "SecretKey"))
461 { 469 {
462 OrthancPlugins::LogInfo("AWS S3 Storage: using credentials from the configuration file"); 470 OrthancPlugins::LogInfo("AWS S3 Storage: using credentials from the configuration file");
463 Aws::Auth::AWSCredentials credentials(accessKey.c_str(), secretKey.c_str()); 471 Aws::Auth::AWSCredentials credentials(accessKey.c_str(), secretKey.c_str());
464 472
465 std::shared_ptr<Aws::S3::S3Client> client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing); 473 client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing);
466
467 OrthancPlugins::LogInfo("AWS S3 storage initialized");
468
469 return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager);
470 } 474 }
471 else 475 else
472 { 476 {
473 // when using default credentials, credentials are not checked at startup but only the first time you try to access the bucket ! 477 // when using default credentials, credentials are not checked at startup but only the first time you try to access the bucket !
474 OrthancPlugins::LogInfo("AWS S3 Storage: using default credentials provider"); 478 OrthancPlugins::LogInfo("AWS S3 Storage: using default credentials provider");
475 std::shared_ptr<Aws::S3::S3Client> client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing); 479 client = Aws::MakeShared<Aws::S3::S3Client>(ALLOCATION_TAG, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing);
476
477 OrthancPlugins::LogInfo("AWS S3 storage initialized");
478
479 return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager);
480 } 480 }
481
482 OrthancPlugins::LogInfo("AWS S3 storage initialized");
483
484 return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager, transferPoolSize, transferBufferSizeMB);
481 } 485 }
482 catch (const std::exception& e) 486 catch (const std::exception& e)
483 { 487 {
484 OrthancPlugins::LogError(std::string("AWS S3 Storage plugin: failed to initialize plugin: ") + e.what()); 488 OrthancPlugins::LogError(std::string("AWS S3 Storage plugin: failed to initialize plugin: ") + e.what());
485 return nullptr; 489 return nullptr;
494 Aws::ShutdownAPI(*sdkOptions_); 498 Aws::ShutdownAPI(*sdkOptions_);
495 api_.reset(); 499 api_.reset();
496 } 500 }
497 501
498 502
499 AwsS3StoragePlugin::AwsS3StoragePlugin(const std::string& nameForLogs, std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager) 503 AwsS3StoragePlugin::AwsS3StoragePlugin(const std::string& nameForLogs,
504 std::shared_ptr<Aws::S3::S3Client> client,
505 const std::string& bucketName,
506 bool enableLegacyStorageStructure,
507 bool storageContainsUnknownFiles,
508 bool useTransferManager,
509 unsigned int transferThreadPoolSize,
510 unsigned int transferBufferSizeMB)
500 : BaseStorage(nameForLogs, enableLegacyStorageStructure), 511 : BaseStorage(nameForLogs, enableLegacyStorageStructure),
501 bucketName_(bucketName), 512 bucketName_(bucketName),
502 storageContainsUnknownFiles_(storageContainsUnknownFiles), 513 storageContainsUnknownFiles_(storageContainsUnknownFiles),
503 useTransferManager_(useTransferManager), 514 useTransferManager_(useTransferManager),
504 client_(client) 515 client_(client)
505 { 516 {
506 if (useTransferManager_) 517 if (useTransferManager_)
507 { 518 {
508 executor_ = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>(ALLOCATION_TAG, 10); 519 executor_ = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>(ALLOCATION_TAG, transferThreadPoolSize);
509 Aws::Transfer::TransferManagerConfiguration transferConfig(executor_.get()); 520 Aws::Transfer::TransferManagerConfiguration transferConfig(executor_.get());
510 transferConfig.s3Client = client_; 521 transferConfig.s3Client = client_;
522 transferConfig.bufferSize = static_cast<uint64_t>(transferBufferSizeMB) * 1024 * 1024;
523 transferConfig.transferBufferMaxHeapSize = static_cast<uint64_t>(transferBufferSizeMB) * 1024 * 1024 * transferThreadPoolSize;
511 524
512 transferManager_ = Aws::Transfer::TransferManager::Create(transferConfig); 525 transferManager_ = Aws::Transfer::TransferManager::Create(transferConfig);
513 } 526 }
514 } 527 }
515 528