Mercurial > hg > orthanc-object-storage

changeset 211:082506ba41df

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
AWS: storageclass
author Alain Mazy <am@orthanc.team>
date Tue, 17 Dec 2024 17:55:17 +0100
parents 408c90c9027f
children 5554e508c0c4 8322306f40ac
files Aws/AwsS3StoragePlugin.cpp NEWS
diffstat 2 files changed, 94 insertions(+), 21 deletions(-) [+]
line wrap: on
line diff
--- a/Aws/AwsS3StoragePlugin.cpp	Wed Oct 09 11:48:14 2024 +0200
+++ b/Aws/AwsS3StoragePlugin.cpp	Tue Dec 17 17:55:17 2024 +0100
@@ -59,10 +59,11 @@
   std::shared_ptr<Aws::S3::S3Client>               client_;
   std::shared_ptr<Aws::Utils::Threading::Executor> executor_;
   std::shared_ptr<Aws::Transfer::TransferManager>  transferManager_;
+  Aws::S3::Model::StorageClass                     storageClass_;
 
 public:
 
-  AwsS3StoragePlugin(const std::string& nameForLogs,  std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager, unsigned int transferThreadPoolSize, unsigned int transferBufferSizeMB);
+  AwsS3StoragePlugin(const std::string& nameForLogs,  std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles, bool useTransferManager, unsigned int transferThreadPoolSize, unsigned int transferBufferSizeMB, Aws::S3::Model::StorageClass storageClass);
 
   virtual ~AwsS3StoragePlugin();
 
@@ -79,15 +80,17 @@
 
 class DirectWriter : public IStorage::IWriter
 {
-  std::string             path_;
-  std::shared_ptr<Aws::S3::S3Client>       client_;
-  std::string             bucketName_;
+  std::string                           path_;
+  std::shared_ptr<Aws::S3::S3Client>    client_;
+  std::string                           bucketName_;
+  Aws::S3::Model::StorageClass          storageClass_;
 
 public:
-  DirectWriter(std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, const std::string& path)
+  DirectWriter(std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, const std::string& path, Aws::S3::Model::StorageClass storageClass)
     : path_(path),
       client_(client),
-      bucketName_(bucketName)
+      bucketName_(bucketName),
+      storageClass_(storageClass)
   {
   }
 
@@ -102,6 +105,11 @@
     putObjectRequest.SetBucket(bucketName_.c_str());
     putObjectRequest.SetKey(path_.c_str());
 
+    if (storageClass_ != Aws::S3::Model::StorageClass::NOT_SET)
+    {
+      putObjectRequest.SetStorageClass(storageClass_);
+    }
+
     std::shared_ptr<Aws::StringStream> stream = Aws::MakeShared<Aws::StringStream>(ALLOCATION_TAG, std::ios_base::in | std::ios_base::binary);
 
     stream->rdbuf()->pubsetbuf(const_cast<char*>(data), size);
@@ -124,10 +132,10 @@
 class DirectReader : public IStorage::IReader
 {
 protected:
-  std::shared_ptr<Aws::S3::S3Client>       client_;
-  std::string             bucketName_;
-  std::list<std::string>  paths_;
-  std::string             uuid_;
+  std::shared_ptr<Aws::S3::S3Client>    client_;
+  std::string                           bucketName_;
+  std::list<std::string>                paths_;
+  std::string                           uuid_;
 
 public:
   DirectReader(std::shared_ptr<Aws::S3::S3Client> client, const std::string& bucketName, const std::list<std::string>& paths, const char* uuid)
@@ -271,15 +279,17 @@
 
 class TransferWriter : public IStorage::IWriter
 {
-  std::string             path_;
-  std::shared_ptr<Aws::Transfer::TransferManager>  transferManager_;
-  std::string             bucketName_;
+  std::string                                       path_;
+  std::shared_ptr<Aws::Transfer::TransferManager>   transferManager_;
+  std::string                                       bucketName_;
+  Aws::S3::Model::StorageClass                      storageClass_;
 
 public:
-  TransferWriter(std::shared_ptr<Aws::Transfer::TransferManager> transferManager, const std::string& bucketName, const std::string& path)
+  TransferWriter(std::shared_ptr<Aws::Transfer::TransferManager> transferManager, const std::string& bucketName, const std::string& path, Aws::S3::Model::StorageClass storageClass)
     : path_(path),
       transferManager_(transferManager),
-      bucketName_(bucketName)
+      bucketName_(bucketName),
+      storageClass_(storageClass)
   {
   }
 
@@ -559,11 +569,60 @@
     bool useTransferManager = false; // new in v 2.3.0
     unsigned int transferPoolSize = 10;
     unsigned int transferBufferSizeMB = 5;
+    std::string strStorageClass;
+    Aws::S3::Model::StorageClass storageClass = Aws::S3::Model::StorageClass::NOT_SET;
 
     pluginSection.LookupBooleanValue(useTransferManager, "UseTransferManager");
     pluginSection.LookupUnsignedIntegerValue(transferPoolSize, "TransferPoolSize");
     pluginSection.LookupUnsignedIntegerValue(transferBufferSizeMB, "TransferBufferSize");
-
+    if (pluginSection.LookupStringValue(strStorageClass, "StorageClass"))
+    {
+      if (strStorageClass == "STANDARD")
+      {
+        storageClass = Aws::S3::Model::StorageClass::STANDARD;
+      }
+      else if (strStorageClass == "REDUCED_REDUNDANCY")
+      {
+        storageClass = Aws::S3::Model::StorageClass::REDUCED_REDUNDANCY;
+      }
+      else if (strStorageClass == "STANDARD_IA")
+      {
+        storageClass = Aws::S3::Model::StorageClass::STANDARD_IA;
+      }
+      else if (strStorageClass == "ONEZONE_IA")
+      {
+        storageClass = Aws::S3::Model::StorageClass::ONEZONE_IA;
+      }
+      else if (strStorageClass == "INTELLIGENT_TIERING")
+      {
+        storageClass = Aws::S3::Model::StorageClass::INTELLIGENT_TIERING;
+      }
+      else if (strStorageClass == "GLACIER")
+      {
+        storageClass = Aws::S3::Model::StorageClass::GLACIER;
+      }
+      else if (strStorageClass == "DEEP_ARCHIVE")
+      {
+        storageClass = Aws::S3::Model::StorageClass::DEEP_ARCHIVE;
+      }
+      else if (strStorageClass == "OUTPOSTS")
+      {
+        storageClass = Aws::S3::Model::StorageClass::OUTPOSTS;
+      }
+      else if (strStorageClass == "GLACIER_IR")
+      {
+        storageClass = Aws::S3::Model::StorageClass::GLACIER_IR;
+      }
+      else if (strStorageClass == "SNOW")
+      {
+        storageClass = Aws::S3::Model::StorageClass::SNOW;
+      }
+      else
+      {
+        LOG(ERROR) << "AWS S3 Storage plugin: unrecognized value for \"StorageClass\": " << strStorageClass;
+        return nullptr;
+      }
+    }
 
     std::shared_ptr<Aws::S3::S3Client> client;
 
@@ -583,7 +642,7 @@
 
     LOG(INFO) << "AWS S3 storage initialized";
 
-    return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager, transferPoolSize, transferBufferSizeMB);
+    return new AwsS3StoragePlugin(nameForLogs, client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles, useTransferManager, transferPoolSize, transferBufferSizeMB, storageClass);
   }
   catch (const std::exception& e)
   {
@@ -610,12 +669,14 @@
                                        bool storageContainsUnknownFiles, 
                                        bool useTransferManager,
                                        unsigned int transferThreadPoolSize,
-                                       unsigned int transferBufferSizeMB)
+                                       unsigned int transferBufferSizeMB,
+                                       Aws::S3::Model::StorageClass storageClass)
   : BaseStorage(nameForLogs, enableLegacyStorageStructure),
     bucketName_(bucketName),
     storageContainsUnknownFiles_(storageContainsUnknownFiles),
     useTransferManager_(useTransferManager),
-    client_(client)
+    client_(client),
+    storageClass_(storageClass)
 {
   if (useTransferManager_)
   {
@@ -624,6 +685,10 @@
     transferConfig.s3Client = client_;
     transferConfig.bufferSize = static_cast<uint64_t>(transferBufferSizeMB) * 1024 * 1024;
     transferConfig.transferBufferMaxHeapSize = static_cast<uint64_t>(transferBufferSizeMB) * 1024 * 1024 * transferThreadPoolSize;
+    if (storageClass_ != Aws::S3::Model::StorageClass::NOT_SET)
+    {
+      transferConfig.putObjectTemplate.SetStorageClass(storageClass_);
+    }
 
     transferManager_ = Aws::Transfer::TransferManager::Create(transferConfig);
   }
@@ -633,11 +698,11 @@
 {
   if (useTransferManager_)
   {
-    return new TransferWriter(transferManager_, bucketName_, GetPath(uuid, type, encryptionEnabled));
+    return new TransferWriter(transferManager_, bucketName_, GetPath(uuid, type, encryptionEnabled), storageClass_);
   }
   else
   {
-    return new DirectWriter(client_, bucketName_, GetPath(uuid, type, encryptionEnabled));
+    return new DirectWriter(client_, bucketName_, GetPath(uuid, type, encryptionEnabled), storageClass_);
   }
 }
 
--- a/NEWS	Wed Oct 09 11:48:14 2024 +0200
+++ b/NEWS	Tue Dec 17 17:55:17 2024 +0100
@@ -1,6 +1,14 @@
 Pending changes in the mainline
 ===============================
 
+* AWS plugin:
+  * New configuration "StorageClass" to set the Storage class of the uploaded files.
+    Allowed values are "STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA", 
+    "INTELLIGENT_TIERING", "GLACIER", "DEEP_ARCHIVE", "OUTPOSTS", "GLACIER_IR", "SNOW".
+    Note that, so far, Orthanc has not been tested against "cold" storage classes.
+    If the configuration is not set, Orthanc won't force any value and the default value
+    defined by AWS will be used (the "STANDARD" class at the time of this release).
+
 
 2024-06-26 - v 2.4.0
 ====================