Mercurial > hg > orthanc-object-storage
annotate Aws/AwsS3StoragePlugin.cpp @ 41:d99afdf6d872
ReadRange for AWS
| author | Alain Mazy <am@osimis.io> |
|---|---|
| date | Fri, 19 Mar 2021 10:13:49 +0100 |
| parents | f55b2afdf53d |
| children | 1c3e34f5c5c6 |
| rev | line source |
|---|---|
| 1 | 1 /** |
| 2 * Cloud storage plugins for Orthanc | |
|
37
f55b2afdf53d
upgrade to year 2021
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
33
diff
changeset
|
3 * Copyright (C) 2020-2021 Osimis S.A., Belgium |
| 1 | 4 * |
| 5 * This program is free software: you can redistribute it and/or | |
| 6 * modify it under the terms of the GNU Affero General Public License | |
| 7 * as published by the Free Software Foundation, either version 3 of | |
| 8 * the License, or (at your option) any later version. | |
| 9 * | |
| 10 * This program is distributed in the hope that it will be useful, but | |
| 11 * WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| 13 * Affero General Public License for more details. | |
| 14 * | |
| 15 * You should have received a copy of the GNU Affero General Public License | |
| 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| 17 **/ | |
| 18 | |
| 19 #include "AwsS3StoragePlugin.h" | |
| 20 | |
| 21 #include <aws/core/Aws.h> | |
| 22 #include <aws/s3/S3Client.h> | |
| 23 #include <aws/s3/model/PutObjectRequest.h> | |
| 24 #include <aws/s3/model/GetObjectRequest.h> | |
| 25 #include <aws/s3/model/ListObjectsRequest.h> | |
| 26 #include <aws/s3/model/DeleteObjectRequest.h> | |
| 27 #include <aws/core/auth/AWSCredentialsProvider.h> | |
| 28 #include <aws/core/utils/memory/stl/AWSStringStream.h> | |
| 29 | |
| 30 #include <boost/lexical_cast.hpp> | |
| 31 #include <iostream> | |
| 32 #include <fstream> | |
| 33 | |
| 34 const char* ALLOCATION_TAG = "OrthancS3"; | |
| 15 | 35 static const char* const PLUGIN_SECTION = "AwsS3Storage"; |
| 1 | 36 |
| 15 | 37 class AwsS3StoragePlugin : public BaseStoragePlugin |
| 1 | 38 { |
| 39 public: | |
| 40 | |
| 41 Aws::S3::S3Client client_; | |
| 42 std::string bucketName_; | |
| 43 | |
| 44 public: | |
| 45 | |
| 15 | 46 AwsS3StoragePlugin(const Aws::S3::S3Client& client, const std::string& bucketName, bool enableLegacyStorageStructure); |
| 1 | 47 |
| 48 virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); | |
| 49 virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); | |
| 50 virtual void DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); | |
| 15 | 51 virtual const char* GetConfigurationSectionName() {return PLUGIN_SECTION;}; |
| 1 | 52 }; |
| 53 | |
| 54 | |
| 55 class Writer : public IStoragePlugin::IWriter | |
| 56 { | |
| 57 std::string path_; | |
| 58 Aws::S3::S3Client client_; | |
| 59 std::string bucketName_; | |
| 60 | |
| 61 public: | |
| 62 Writer(const Aws::S3::S3Client& client, const std::string& bucketName, const std::string& path) | |
| 63 : path_(path), | |
| 64 client_(client), | |
| 65 bucketName_(bucketName) | |
| 66 { | |
| 67 } | |
| 68 | |
| 69 virtual ~Writer() | |
| 70 { | |
| 71 } | |
| 72 | |
| 73 virtual void Write(const char* data, size_t size) | |
| 74 { | |
| 75 Aws::S3::Model::PutObjectRequest putObjectRequest; | |
| 76 | |
| 77 putObjectRequest.SetBucket(bucketName_.c_str()); | |
| 78 putObjectRequest.SetKey(path_.c_str()); | |
| 79 | |
| 80 std::shared_ptr<Aws::StringStream> stream = Aws::MakeShared<Aws::StringStream>(ALLOCATION_TAG, std::ios_base::in | std::ios_base::binary); | |
| 81 | |
| 82 stream->rdbuf()->pubsetbuf(const_cast<char*>(data), size); | |
| 83 stream->rdbuf()->pubseekpos(size); | |
| 84 stream->seekg(0); | |
| 85 | |
| 86 putObjectRequest.SetBody(stream); | |
| 87 | |
| 88 auto result = client_.PutObject(putObjectRequest); | |
| 89 | |
| 90 if (!result.IsSuccess()) | |
| 91 { | |
| 92 throw StoragePluginException(std::string("error while writing file ") + path_ + ": " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str()); | |
| 93 } | |
| 94 } | |
| 95 }; | |
| 96 | |
| 97 | |
| 98 class Reader : public IStoragePlugin::IReader | |
| 99 { | |
| 100 std::string path_; | |
| 101 Aws::S3::S3Client client_; | |
| 102 std::string bucketName_; | |
| 103 | |
| 104 public: | |
| 105 Reader(const Aws::S3::S3Client& client, const std::string& bucketName, const std::string& path) | |
| 106 : path_(path), | |
| 107 client_(client), | |
| 108 bucketName_(bucketName) | |
| 109 { | |
| 110 } | |
| 111 | |
| 112 virtual ~Reader() | |
| 113 { | |
| 114 | |
| 115 } | |
| 116 virtual size_t GetSize() | |
| 117 { | |
| 118 Aws::S3::Model::ListObjectsRequest listObjectRequest; | |
| 119 listObjectRequest.SetBucket(bucketName_.c_str()); | |
| 120 listObjectRequest.SetPrefix(path_.c_str()); | |
| 121 | |
| 122 auto result = client_.ListObjects(listObjectRequest); | |
| 123 | |
| 124 if (result.IsSuccess()) | |
| 125 { | |
| 126 Aws::Vector<Aws::S3::Model::Object> objectList = | |
| 127 result.GetResult().GetContents(); | |
| 128 | |
| 129 if (objectList.size() == 1) | |
| 130 { | |
| 131 return objectList[0].GetSize(); | |
| 132 } | |
| 32 | 133 else if (objectList.size() > 1) |
| 134 { | |
| 135 throw StoragePluginException(std::string("error while reading file ") + path_ + ": multiple objet with same name !"); | |
| 136 } | |
| 137 throw StoragePluginException(std::string("error while reading file ") + path_ + ": object not found !"); | |
| 1 | 138 } |
| 139 else | |
| 140 { | |
| 141 throw StoragePluginException(std::string("error while reading file ") + path_ + ": " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str()); | |
| 142 } | |
| 143 } | |
| 41 | 144 |
| 145 virtual void ReadWhole(char* data, size_t size) | |
| 146 { | |
| 147 _Read(data, size, 0, false); | |
| 148 } | |
| 149 | |
| 150 virtual void ReadRange(char* data, size_t size, size_t fromOffset) | |
| 151 { | |
| 152 _Read(data, size, fromOffset, true); | |
| 153 } | |
| 154 | |
| 155 void _Read(char* data, size_t size, size_t fromOffset, bool useRange) | |
| 1 | 156 { |
| 157 Aws::S3::Model::GetObjectRequest getObjectRequest; | |
| 158 getObjectRequest.SetBucket(bucketName_.c_str()); | |
| 159 getObjectRequest.SetKey(path_.c_str()); | |
| 160 | |
| 41 | 161 if (useRange) |
| 162 { | |
| 163 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests | |
| 164 std::string range = std::string("bytes=") + boost::lexical_cast<std::string>(fromOffset) + "-" + boost::lexical_cast<std::string>(fromOffset + size -1); | |
| 165 getObjectRequest.SetRange(range.c_str()); | |
| 166 } | |
| 167 | |
| 1 | 168 getObjectRequest.SetResponseStreamFactory( |
| 169 [data, size]() | |
| 170 { | |
| 171 std::unique_ptr<Aws::StringStream> | |
| 172 istream(Aws::New<Aws::StringStream>(ALLOCATION_TAG)); | |
| 173 | |
| 174 istream->rdbuf()->pubsetbuf(static_cast<char*>(data), | |
| 175 size); | |
| 176 | |
| 177 return istream.release(); | |
| 178 }); | |
| 179 | |
| 180 // Get the object | |
| 181 auto result = client_.GetObject(getObjectRequest); | |
| 182 if (result.IsSuccess()) | |
| 183 { | |
| 184 } | |
| 185 else | |
| 186 { | |
| 187 throw StoragePluginException(std::string("error while reading file ") + path_ + ": " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str()); | |
| 188 } | |
| 189 } | |
| 190 | |
| 191 }; | |
| 192 | |
| 193 | |
| 194 | |
| 41 | 195 |
| 196 | |
| 1 | 197 const char* AwsS3StoragePluginFactory::GetStoragePluginName() |
| 198 { | |
| 199 return "AWS S3 Storage"; | |
| 200 } | |
| 201 | |
| 202 IStoragePlugin* AwsS3StoragePluginFactory::CreateStoragePlugin(const OrthancPlugins::OrthancConfiguration& orthancConfig) | |
| 203 { | |
| 15 | 204 bool enableLegacyStorageStructure; |
| 205 | |
| 1 | 206 if (!orthancConfig.IsSection(PLUGIN_SECTION)) |
| 207 { | |
| 208 OrthancPlugins::LogWarning(std::string(GetStoragePluginName()) + " plugin, section missing. Plugin is not enabled."); | |
| 209 return nullptr; | |
| 210 } | |
| 211 | |
| 212 OrthancPlugins::OrthancConfiguration pluginSection; | |
| 213 orthancConfig.GetSection(pluginSection, PLUGIN_SECTION); | |
| 214 | |
| 15 | 215 if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, pluginSection)) |
| 216 { | |
| 217 return nullptr; | |
| 218 } | |
| 219 | |
| 1 | 220 std::string bucketName; |
| 221 std::string region; | |
| 222 std::string accessKey; | |
| 223 std::string secretKey; | |
| 224 | |
| 225 if (!pluginSection.LookupStringValue(bucketName, "BucketName")) | |
| 226 { | |
| 227 OrthancPlugins::LogError("AwsS3Storage/BucketName configuration missing. Unable to initialize plugin"); | |
| 228 return nullptr; | |
| 229 } | |
| 230 | |
| 231 if (!pluginSection.LookupStringValue(region, "Region")) | |
| 232 { | |
| 233 OrthancPlugins::LogError("AwsS3Storage/Region configuration missing. Unable to initialize plugin"); | |
| 234 return nullptr; | |
| 235 } | |
| 236 | |
| 237 if (!pluginSection.LookupStringValue(accessKey, "AccessKey")) | |
| 238 { | |
| 239 OrthancPlugins::LogError("AwsS3Storage/AccessKey configuration missing. Unable to initialize plugin"); | |
| 240 return nullptr; | |
| 241 } | |
| 242 | |
| 243 if (!pluginSection.LookupStringValue(secretKey, "SecretKey")) | |
| 244 { | |
| 245 OrthancPlugins::LogError("AwsS3Storage/SecretKey configuration missing. Unable to initialize plugin"); | |
| 246 return nullptr; | |
| 247 } | |
| 248 | |
|
6
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
249 std::string endpoint = pluginSection.GetStringValue("Endpoint", ""); |
|
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
250 unsigned int connectTimeout = pluginSection.GetUnsignedIntegerValue("ConnectTimeout", 30); |
|
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
251 unsigned int requestTimeout = pluginSection.GetUnsignedIntegerValue("RequestTimeout", 1200); |
|
27
e1f52b851827
Added "VirtualAddressing" configuration option in the AWS S3 plugin (for compatibility with minio)
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
15
diff
changeset
|
252 bool virtualAddressing = pluginSection.GetBooleanValue("VirtualAddressing", true); |
|
e1f52b851827
Added "VirtualAddressing" configuration option in the AWS S3 plugin (for compatibility with minio)
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
15
diff
changeset
|
253 |
| 1 | 254 try |
| 255 { | |
| 256 Aws::SDKOptions options; | |
| 257 Aws::InitAPI(options); | |
| 258 | |
| 259 Aws::Auth::AWSCredentials credentials(accessKey.c_str(), secretKey.c_str()); | |
| 260 Aws::Client::ClientConfiguration configuration; | |
| 261 configuration.region = region.c_str(); | |
|
6
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
262 configuration.scheme = Aws::Http::Scheme::HTTPS; |
|
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
263 configuration.connectTimeoutMs = connectTimeout * 1000; |
|
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
264 configuration.requestTimeoutMs = requestTimeout * 1000; |
|
27
e1f52b851827
Added "VirtualAddressing" configuration option in the AWS S3 plugin (for compatibility with minio)
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
15
diff
changeset
|
265 configuration.httpRequestTimeoutMs = requestTimeout * 1000; |
|
6
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
266 |
|
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
267 if (!endpoint.empty()) |
|
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
268 { |
|
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
269 configuration.endpointOverride = endpoint.c_str(); |
|
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
270 } |
|
393fcf337462
AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents:
1
diff
changeset
|
271 |
|
27
e1f52b851827
Added "VirtualAddressing" configuration option in the AWS S3 plugin (for compatibility with minio)
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
15
diff
changeset
|
272 Aws::S3::S3Client client(credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing); |
| 1 | 273 |
| 274 OrthancPlugins::LogInfo("AWS S3 storage initialized"); | |
| 275 | |
| 15 | 276 return new AwsS3StoragePlugin(client, bucketName, enableLegacyStorageStructure); |
| 1 | 277 } |
| 278 catch (const std::exception& e) | |
| 279 { | |
| 280 OrthancPlugins::LogError(std::string("AzureBlobStorage plugin: failed to initialize plugin: ") + e.what()); | |
| 281 return nullptr; | |
| 282 } | |
| 283 | |
| 284 } | |
| 285 | |
| 15 | 286 AwsS3StoragePlugin::AwsS3StoragePlugin(const Aws::S3::S3Client& client, const std::string& bucketName, bool enableLegacyStorageStructure) |
| 287 : BaseStoragePlugin(enableLegacyStorageStructure), | |
| 288 client_(client), | |
| 1 | 289 bucketName_(bucketName) |
| 290 { | |
| 291 | |
| 292 } | |
| 293 | |
| 294 IStoragePlugin::IWriter* AwsS3StoragePlugin::GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
| 295 { | |
| 296 return new Writer(client_, bucketName_, GetPath(uuid, type, encryptionEnabled)); | |
| 297 } | |
| 298 | |
| 299 IStoragePlugin::IReader* AwsS3StoragePlugin::GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
| 300 { | |
| 301 return new Reader(client_, bucketName_, GetPath(uuid, type, encryptionEnabled)); | |
| 302 } | |
| 303 | |
| 304 void AwsS3StoragePlugin::DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
| 305 { | |
| 306 std::string path = GetPath(uuid, type, encryptionEnabled); | |
| 307 | |
| 308 Aws::S3::Model::DeleteObjectRequest deleteObjectRequest; | |
| 309 deleteObjectRequest.SetBucket(bucketName_.c_str()); | |
| 310 deleteObjectRequest.SetKey(path.c_str()); | |
| 311 | |
| 312 auto result = client_.DeleteObject(deleteObjectRequest); | |
| 313 | |
| 314 if (!result.IsSuccess()) | |
| 315 { | |
| 316 throw StoragePluginException(std::string("error while deleting file ") + path + ": " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str()); | |
| 317 } | |
| 318 | |
| 319 } |