Mercurial > hg > orthanc-object-storage
annotate Azure/AzureBlobStoragePlugin.cpp @ 68:58a03fce4897
new option for AWS: EnableLegacyUnknownFiles
author | Alain Mazy <am@osimis.io> |
---|---|
date | Fri, 09 Jul 2021 15:40:07 +0200 |
parents | b922ae86bbe1 |
children | 16e419fe80c5 |
rev | line source |
---|---|
1 | 1 /** |
2 * Cloud storage plugins for Orthanc | |
37
f55b2afdf53d
upgrade to year 2021
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
34
diff
changeset
|
3 * Copyright (C) 2020-2021 Osimis S.A., Belgium |
1 | 4 * |
5 * This program is free software: you can redistribute it and/or | |
6 * modify it under the terms of the GNU Affero General Public License | |
7 * as published by the Free Software Foundation, either version 3 of | |
8 * the License, or (at your option) any later version. | |
9 * | |
10 * This program is distributed in the hope that it will be useful, but | |
11 * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 * Affero General Public License for more details. | |
14 * | |
15 * You should have received a copy of the GNU Affero General Public License | |
16 * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
17 **/ | |
18 | |
56
b922ae86bbe1
full static linking against AWS SDK
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
48
diff
changeset
|
19 |
1 | 20 #include "AzureBlobStoragePlugin.h" |
21 | |
22 #include <was/storage_account.h> | |
23 #include <was/blob.h> | |
24 #include <boost/lexical_cast.hpp> | |
12
a203d0a4fd8f
Azure: now removing spaces and CR at the end of the connectionString since it prevented the plugin to initialize.
Alain Mazy
parents:
11
diff
changeset
|
25 #include <boost/algorithm/string.hpp> |
1 | 26 #include "cpprest/rawptrstream.h" |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
27 #include "cpprest/details/basic_types.h" |
1 | 28 |
29 | |
30 // Create aliases to make the code easier to read. | |
31 namespace as = azure::storage; | |
15 | 32 static const char* const PLUGIN_SECTION = "AzureBlobStorage"; |
1 | 33 |
15 | 34 class AzureBlobStoragePlugin : public BaseStoragePlugin |
1 | 35 { |
36 public: | |
37 | |
38 as::cloud_blob_client blobClient_; | |
39 as::cloud_blob_container blobContainer_; | |
40 public: | |
41 | |
42 // AzureBlobStoragePlugin(const std::string& connectionString, | |
43 // const std::string& containerName | |
44 // ); | |
15 | 45 AzureBlobStoragePlugin(const as::cloud_blob_client& blobClient, const as::cloud_blob_container& blobContainer, bool enableLegacyStorageStructure); |
1 | 46 |
47 virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); | |
48 virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); | |
49 virtual void DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); | |
15 | 50 virtual const char* GetConfigurationSectionName() {return PLUGIN_SECTION;}; |
1 | 51 }; |
52 | |
53 | |
54 class Writer : public IStoragePlugin::IWriter | |
55 { | |
56 std::string path_; | |
57 as::cloud_blob_client client_; | |
58 as::cloud_blob_container container_; | |
59 | |
60 public: | |
61 Writer(const as::cloud_blob_container& container, const std::string& path, const as::cloud_blob_client& client) | |
62 : path_(path), | |
63 client_(client), | |
64 container_(container) | |
65 { | |
66 } | |
67 | |
68 virtual ~Writer() | |
69 { | |
70 } | |
71 | |
72 virtual void Write(const char* data, size_t size) | |
73 { | |
74 try | |
75 { | |
76 concurrency::streams::istream inputStream = concurrency::streams::rawptr_stream<uint8_t>::open_istream(reinterpret_cast<const uint8_t*>(data), size); | |
34 | 77 azure::storage::cloud_block_blob blob = container_.get_block_blob_reference(utility::conversions::to_string_t(path_)); |
1 | 78 blob.upload_from_stream(inputStream); |
79 inputStream.close().wait(); | |
80 } | |
81 catch (std::exception& ex) | |
82 { | |
83 throw StoragePluginException("AzureBlobStorage: error writing file " + std::string(path_) + ": " + ex.what()); | |
84 } | |
85 } | |
86 }; | |
87 | |
88 | |
89 class Reader : public IStoragePlugin::IReader | |
90 { | |
91 std::string path_; | |
92 as::cloud_blob_client client_; | |
93 as::cloud_blob_container container_; | |
94 as::cloud_block_blob block_; | |
95 | |
96 public: | |
97 Reader(const as::cloud_blob_container& container, const std::string& path, const as::cloud_blob_client& client) | |
98 : path_(path), | |
99 client_(client), | |
100 container_(container) | |
101 { | |
102 try | |
103 { | |
34 | 104 block_ = container_.get_block_blob_reference(utility::conversions::to_string_t(path_)); |
1 | 105 block_.download_attributes(); // to retrieve the properties |
106 } | |
107 catch (std::exception& ex) | |
108 { | |
109 throw StoragePluginException("AzureBlobStorage: error opening file for reading " + std::string(path_) + ": " + ex.what()); | |
110 } | |
111 } | |
112 | |
113 virtual ~Reader() | |
114 { | |
115 | |
116 } | |
117 virtual size_t GetSize() | |
118 { | |
119 try | |
120 { | |
121 return block_.properties().size(); | |
122 } | |
123 catch (std::exception& ex) | |
124 { | |
125 throw StoragePluginException("AzureBlobStorage: error while reading file " + std::string(path_) + ": " + ex.what()); | |
126 } | |
127 } | |
128 | |
39
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
129 virtual void ReadWhole(char* data, size_t size) |
1 | 130 { |
131 try | |
132 { | |
133 concurrency::streams::ostream outputStream = concurrency::streams::rawptr_stream<uint8_t>::open_ostream(reinterpret_cast<uint8_t*>(data), size); | |
134 | |
135 block_.download_to_stream(outputStream); | |
136 } | |
137 catch (std::exception& ex) | |
138 { | |
139 throw StoragePluginException("AzureBlobStorage: error while reading file " + std::string(path_) + ": " + ex.what()); | |
140 } | |
141 } | |
142 | |
39
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
143 virtual void ReadRange(char* data, size_t size, size_t fromOffset) |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
144 { |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
145 try |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
146 { |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
147 concurrency::streams::ostream outputStream = concurrency::streams::rawptr_stream<uint8_t>::open_ostream(reinterpret_cast<uint8_t*>(data), size); |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
148 |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
149 block_.download_range_to_stream(outputStream, fromOffset, size); |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
150 } |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
151 catch (std::exception& ex) |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
152 { |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
153 throw StoragePluginException("AzureBlobStorage: error while reading partial file " + std::string(path_) + ": " + ex.what()); |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
154 } |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
155 } |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
156 |
1 | 157 }; |
158 | |
159 | |
160 | |
161 const char* AzureBlobStoragePluginFactory::GetStoragePluginName() | |
162 { | |
163 return "Azure Blob Storage"; | |
164 } | |
165 | |
48 | 166 bool IsSasTokenAccountLevel(utility::string_t sasToken) |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
167 { |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
168 // Use cpprestsdk's utility::string_t here since the expected argument is the return value of |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
169 // as::storage_credentials::sas_token(), which is type utility::string_t |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
170 size_t newIdx = 0; |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
171 size_t prevIdx = 0; |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
172 while ((newIdx = sasToken.find('&', prevIdx)) != utility::string_t::npos) |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
173 { |
48 | 174 utility::string_t kvpair = sasToken.substr(prevIdx, newIdx - prevIdx); |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
175 prevIdx = newIdx + 1; // start next time from char after '&' |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
176 |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
177 size_t equalsIdx = kvpair.find('='); |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
178 utility::string_t key = kvpair.substr(0, equalsIdx); |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
179 if (key == "srt") // only account SAS has this parameter |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
180 return true; |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
181 } |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
182 |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
183 return false; |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
184 } |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
185 |
1 | 186 IStoragePlugin* AzureBlobStoragePluginFactory::CreateStoragePlugin(const OrthancPlugins::OrthancConfiguration& orthancConfig) |
187 { | |
188 std::string connectionString; | |
189 std::string containerName; | |
15 | 190 bool enableLegacyStorageStructure; |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
191 bool createContainerIfNotExists; |
1 | 192 |
193 if (orthancConfig.IsSection(PLUGIN_SECTION)) | |
194 { | |
195 OrthancPlugins::OrthancConfiguration pluginSection; | |
196 orthancConfig.GetSection(pluginSection, PLUGIN_SECTION); | |
197 | |
15 | 198 if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, pluginSection)) |
199 { | |
200 return nullptr; | |
201 } | |
202 | |
1 | 203 if (!pluginSection.LookupStringValue(connectionString, "ConnectionString")) |
204 { | |
205 OrthancPlugins::LogError("AzureBlobStorage/ConnectionString configuration missing. Unable to initialize plugin"); | |
206 return nullptr; | |
207 } | |
208 | |
209 if (!pluginSection.LookupStringValue(containerName, "ContainerName")) | |
210 { | |
211 OrthancPlugins::LogError("AzureBlobStorage/ContainerName configuration missing. Unable to initialize plugin"); | |
212 return nullptr; | |
213 } | |
214 | |
12
a203d0a4fd8f
Azure: now removing spaces and CR at the end of the connectionString since it prevented the plugin to initialize.
Alain Mazy
parents:
11
diff
changeset
|
215 boost::trim(connectionString); // without that, if there's an EOL in the string, it fails with "provided uri is invalid" |
a203d0a4fd8f
Azure: now removing spaces and CR at the end of the connectionString since it prevented the plugin to initialize.
Alain Mazy
parents:
11
diff
changeset
|
216 boost::trim(containerName); |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
217 |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
218 createContainerIfNotExists = pluginSection.GetBooleanValue("CreateContainerIfNotExists", true); |
1 | 219 } |
220 else if (orthancConfig.IsSection("BlobStorage")) // backward compatibility with the old plugin: | |
221 { | |
222 OrthancPlugins::LogWarning("AzureBlobStorage: you're using an old configuration format for the plugin."); | |
223 | |
224 OrthancPlugins::OrthancConfiguration pluginSection; | |
225 orthancConfig.GetSection(pluginSection, "BlobStorage"); | |
226 | |
227 std::string accountName; | |
228 std::string accountKey; | |
229 | |
230 if (!pluginSection.LookupStringValue(containerName, "ContainerName")) | |
231 { | |
232 OrthancPlugins::LogError("BlobStorage/AccountName configuration missing. Unable to initialize plugin"); | |
233 return nullptr; | |
234 } | |
235 | |
236 if (!pluginSection.LookupStringValue(accountName, "AccountName")) | |
237 { | |
238 OrthancPlugins::LogError("BlobStorage/AccountName configuration missing. Unable to initialize plugin"); | |
239 return nullptr; | |
240 } | |
241 | |
242 if (!pluginSection.LookupStringValue(accountKey, "AccountKey")) | |
243 { | |
244 OrthancPlugins::LogError("BlobStorage/ContainerName configuration missing. Unable to initialize plugin"); | |
245 return nullptr; | |
246 } | |
247 | |
248 std::ostringstream connectionStringBuilder; | |
249 connectionStringBuilder << "DefaultEndpointsProtocol=https;AccountName=" << accountName << ";AccountKey=" << accountKey; | |
250 connectionString = connectionStringBuilder.str(); | |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
251 |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
252 createContainerIfNotExists = pluginSection.GetBooleanValue("CreateContainerIfNotExists", true); |
1 | 253 } |
254 else | |
255 { | |
256 OrthancPlugins::LogWarning(std::string(GetStoragePluginName()) + " plugin, section missing. Plugin is not enabled."); | |
257 return nullptr; | |
258 } | |
259 | |
260 try | |
261 { | |
11 | 262 OrthancPlugins::LogInfo("Connecting to Azure storage ..."); |
263 | |
34 | 264 as::cloud_storage_account storageAccount = as::cloud_storage_account::parse(utility::conversions::to_string_t(connectionString)); |
11 | 265 OrthancPlugins::LogInfo("Storage account created"); |
1 | 266 |
267 as::cloud_blob_client blobClient = storageAccount.create_cloud_blob_client(); | |
11 | 268 OrthancPlugins::LogInfo("Blob client created"); |
269 | |
34 | 270 as::cloud_blob_container blobContainer = blobClient.get_container_reference(utility::conversions::to_string_t(containerName)); |
11 | 271 OrthancPlugins::LogInfo("Accessing blob container"); |
1 | 272 |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
273 // blobContainer.create_if_not_exists() throws an error if a service SAS (for a blob container) |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
274 // was used in the connectionString. |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
275 // Only allow the use of this function when using storage account-level credentials, whether |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
276 // through accountName/accountKey combo or account SAS. |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
277 if ((storageAccount.credentials().is_account_key() || |
48 | 278 (storageAccount.credentials().is_sas() && IsSasTokenAccountLevel(storageAccount.credentials().sas_token()))) |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
279 && createContainerIfNotExists) |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
280 { |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
281 // Return value is true if the container did not exist and was successfully created. |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
282 bool containerCreated = blobContainer.create_if_not_exists(); |
1 | 283 |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
284 if (containerCreated) |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
285 { |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
286 OrthancPlugins::LogWarning("Blob Storage Area container has been created. **** check in the Azure console that your container is private ****"); |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
287 } |
1 | 288 } |
289 | |
290 OrthancPlugins::LogInfo("Blob storage initialized"); | |
291 | |
15 | 292 return new AzureBlobStoragePlugin(blobClient, blobContainer, enableLegacyStorageStructure); |
1 | 293 } |
294 catch (const std::exception& e) | |
295 { | |
296 OrthancPlugins::LogError(std::string("AzureBlobStorage plugin: failed to initialize plugin: ") + e.what()); | |
297 return nullptr; | |
298 } | |
299 | |
300 } | |
301 | |
15 | 302 AzureBlobStoragePlugin::AzureBlobStoragePlugin(const as::cloud_blob_client& blobClient, const as::cloud_blob_container& blobContainer, bool enableLegacyStorageStructure) |
303 : BaseStoragePlugin(enableLegacyStorageStructure), | |
304 blobClient_(blobClient), | |
1 | 305 blobContainer_(blobContainer) |
306 { | |
307 | |
308 } | |
309 | |
310 IStoragePlugin::IWriter* AzureBlobStoragePlugin::GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
311 { | |
312 return new Writer(blobContainer_, GetPath(uuid, type, encryptionEnabled), blobClient_); | |
313 } | |
314 | |
315 IStoragePlugin::IReader* AzureBlobStoragePlugin::GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
316 { | |
317 return new Reader(blobContainer_, GetPath(uuid, type, encryptionEnabled), blobClient_); | |
318 } | |
319 | |
320 void AzureBlobStoragePlugin::DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
321 { | |
322 std::string path = GetPath(uuid, type, encryptionEnabled); | |
323 | |
324 try | |
325 { | |
34 | 326 as::cloud_block_blob blockBlob = blobContainer_.get_block_blob_reference(utility::conversions::to_string_t(path)); |
1 | 327 |
328 blockBlob.delete_blob(); | |
329 } | |
330 catch (std::exception& ex) | |
331 { | |
332 throw StoragePluginException("AzureBlobStorage: error while deleting file " + std::string(path) + ": " + ex.what()); | |
333 } | |
334 } |