Mercurial > hg > orthanc-object-storage
annotate Azure/AzureBlobStoragePlugin.cpp @ 48:ff113c3561c5
coding style + doc
author | Alain Mazy <am@osimis.io> |
---|---|
date | Fri, 02 Apr 2021 09:46:41 +0200 |
parents | 1691da4ae9c3 |
children | b922ae86bbe1 |
rev | line source |
---|---|
1 | 1 /** |
2 * Cloud storage plugins for Orthanc | |
37
f55b2afdf53d
upgrade to year 2021
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
34
diff
changeset
|
3 * Copyright (C) 2020-2021 Osimis S.A., Belgium |
1 | 4 * |
5 * This program is free software: you can redistribute it and/or | |
6 * modify it under the terms of the GNU Affero General Public License | |
7 * as published by the Free Software Foundation, either version 3 of | |
8 * the License, or (at your option) any later version. | |
9 * | |
10 * This program is distributed in the hope that it will be useful, but | |
11 * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 * Affero General Public License for more details. | |
14 * | |
15 * You should have received a copy of the GNU Affero General Public License | |
16 * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
17 **/ | |
18 | |
19 #include "AzureBlobStoragePlugin.h" | |
20 | |
21 #include <was/storage_account.h> | |
22 #include <was/blob.h> | |
23 #include <boost/lexical_cast.hpp> | |
12
a203d0a4fd8f
Azure: now removing spaces and CR at the end of the connectionString since it prevented the plugin to initialize.
Alain Mazy
parents:
11
diff
changeset
|
24 #include <boost/algorithm/string.hpp> |
1 | 25 #include "cpprest/rawptrstream.h" |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
26 #include "cpprest/details/basic_types.h" |
1 | 27 |
28 | |
29 // Create aliases to make the code easier to read. | |
30 namespace as = azure::storage; | |
15 | 31 static const char* const PLUGIN_SECTION = "AzureBlobStorage"; |
1 | 32 |
15 | 33 class AzureBlobStoragePlugin : public BaseStoragePlugin |
1 | 34 { |
35 public: | |
36 | |
37 as::cloud_blob_client blobClient_; | |
38 as::cloud_blob_container blobContainer_; | |
39 public: | |
40 | |
41 // AzureBlobStoragePlugin(const std::string& connectionString, | |
42 // const std::string& containerName | |
43 // ); | |
15 | 44 AzureBlobStoragePlugin(const as::cloud_blob_client& blobClient, const as::cloud_blob_container& blobContainer, bool enableLegacyStorageStructure); |
1 | 45 |
46 virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); | |
47 virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); | |
48 virtual void DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); | |
15 | 49 virtual const char* GetConfigurationSectionName() {return PLUGIN_SECTION;}; |
1 | 50 }; |
51 | |
52 | |
53 class Writer : public IStoragePlugin::IWriter | |
54 { | |
55 std::string path_; | |
56 as::cloud_blob_client client_; | |
57 as::cloud_blob_container container_; | |
58 | |
59 public: | |
60 Writer(const as::cloud_blob_container& container, const std::string& path, const as::cloud_blob_client& client) | |
61 : path_(path), | |
62 client_(client), | |
63 container_(container) | |
64 { | |
65 } | |
66 | |
67 virtual ~Writer() | |
68 { | |
69 } | |
70 | |
71 virtual void Write(const char* data, size_t size) | |
72 { | |
73 try | |
74 { | |
75 concurrency::streams::istream inputStream = concurrency::streams::rawptr_stream<uint8_t>::open_istream(reinterpret_cast<const uint8_t*>(data), size); | |
34 | 76 azure::storage::cloud_block_blob blob = container_.get_block_blob_reference(utility::conversions::to_string_t(path_)); |
1 | 77 blob.upload_from_stream(inputStream); |
78 inputStream.close().wait(); | |
79 } | |
80 catch (std::exception& ex) | |
81 { | |
82 throw StoragePluginException("AzureBlobStorage: error writing file " + std::string(path_) + ": " + ex.what()); | |
83 } | |
84 } | |
85 }; | |
86 | |
87 | |
88 class Reader : public IStoragePlugin::IReader | |
89 { | |
90 std::string path_; | |
91 as::cloud_blob_client client_; | |
92 as::cloud_blob_container container_; | |
93 as::cloud_block_blob block_; | |
94 | |
95 public: | |
96 Reader(const as::cloud_blob_container& container, const std::string& path, const as::cloud_blob_client& client) | |
97 : path_(path), | |
98 client_(client), | |
99 container_(container) | |
100 { | |
101 try | |
102 { | |
34 | 103 block_ = container_.get_block_blob_reference(utility::conversions::to_string_t(path_)); |
1 | 104 block_.download_attributes(); // to retrieve the properties |
105 } | |
106 catch (std::exception& ex) | |
107 { | |
108 throw StoragePluginException("AzureBlobStorage: error opening file for reading " + std::string(path_) + ": " + ex.what()); | |
109 } | |
110 } | |
111 | |
112 virtual ~Reader() | |
113 { | |
114 | |
115 } | |
116 virtual size_t GetSize() | |
117 { | |
118 try | |
119 { | |
120 return block_.properties().size(); | |
121 } | |
122 catch (std::exception& ex) | |
123 { | |
124 throw StoragePluginException("AzureBlobStorage: error while reading file " + std::string(path_) + ": " + ex.what()); | |
125 } | |
126 } | |
127 | |
39
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
128 virtual void ReadWhole(char* data, size_t size) |
1 | 129 { |
130 try | |
131 { | |
132 concurrency::streams::ostream outputStream = concurrency::streams::rawptr_stream<uint8_t>::open_ostream(reinterpret_cast<uint8_t*>(data), size); | |
133 | |
134 block_.download_to_stream(outputStream); | |
135 } | |
136 catch (std::exception& ex) | |
137 { | |
138 throw StoragePluginException("AzureBlobStorage: error while reading file " + std::string(path_) + ": " + ex.what()); | |
139 } | |
140 } | |
141 | |
39
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
142 virtual void ReadRange(char* data, size_t size, size_t fromOffset) |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
143 { |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
144 try |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
145 { |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
146 concurrency::streams::ostream outputStream = concurrency::streams::rawptr_stream<uint8_t>::open_ostream(reinterpret_cast<uint8_t*>(data), size); |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
147 |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
148 block_.download_range_to_stream(outputStream, fromOffset, size); |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
149 } |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
150 catch (std::exception& ex) |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
151 { |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
152 throw StoragePluginException("AzureBlobStorage: error while reading partial file " + std::string(path_) + ": " + ex.what()); |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
153 } |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
154 } |
50d0be413c42
implemented ReadRange (only in Azure plugin right now)
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
155 |
1 | 156 }; |
157 | |
158 | |
159 | |
160 const char* AzureBlobStoragePluginFactory::GetStoragePluginName() | |
161 { | |
162 return "Azure Blob Storage"; | |
163 } | |
164 | |
48 | 165 bool IsSasTokenAccountLevel(utility::string_t sasToken) |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
166 { |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
167 // Use cpprestsdk's utility::string_t here since the expected argument is the return value of |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
168 // as::storage_credentials::sas_token(), which is type utility::string_t |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
169 size_t newIdx = 0; |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
170 size_t prevIdx = 0; |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
171 while ((newIdx = sasToken.find('&', prevIdx)) != utility::string_t::npos) |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
172 { |
48 | 173 utility::string_t kvpair = sasToken.substr(prevIdx, newIdx - prevIdx); |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
174 prevIdx = newIdx + 1; // start next time from char after '&' |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
175 |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
176 size_t equalsIdx = kvpair.find('='); |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
177 utility::string_t key = kvpair.substr(0, equalsIdx); |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
178 if (key == "srt") // only account SAS has this parameter |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
179 return true; |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
180 } |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
181 |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
182 return false; |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
183 } |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
184 |
1 | 185 IStoragePlugin* AzureBlobStoragePluginFactory::CreateStoragePlugin(const OrthancPlugins::OrthancConfiguration& orthancConfig) |
186 { | |
187 std::string connectionString; | |
188 std::string containerName; | |
15 | 189 bool enableLegacyStorageStructure; |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
190 bool createContainerIfNotExists; |
1 | 191 |
192 if (orthancConfig.IsSection(PLUGIN_SECTION)) | |
193 { | |
194 OrthancPlugins::OrthancConfiguration pluginSection; | |
195 orthancConfig.GetSection(pluginSection, PLUGIN_SECTION); | |
196 | |
15 | 197 if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, pluginSection)) |
198 { | |
199 return nullptr; | |
200 } | |
201 | |
1 | 202 if (!pluginSection.LookupStringValue(connectionString, "ConnectionString")) |
203 { | |
204 OrthancPlugins::LogError("AzureBlobStorage/ConnectionString configuration missing. Unable to initialize plugin"); | |
205 return nullptr; | |
206 } | |
207 | |
208 if (!pluginSection.LookupStringValue(containerName, "ContainerName")) | |
209 { | |
210 OrthancPlugins::LogError("AzureBlobStorage/ContainerName configuration missing. Unable to initialize plugin"); | |
211 return nullptr; | |
212 } | |
213 | |
12
a203d0a4fd8f
Azure: now removing spaces and CR at the end of the connectionString since it prevented the plugin to initialize.
Alain Mazy
parents:
11
diff
changeset
|
214 boost::trim(connectionString); // without that, if there's an EOL in the string, it fails with "provided uri is invalid" |
a203d0a4fd8f
Azure: now removing spaces and CR at the end of the connectionString since it prevented the plugin to initialize.
Alain Mazy
parents:
11
diff
changeset
|
215 boost::trim(containerName); |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
216 |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
217 createContainerIfNotExists = pluginSection.GetBooleanValue("CreateContainerIfNotExists", true); |
1 | 218 } |
219 else if (orthancConfig.IsSection("BlobStorage")) // backward compatibility with the old plugin: | |
220 { | |
221 OrthancPlugins::LogWarning("AzureBlobStorage: you're using an old configuration format for the plugin."); | |
222 | |
223 OrthancPlugins::OrthancConfiguration pluginSection; | |
224 orthancConfig.GetSection(pluginSection, "BlobStorage"); | |
225 | |
226 std::string accountName; | |
227 std::string accountKey; | |
228 | |
229 if (!pluginSection.LookupStringValue(containerName, "ContainerName")) | |
230 { | |
231 OrthancPlugins::LogError("BlobStorage/AccountName configuration missing. Unable to initialize plugin"); | |
232 return nullptr; | |
233 } | |
234 | |
235 if (!pluginSection.LookupStringValue(accountName, "AccountName")) | |
236 { | |
237 OrthancPlugins::LogError("BlobStorage/AccountName configuration missing. Unable to initialize plugin"); | |
238 return nullptr; | |
239 } | |
240 | |
241 if (!pluginSection.LookupStringValue(accountKey, "AccountKey")) | |
242 { | |
243 OrthancPlugins::LogError("BlobStorage/ContainerName configuration missing. Unable to initialize plugin"); | |
244 return nullptr; | |
245 } | |
246 | |
247 std::ostringstream connectionStringBuilder; | |
248 connectionStringBuilder << "DefaultEndpointsProtocol=https;AccountName=" << accountName << ";AccountKey=" << accountKey; | |
249 connectionString = connectionStringBuilder.str(); | |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
250 |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
251 createContainerIfNotExists = pluginSection.GetBooleanValue("CreateContainerIfNotExists", true); |
1 | 252 } |
253 else | |
254 { | |
255 OrthancPlugins::LogWarning(std::string(GetStoragePluginName()) + " plugin, section missing. Plugin is not enabled."); | |
256 return nullptr; | |
257 } | |
258 | |
259 try | |
260 { | |
11 | 261 OrthancPlugins::LogInfo("Connecting to Azure storage ..."); |
262 | |
34 | 263 as::cloud_storage_account storageAccount = as::cloud_storage_account::parse(utility::conversions::to_string_t(connectionString)); |
11 | 264 OrthancPlugins::LogInfo("Storage account created"); |
1 | 265 |
266 as::cloud_blob_client blobClient = storageAccount.create_cloud_blob_client(); | |
11 | 267 OrthancPlugins::LogInfo("Blob client created"); |
268 | |
34 | 269 as::cloud_blob_container blobContainer = blobClient.get_container_reference(utility::conversions::to_string_t(containerName)); |
11 | 270 OrthancPlugins::LogInfo("Accessing blob container"); |
1 | 271 |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
272 // blobContainer.create_if_not_exists() throws an error if a service SAS (for a blob container) |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
273 // was used in the connectionString. |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
274 // Only allow the use of this function when using storage account-level credentials, whether |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
275 // through accountName/accountKey combo or account SAS. |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
276 if ((storageAccount.credentials().is_account_key() || |
48 | 277 (storageAccount.credentials().is_sas() && IsSasTokenAccountLevel(storageAccount.credentials().sas_token()))) |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
278 && createContainerIfNotExists) |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
279 { |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
280 // Return value is true if the container did not exist and was successfully created. |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
281 bool containerCreated = blobContainer.create_if_not_exists(); |
1 | 282 |
46
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
283 if (containerCreated) |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
284 { |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
285 OrthancPlugins::LogWarning("Blob Storage Area container has been created. **** check in the Azure console that your container is private ****"); |
3b8fab63313d
Add "CreateContainerIfNotExists" config for Azure
Mark Poscablo <Mark.Poscablo@varian.com>
parents:
15
diff
changeset
|
286 } |
1 | 287 } |
288 | |
289 OrthancPlugins::LogInfo("Blob storage initialized"); | |
290 | |
15 | 291 return new AzureBlobStoragePlugin(blobClient, blobContainer, enableLegacyStorageStructure); |
1 | 292 } |
293 catch (const std::exception& e) | |
294 { | |
295 OrthancPlugins::LogError(std::string("AzureBlobStorage plugin: failed to initialize plugin: ") + e.what()); | |
296 return nullptr; | |
297 } | |
298 | |
299 } | |
300 | |
15 | 301 AzureBlobStoragePlugin::AzureBlobStoragePlugin(const as::cloud_blob_client& blobClient, const as::cloud_blob_container& blobContainer, bool enableLegacyStorageStructure) |
302 : BaseStoragePlugin(enableLegacyStorageStructure), | |
303 blobClient_(blobClient), | |
1 | 304 blobContainer_(blobContainer) |
305 { | |
306 | |
307 } | |
308 | |
309 IStoragePlugin::IWriter* AzureBlobStoragePlugin::GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
310 { | |
311 return new Writer(blobContainer_, GetPath(uuid, type, encryptionEnabled), blobClient_); | |
312 } | |
313 | |
314 IStoragePlugin::IReader* AzureBlobStoragePlugin::GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
315 { | |
316 return new Reader(blobContainer_, GetPath(uuid, type, encryptionEnabled), blobClient_); | |
317 } | |
318 | |
319 void AzureBlobStoragePlugin::DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
320 { | |
321 std::string path = GetPath(uuid, type, encryptionEnabled); | |
322 | |
323 try | |
324 { | |
34 | 325 as::cloud_block_blob blockBlob = blobContainer_.get_block_blob_reference(utility::conversions::to_string_t(path)); |
1 | 326 |
327 blockBlob.delete_blob(); | |
328 } | |
329 catch (std::exception& ex) | |
330 { | |
331 throw StoragePluginException("AzureBlobStorage: error while deleting file " + std::string(path) + ": " + ex.what()); | |
332 } | |
333 } |