Mercurial > hg > orthanc-object-storage
view Azure/AzureBlobStoragePlugin.cpp @ 113:78c075412ab4
more config for S3 transfer mode
author | Alain Mazy <am@osimis.io> |
---|---|
date | Tue, 17 Oct 2023 13:01:04 +0200 |
parents | 407bd022b0cf |
children | 3c7e0374f28e |
line wrap: on
line source
/** * Cloud storage plugins for Orthanc * Copyright (C) 2020-2021 Osimis S.A., Belgium * * This program is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * as published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. **/ #include "AzureBlobStoragePlugin.h" #include <azure/storage/blobs.hpp> #include <boost/lexical_cast.hpp> #include <boost/algorithm/string.hpp> // #include "cpprest/rawptrstream.h" // #include "cpprest/details/basic_types.h" // Create aliases to make the code easier to read. namespace as = Azure::Storage::Blobs; class AzureBlobStoragePlugin : public BaseStorage { public: as::BlobContainerClient blobClient_; bool storageContainsUnknownFiles_; public: AzureBlobStoragePlugin(const std::string& nameForLogs, const as::BlobContainerClient& blobClient, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles ); virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); virtual void DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); virtual bool HasFileExists() {return false;}; }; class Writer : public IStorage::IWriter { std::string path_; as::BlobContainerClient client_; public: Writer(const std::string& path, const as::BlobContainerClient& client) : path_(path), client_(client) { } virtual ~Writer() { } virtual void Write(const char* data, size_t size) { try { as::BlockBlobClient blobClient = client_.GetBlockBlobClient(path_); blobClient.UploadFrom(reinterpret_cast<const uint8_t*>(data), size); } catch (std::exception& ex) { throw StoragePluginException("AzureBlobStorage: error writing file " + std::string(path_) + ": " + ex.what()); } } }; class Reader : public IStorage::IReader { std::string path_; as::BlobContainerClient client_; int64_t size_; public: Reader(const std::list<std::string>& paths, const as::BlobContainerClient& client) : client_(client) { std::string firstExceptionMessage; for (auto& path: paths) { try { as::BlockBlobClient blobClient = client_.GetBlockBlobClient(path); auto properties = blobClient.GetProperties().Value; size_ = properties.BlobSize; path_ = path; return; } catch (std::exception& ex) { if (firstExceptionMessage.empty()) { firstExceptionMessage = "AzureBlobStorage: error opening file for reading " + std::string(path) + ": " + ex.what(); } //ignore to retry } } throw StoragePluginException(firstExceptionMessage); } virtual ~Reader() { } virtual size_t GetSize() { try { return static_cast<size_t>(size_); } catch (std::exception& ex) { throw StoragePluginException("AzureBlobStorage: error while reading file " + std::string(path_) + ": " + ex.what()); } } virtual void ReadWhole(char* data, size_t size) { try { as::BlockBlobClient blobClient = client_.GetBlockBlobClient(path_); blobClient.DownloadTo(reinterpret_cast<uint8_t*>(data), static_cast<int64_t>(size)); } catch (std::exception& ex) { throw StoragePluginException("AzureBlobStorage: error while reading file " + std::string(path_) + ": " + ex.what()); } } virtual void ReadRange(char* data, size_t size, size_t fromOffset) { try { as::BlockBlobClient blobClient = client_.GetBlockBlobClient(path_); as::DownloadBlobToOptions options; options.Range = Azure::Core::Http::HttpRange(); options.Range.Value().Length = static_cast<int64_t>(size); options.Range.Value().Offset = static_cast<int64_t>(fromOffset); blobClient.DownloadTo(reinterpret_cast<uint8_t*>(data), static_cast<int64_t>(size), options); } catch (std::exception& ex) { throw StoragePluginException("AzureBlobStorage: error while reading partial file " + std::string(path_) + ": " + ex.what()); } } }; const char* AzureBlobStoragePluginFactory::GetStoragePluginName() { return "Azure Blob Storage"; } const char* AzureBlobStoragePluginFactory::GetStorageDescription() { return "Stores the Orthanc storage area in Azure Blob"; } // bool IsSasTokenAccountLevel(utility::string_t sasToken) // { // // Use cpprestsdk's utility::string_t here since the expected argument is the return value of // // as::storage_credentials::sas_token(), which is type utility::string_t (which is a std::wstring on Windows and a std::string on Linux) // size_t newIdx = 0; // size_t prevIdx = 0; // while ((newIdx = sasToken.find('&', prevIdx)) != utility::string_t::npos) // { // utility::string_t kvpair = sasToken.substr(prevIdx, newIdx - prevIdx); // prevIdx = newIdx + 1; // start next time from char after '&' // size_t equalsIdx = kvpair.find('='); // utility::string_t key = kvpair.substr(0, equalsIdx); // #ifdef WIN32 // const wchar_t* srt = L"srt"; // #else // const char* srt = "srt"; // #endif // if (key == srt) // only account SAS has this parameter // return true; // } // return false; // } IStorage* AzureBlobStoragePluginFactory::CreateStorage(const std::string& nameForLogs, const OrthancPlugins::OrthancConfiguration& orthancConfig) { std::string connectionString; std::string containerName; bool enableLegacyStorageStructure; bool storageContainsUnknownFiles; bool createContainerIfNotExists; if (orthancConfig.IsSection(GetConfigurationSectionName())) { OrthancPlugins::OrthancConfiguration pluginSection; orthancConfig.GetSection(pluginSection, GetConfigurationSectionName()); if (!BaseStorage::ReadCommonConfiguration(enableLegacyStorageStructure, storageContainsUnknownFiles, pluginSection)) { return nullptr; } if (!pluginSection.LookupStringValue(connectionString, "ConnectionString")) { OrthancPlugins::LogError("AzureBlobStorage/ConnectionString configuration missing. Unable to initialize plugin"); return nullptr; } if (!pluginSection.LookupStringValue(containerName, "ContainerName")) { OrthancPlugins::LogError("AzureBlobStorage/ContainerName configuration missing. Unable to initialize plugin"); return nullptr; } boost::trim(connectionString); // without that, if there's an EOL in the string, it fails with "provided uri is invalid" boost::trim(containerName); createContainerIfNotExists = pluginSection.GetBooleanValue("CreateContainerIfNotExists", true); } else if (orthancConfig.IsSection("BlobStorage")) // backward compatibility with the old plugin: { OrthancPlugins::LogWarning("AzureBlobStorage: you're using an old configuration format for the plugin."); OrthancPlugins::OrthancConfiguration pluginSection; orthancConfig.GetSection(pluginSection, "BlobStorage"); std::string accountName; std::string accountKey; if (!pluginSection.LookupStringValue(containerName, "ContainerName")) { OrthancPlugins::LogError("BlobStorage/AccountName configuration missing. Unable to initialize plugin"); return nullptr; } if (!pluginSection.LookupStringValue(accountName, "AccountName")) { OrthancPlugins::LogError("BlobStorage/AccountName configuration missing. Unable to initialize plugin"); return nullptr; } if (!pluginSection.LookupStringValue(accountKey, "AccountKey")) { OrthancPlugins::LogError("BlobStorage/ContainerName configuration missing. Unable to initialize plugin"); return nullptr; } std::ostringstream connectionStringBuilder; connectionStringBuilder << "DefaultEndpointsProtocol=https;AccountName=" << accountName << ";AccountKey=" << accountKey; connectionString = connectionStringBuilder.str(); createContainerIfNotExists = pluginSection.GetBooleanValue("CreateContainerIfNotExists", true); } else { OrthancPlugins::LogWarning(std::string(GetStoragePluginName()) + " plugin, section missing. Plugin is not enabled."); return nullptr; } try { OrthancPlugins::LogInfo("Connecting to Azure storage ..."); as::BlobContainerClient client = as::BlobContainerClient::CreateFromConnectionString(connectionString, containerName); OrthancPlugins::LogInfo("Blob client created"); if (createContainerIfNotExists) { // Note: in version up to 2.1.2, we had this code: // // blobContainer.create_if_not_exists() throws an error if a service SAS (for a blob container) // // was used in the connectionString. // // Only allow the use of this function when using storage account-level credentials, whether // // through accountName/accountKey combo or account SAS. // if ((storageAccount.credentials().is_account_key() || // (storageAccount.credentials().is_sas() && IsSasTokenAccountLevel(storageAccount.credentials().sas_token()))) // && createContainerIfNotExists) auto createResult = client.CreateIfNotExists(); if (createResult.Value.Created) { OrthancPlugins::LogWarning("Blob Storage Area container has been created. **** check in the Azure console that your container is private ****"); } } OrthancPlugins::LogInfo("Blob storage initialized"); return new AzureBlobStoragePlugin(nameForLogs, client, enableLegacyStorageStructure, storageContainsUnknownFiles); } catch (const std::exception& e) { OrthancPlugins::LogError(std::string("AzureBlobStorage plugin: failed to initialize plugin: ") + e.what()); return nullptr; } } AzureBlobStoragePlugin::AzureBlobStoragePlugin(const std::string& nameForLogs, const as::BlobContainerClient& blobClient, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles) : BaseStorage(nameForLogs, enableLegacyStorageStructure), blobClient_(blobClient), storageContainsUnknownFiles_(storageContainsUnknownFiles) { } IStorage::IWriter* AzureBlobStoragePlugin::GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) { return new Writer(GetPath(uuid, type, encryptionEnabled), blobClient_); } IStorage::IReader* AzureBlobStoragePlugin::GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) { std::list<std::string> paths; paths.push_back(GetPath(uuid, type, encryptionEnabled, false)); if (storageContainsUnknownFiles_) { paths.push_back(GetPath(uuid, type, encryptionEnabled, true)); } return new Reader(paths, blobClient_); } void AzureBlobStoragePlugin::DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) { std::string path = GetPath(uuid, type, encryptionEnabled); try { as::BlockBlobClient blobClient = blobClient_.GetBlockBlobClient(path); blobClient.Delete(); } catch (std::exception& ex) { throw StoragePluginException("AzureBlobStorage: error while deleting file " + std::string(path) + ": " + ex.what()); } }