# HG changeset patch # User Alain Mazy # Date 1630930916 -7200 # Node ID 16e419fe80c511d7bf742f4aad180a8c6f09b2a4 # Parent 58a03fce4897993344c8d1c4e362437ca028445a Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option diff -r 58a03fce4897 -r 16e419fe80c5 Aws/AwsS3StoragePlugin.cpp --- a/Aws/AwsS3StoragePlugin.cpp Fri Jul 09 15:40:07 2021 +0200 +++ b/Aws/AwsS3StoragePlugin.cpp Mon Sep 06 14:21:56 2021 +0200 @@ -120,35 +120,6 @@ } - size_t _GetSize(const std::string& path) - { - Aws::S3::Model::ListObjectsRequest listObjectRequest; - listObjectRequest.SetBucket(bucketName_.c_str()); - listObjectRequest.SetPrefix(path.c_str()); - - auto result = client_.ListObjects(listObjectRequest); - - if (result.IsSuccess()) - { - Aws::Vector objectList = - result.GetResult().GetContents(); - - if (objectList.size() == 1) - { - return objectList[0].GetSize(); - } - else if (objectList.size() > 1) - { - throw StoragePluginException(std::string("error while reading file ") + path + ": multiple objet with same name !"); - } - throw StoragePluginException(std::string("error while reading file ") + path + ": object not found !"); - } - else - { - throw StoragePluginException(std::string("error while reading file ") + path + ": " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str()); - } - } - virtual size_t GetSize() { std::string firstExceptionMessage; @@ -181,6 +152,38 @@ _Read(data, size, fromOffset, true); } +private: + + size_t _GetSize(const std::string& path) + { + Aws::S3::Model::ListObjectsRequest listObjectRequest; + listObjectRequest.SetBucket(bucketName_.c_str()); + listObjectRequest.SetPrefix(path.c_str()); + + auto result = client_.ListObjects(listObjectRequest); + + if (result.IsSuccess()) + { + Aws::Vector objectList = + result.GetResult().GetContents(); + + if (objectList.size() == 1) + { + return objectList[0].GetSize(); + } + else if (objectList.size() > 1) + { + throw StoragePluginException(std::string("error while reading file ") + path + ": multiple objet with same name !"); + } + throw StoragePluginException(std::string("error while reading file ") + path + ": object not found !"); + } + else + { + throw StoragePluginException(std::string("error while reading file ") + path + ": " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str()); + } + } + + void _Read(char* data, size_t size, size_t fromOffset, bool useRange) { std::string firstExceptionMessage; @@ -271,6 +274,7 @@ Aws::InitAPI(*sdkOptions_); bool enableLegacyStorageStructure; + bool storageContainsUnknownFiles; if (!orthancConfig.IsSection(PLUGIN_SECTION)) { @@ -281,7 +285,7 @@ OrthancPlugins::OrthancConfiguration pluginSection; orthancConfig.GetSection(pluginSection, PLUGIN_SECTION); - if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, pluginSection)) + if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, storageContainsUnknownFiles, pluginSection)) { return nullptr; } @@ -307,7 +311,6 @@ const unsigned int connectTimeout = pluginSection.GetUnsignedIntegerValue("ConnectTimeout", 30); const unsigned int requestTimeout = pluginSection.GetUnsignedIntegerValue("RequestTimeout", 1200); const bool virtualAddressing = pluginSection.GetBooleanValue("VirtualAddressing", true); - const bool storageContainsUnknownFiles = pluginSection.GetBooleanValue("EnableLegacyUnknownFiles", false); const std::string caFile = orthancConfig.GetStringValue("HttpsCACertificates", ""); try diff -r 58a03fce4897 -r 16e419fe80c5 Azure/AzureBlobStoragePlugin.cpp --- a/Azure/AzureBlobStoragePlugin.cpp Fri Jul 09 15:40:07 2021 +0200 +++ b/Azure/AzureBlobStoragePlugin.cpp Mon Sep 06 14:21:56 2021 +0200 @@ -37,12 +37,15 @@ as::cloud_blob_client blobClient_; as::cloud_blob_container blobContainer_; + bool storageContainsUnknownFiles_; + public: -// AzureBlobStoragePlugin(const std::string& connectionString, -// const std::string& containerName -// ); - AzureBlobStoragePlugin(const as::cloud_blob_client& blobClient, const as::cloud_blob_container& blobContainer, bool enableLegacyStorageStructure); + AzureBlobStoragePlugin(const as::cloud_blob_client& blobClient, + const as::cloud_blob_container& blobContainer, + bool enableLegacyStorageStructure, + bool storageContainsUnknownFiles + ); virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); @@ -88,32 +91,42 @@ class Reader : public IStoragePlugin::IReader { - std::string path_; + std::string path_; as::cloud_blob_client client_; as::cloud_blob_container container_; as::cloud_block_blob block_; public: - Reader(const as::cloud_blob_container& container, const std::string& path, const as::cloud_blob_client& client) - : path_(path), - client_(client), + Reader(const as::cloud_blob_container& container, const std::list& paths, const as::cloud_blob_client& client) + : client_(client), container_(container) { - try + std::string firstExceptionMessage; + + for (auto& path: paths) { - block_ = container_.get_block_blob_reference(utility::conversions::to_string_t(path_)); - block_.download_attributes(); // to retrieve the properties + try + { + block_ = container_.get_block_blob_reference(utility::conversions::to_string_t(path)); + block_.download_attributes(); // to retrieve the properties + path_ = path; + } + catch (std::exception& ex) + { + if (firstExceptionMessage.empty()) + { + firstExceptionMessage = "AzureBlobStorage: error opening file for reading " + std::string(path) + ": " + ex.what(); + } + //ignore to retry + } } - catch (std::exception& ex) - { - throw StoragePluginException("AzureBlobStorage: error opening file for reading " + std::string(path_) + ": " + ex.what()); - } + throw StoragePluginException(firstExceptionMessage); } virtual ~Reader() { + } - } virtual size_t GetSize() { try @@ -153,7 +166,6 @@ throw StoragePluginException("AzureBlobStorage: error while reading partial file " + std::string(path_) + ": " + ex.what()); } } - }; @@ -188,6 +200,7 @@ std::string connectionString; std::string containerName; bool enableLegacyStorageStructure; + bool storageContainsUnknownFiles; bool createContainerIfNotExists; if (orthancConfig.IsSection(PLUGIN_SECTION)) @@ -195,7 +208,7 @@ OrthancPlugins::OrthancConfiguration pluginSection; orthancConfig.GetSection(pluginSection, PLUGIN_SECTION); - if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, pluginSection)) + if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, storageContainsUnknownFiles, pluginSection)) { return nullptr; } @@ -289,7 +302,7 @@ OrthancPlugins::LogInfo("Blob storage initialized"); - return new AzureBlobStoragePlugin(blobClient, blobContainer, enableLegacyStorageStructure); + return new AzureBlobStoragePlugin(blobClient, blobContainer, enableLegacyStorageStructure, storageContainsUnknownFiles); } catch (const std::exception& e) { @@ -299,10 +312,11 @@ } -AzureBlobStoragePlugin::AzureBlobStoragePlugin(const as::cloud_blob_client& blobClient, const as::cloud_blob_container& blobContainer, bool enableLegacyStorageStructure) +AzureBlobStoragePlugin::AzureBlobStoragePlugin(const as::cloud_blob_client& blobClient, const as::cloud_blob_container& blobContainer, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles) : BaseStoragePlugin(enableLegacyStorageStructure), blobClient_(blobClient), - blobContainer_(blobContainer) + blobContainer_(blobContainer), + storageContainsUnknownFiles_(storageContainsUnknownFiles) { } @@ -314,7 +328,14 @@ IStoragePlugin::IReader* AzureBlobStoragePlugin::GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) { - return new Reader(blobContainer_, GetPath(uuid, type, encryptionEnabled), blobClient_); + std::list paths; + paths.push_back(GetPath(uuid, type, encryptionEnabled, false)); + if (storageContainsUnknownFiles_) + { + paths.push_back(GetPath(uuid, type, encryptionEnabled, true)); + } + + return new Reader(blobContainer_, paths, blobClient_); } void AzureBlobStoragePlugin::DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) diff -r 58a03fce4897 -r 16e419fe80c5 Common/BaseStoragePlugin.cpp --- a/Common/BaseStoragePlugin.cpp Fri Jul 09 15:40:07 2021 +0200 +++ b/Common/BaseStoragePlugin.cpp Mon Sep 06 14:21:56 2021 +0200 @@ -72,7 +72,7 @@ } } -bool BaseStoragePlugin::ReadCommonConfiguration(bool& enableLegacyStorageStructure, const OrthancPlugins::OrthancConfiguration& pluginSection) +bool BaseStoragePlugin::ReadCommonConfiguration(bool& enableLegacyStorageStructure, bool& storageContainsUnknownFiles, const OrthancPlugins::OrthancConfiguration& pluginSection) { std::string storageStructure = pluginSection.GetStringValue("StorageStructure", "flat"); if (storageStructure == "flat") @@ -89,5 +89,7 @@ } } + storageContainsUnknownFiles = pluginSection.GetBooleanValue("EnableLegacyUnknownFiles", false); + return true; } diff -r 58a03fce4897 -r 16e419fe80c5 Common/BaseStoragePlugin.h --- a/Common/BaseStoragePlugin.h Fri Jul 09 15:40:07 2021 +0200 +++ b/Common/BaseStoragePlugin.h Mon Sep 06 14:21:56 2021 +0200 @@ -43,5 +43,5 @@ static std::string GetPath(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled, bool legacyFileStructure, const std::string& rootFolder); static std::string GetOrthancFileSystemPath(const std::string& uuid, const std::string& fileSystemRootPath); - static bool ReadCommonConfiguration(bool& enableLegacyStorageStructure, const OrthancPlugins::OrthancConfiguration& pluginSection); + static bool ReadCommonConfiguration(bool& enableLegacyStorageStructure, bool& storageContainsUnknownFiles, const OrthancPlugins::OrthancConfiguration& pluginSection); }; diff -r 58a03fce4897 -r 16e419fe80c5 Google/GoogleStoragePlugin.cpp --- a/Google/GoogleStoragePlugin.cpp Fri Jul 09 15:40:07 2021 +0200 +++ b/Google/GoogleStoragePlugin.cpp Mon Sep 06 14:21:56 2021 +0200 @@ -25,10 +25,29 @@ namespace gcs = google::cloud::storage; static const char* const PLUGIN_SECTION = "GoogleCloudStorage"; -const char* GoogleStoragePlugin::GetConfigurationSectionName() + +class GoogleStoragePlugin : public BaseStoragePlugin { - return PLUGIN_SECTION; -} +public: + + std::string bucketName_; + google::cloud::storage::Client mainClient_; // the client that is created at startup. Each thread should copy it when it needs it. (from the doc: Instances of this class created via copy-construction or copy-assignment share the underlying pool of connections. Access to these copies via multiple threads is guaranteed to work. Two threads operating on the same instance of this class is not guaranteed to work.) + bool storageContainsUnknownFiles_; + +public: + + GoogleStoragePlugin(const std::string& bucketName, + google::cloud::storage::Client& mainClient, + bool enableLegacyStorageStructure, + bool storageContainsUnknownFiles + ); + + virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); + virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); + virtual void DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); + virtual const char* GetConfigurationSectionName() {return PLUGIN_SECTION;} +}; + class Writer : public IStoragePlugin::IWriter { @@ -72,13 +91,13 @@ class Reader : public IStoragePlugin::IReader { - std::string path_; - gcs::Client client_; - std::string bucketName_; + std::list paths_; + gcs::Client client_; + std::string bucketName_; public: - Reader(const std::string& bucketName, const std::string& path, gcs::Client& client) - : path_(path), + Reader(const std::string& bucketName, const std::list& paths, gcs::Client& client) + : paths_(paths), client_(client), bucketName_(bucketName) { @@ -88,9 +107,111 @@ { } + virtual size_t GetSize() { - auto objectMetadata = client_.GetObjectMetadata(bucketName_, path_); + std::string firstExceptionMessage; + + for (auto& path: paths_) + { + try + { + return _GetSize(path); + } + catch (StoragePluginException& ex) + { + if (firstExceptionMessage.empty()) + { + firstExceptionMessage = ex.what(); + } + //ignore to retry + } + } + throw StoragePluginException(firstExceptionMessage); + } + + void ReadWhole(char* data, size_t size) + { + std::string firstExceptionMessage; + + for (auto& path: paths_) + { + try + { + return _ReadWhole(path, data, size); + } + catch (StoragePluginException& ex) + { + if (firstExceptionMessage.empty()) + { + firstExceptionMessage = ex.what(); + } + //ignore to retry + } + } + throw StoragePluginException(firstExceptionMessage); + } + + void ReadRange(char* data, size_t size, size_t fromOffset) + { + std::string firstExceptionMessage; + + for (auto& path: paths_) + { + try + { + return _ReadRange(path, data, size, fromOffset); + } + catch (StoragePluginException& ex) + { + if (firstExceptionMessage.empty()) + { + firstExceptionMessage = ex.what(); + } + //ignore to retry + } + } + throw StoragePluginException(firstExceptionMessage); + } + +private: + virtual void _ReadWhole(const std::string& path, char* data, size_t size) + { + auto reader = client_.ReadObject(bucketName_, path); + + if (!reader) + { + throw StoragePluginException("error while opening/reading file " + std::string(path) + ": " + reader.status().message()); + } + + reader.read(data, size); + + if (!reader) + { + throw StoragePluginException("error while reading file " + std::string(path) + ": " + reader.status().message()); + } + } + + virtual void _ReadRange(const std::string& path, char* data, size_t size, size_t fromOffset) + { + auto reader = client_.ReadObject(bucketName_, path, gcs::ReadRange(fromOffset, fromOffset + size)); + + if (!reader) + { + throw StoragePluginException("error while opening/reading file " + std::string(path) + ": " + reader.status().message()); + } + + reader.read(data, size); + + if (!reader) + { + throw StoragePluginException("error while reading file " + std::string(path) + ": " + reader.status().message()); + } + } + + size_t _GetSize(const std::string& path) + { + auto objectMetadata = client_.GetObjectMetadata(bucketName_, path); if (objectMetadata) { @@ -100,45 +221,10 @@ } else { - throw StoragePluginException("error while getting the size of " + std::string(path_) + ": " + objectMetadata.status().message()); + throw StoragePluginException("error while getting the size of " + std::string(path) + ": " + objectMetadata.status().message()); } } - virtual void ReadWhole(char* data, size_t size) - { - auto reader = client_.ReadObject(bucketName_, path_); - - if (!reader) - { - throw StoragePluginException("error while opening/reading file " + std::string(path_) + ": " + reader.status().message()); - } - - reader.read(data, size); - - if (!reader) - { - throw StoragePluginException("error while reading file " + std::string(path_) + ": " + reader.status().message()); - } - } - - virtual void ReadRange(char* data, size_t size, size_t fromOffset) - { - auto reader = client_.ReadObject(bucketName_, path_, gcs::ReadRange(fromOffset, fromOffset + size)); - - if (!reader) - { - throw StoragePluginException("error while opening/reading file " + std::string(path_) + ": " + reader.status().message()); - } - - reader.read(data, size); - - if (!reader) - { - throw StoragePluginException("error while reading file " + std::string(path_) + ": " + reader.status().message()); - } - } - - }; @@ -151,6 +237,7 @@ IStoragePlugin* GoogleStoragePluginFactory::CreateStoragePlugin(const OrthancPlugins::OrthancConfiguration& orthancConfig) { bool enableLegacyStorageStructure; + bool storageContainsUnknownFiles; if (!orthancConfig.IsSection(PLUGIN_SECTION)) { @@ -161,7 +248,7 @@ OrthancPlugins::OrthancConfiguration pluginSection; orthancConfig.GetSection(pluginSection, PLUGIN_SECTION); - if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, pluginSection)) + if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, storageContainsUnknownFiles, pluginSection)) { return nullptr; } @@ -198,13 +285,14 @@ return nullptr; } - return new GoogleStoragePlugin(googleBucketName, mainClient.value(), enableLegacyStorageStructure); + return new GoogleStoragePlugin(googleBucketName, mainClient.value(), enableLegacyStorageStructure, storageContainsUnknownFiles); } -GoogleStoragePlugin::GoogleStoragePlugin(const std::string &bucketName, google::cloud::storage::Client& mainClient, bool enableLegacyStorageStructure) +GoogleStoragePlugin::GoogleStoragePlugin(const std::string &bucketName, google::cloud::storage::Client& mainClient, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles) : BaseStoragePlugin(enableLegacyStorageStructure), bucketName_(bucketName), - mainClient_(mainClient) + mainClient_(mainClient), + storageContainsUnknownFiles_(storageContainsUnknownFiles) { } @@ -216,7 +304,14 @@ IStoragePlugin::IReader* GoogleStoragePlugin::GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) { - return new Reader(bucketName_, GetPath(uuid, type, encryptionEnabled), mainClient_); + std::list paths; + paths.push_back(GetPath(uuid, type, encryptionEnabled, false)); + if (storageContainsUnknownFiles_) + { + paths.push_back(GetPath(uuid, type, encryptionEnabled, true)); + } + + return new Reader(bucketName_, paths, mainClient_); } void GoogleStoragePlugin::DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) diff -r 58a03fce4897 -r 16e419fe80c5 Google/GoogleStoragePlugin.h --- a/Google/GoogleStoragePlugin.h Fri Jul 09 15:40:07 2021 +0200 +++ b/Google/GoogleStoragePlugin.h Mon Sep 06 14:21:56 2021 +0200 @@ -29,24 +29,3 @@ static const char* GetStoragePluginName(); static IStoragePlugin* CreateStoragePlugin(const OrthancPlugins::OrthancConfiguration& orthancConfig); }; - - -class GoogleStoragePlugin : public BaseStoragePlugin -{ -public: - - std::string bucketName_; - google::cloud::storage::Client mainClient_; // the client that is created at startup. Each thread should copy it when it needs it. (from the doc: Instances of this class created via copy-construction or copy-assignment share the underlying pool of connections. Access to these copies via multiple threads is guaranteed to work. Two threads operating on the same instance of this class is not guaranteed to work.) - -public: - - GoogleStoragePlugin(const std::string& bucketName, - google::cloud::storage::Client& mainClient, - bool enableLegacyStorageStructure - ); - - virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); - virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); - virtual void DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled); - virtual const char* GetConfigurationSectionName(); -}; diff -r 58a03fce4897 -r 16e419fe80c5 NEWS --- a/NEWS Fri Jul 09 15:40:07 2021 +0200 +++ b/NEWS Mon Sep 06 14:21:56 2021 +0200 @@ -1,3 +1,7 @@ +Pending changes in the mainline: + +* Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option + 2021-07-09 - v 1.3.2 ====================