Mercurial > hg > orthanc-object-storage
changeset 33:70da4ce5c7cc
merge
| author | Alain Mazy |
|---|---|
| date | Fri, 09 Oct 2020 10:23:27 +0200 |
| parents | 8d2b29fd4de5 (current diff) 662b9d3f217d (diff) |
| children | 7ddd840563c9 8a7a5defd5d0 |
| files | Aws/AwsS3StoragePlugin.cpp Common/StoragePlugin.cpp |
| diffstat | 10 files changed, 170 insertions(+), 61 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Fri Oct 09 10:16:50 2020 +0200 +++ b/.hgignore Fri Oct 09 10:23:27 2020 +0200 @@ -1,1 +1,3 @@ +syntax: glob CMakeLists.txt.user* +*~
--- a/Aws/AwsS3StoragePlugin.cpp Fri Oct 09 10:16:50 2020 +0200 +++ b/Aws/AwsS3StoragePlugin.cpp Fri Oct 09 10:23:27 2020 +0200 @@ -229,7 +229,8 @@ std::string endpoint = pluginSection.GetStringValue("Endpoint", ""); unsigned int connectTimeout = pluginSection.GetUnsignedIntegerValue("ConnectTimeout", 30); unsigned int requestTimeout = pluginSection.GetUnsignedIntegerValue("RequestTimeout", 1200); - + bool virtualAddressing = pluginSection.GetBooleanValue("VirtualAddressing", true); + try { Aws::SDKOptions options; @@ -241,13 +242,14 @@ configuration.scheme = Aws::Http::Scheme::HTTPS; configuration.connectTimeoutMs = connectTimeout * 1000; configuration.requestTimeoutMs = requestTimeout * 1000; + configuration.httpRequestTimeoutMs = requestTimeout * 1000; if (!endpoint.empty()) { configuration.endpointOverride = endpoint.c_str(); } - Aws::S3::S3Client client(credentials, configuration); + Aws::S3::S3Client client(credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing); OrthancPlugins::LogInfo("AWS S3 storage initialized");
--- a/Aws/CMakeLists.txt Fri Oct 09 10:16:50 2020 +0200 +++ b/Aws/CMakeLists.txt Fri Oct 09 10:23:27 2020 +0200 @@ -10,6 +10,8 @@ set(ORTHANC_FRAMEWORK_SOURCE "hg" CACHE STRING "orthanc source") set(ORTHANC_FRAMEWORK_VERSION "1.7.3" CACHE STRING "orthanc framework version") +set(USE_VCPKG_PACKAGES ON CACHE BOOL "Use vcpkg to link against crypto++ and AWS SDK") +set(STATIC_AWS_CLIENT ON CACHE BOOL "Statically link against AWS client library (only if USE_VCPKG_PACKAGES=OFF)") set(ALLOW_DOWNLOADS ON) # Download and setup the Orthanc framework @@ -19,6 +21,9 @@ set(ENABLE_GOOGLE_TEST ON) set(ORTHANC_FRAMEWORK_PLUGIN ON) +set(ENABLE_MODULE_IMAGES OFF) +set(ENABLE_MODULE_JOBS OFF) +set(ENABLE_MODULE_DICOM OFF) include(${ORTHANC_FRAMEWORK_ROOT}/../Resources/CMake/OrthancFrameworkConfiguration.cmake) include(${ORTHANC_FRAMEWORK_ROOT}/../../OrthancServer/Plugins/Samples/Common/OrthancPluginsExports.cmake) @@ -38,10 +43,84 @@ ) -find_package(cryptopp CONFIG REQUIRED) -find_package(AWSSDK REQUIRED COMPONENTS s3) +if (USE_VCPKG_PACKAGES) + find_package(cryptopp CONFIG REQUIRED) + find_package(AWSSDK REQUIRED COMPONENTS s3) + include_directories(${WASTORAGE_INCLUDE_DIR}) + set(CRYPTOPP_LIBRARIES cryptopp-static) +else() + ## + ## Inclusion of system-wide crypto++ + ## + check_include_file_cxx(cryptopp/cryptlib.h HAVE_CRYPTOPP_H) + if (NOT HAVE_CRYPTOPP_H) + message(FATAL_ERROR "Please install the libcrypto++-dev package") + endif() + + include(CheckCXXSymbolExists) + set(CMAKE_REQUIRED_LIBRARIES cryptopp) + check_cxx_symbol_exists("CryptoPP::SHA1::InitState" cryptopp/sha.h HAVE_LIBCRYPTOPP) + if (NOT HAVE_LIBCRYPTOPP) + message(FATAL_ERROR "Unable to find the cryptopp library") + endif() + + set(CRYPTOPP_LIBRARIES cryptopp) + + ## + ## Building the C++ SDK for Amazon AWS + ## WARNING: This is *not* compatible with Ninja (yet) + ## + if (STATIC_AWS_CLIENT) + set(Flags -DBUILD_SHARED_LIBS=OFF) # Create static library + else() + set(Flags -DBUILD_SHARED_LIBS=ON) + endif() + + include(ExternalProject) + externalproject_add(AwsSdkCpp + GIT_REPOSITORY https://github.com/aws/aws-sdk-cpp + GIT_TAG 1.8.42 -include_directories(${WASTORAGE_INCLUDE_DIR}) + CMAKE_ARGS + -DBUILD_ONLY=s3 #-DBUILD_ONLY=s3;transfer + -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DENABLE_TESTING=OFF + ${Flags} + + UPDATE_COMMAND "" # Don't run "cmake" on AWS each time "make/ninja" is run + INSTALL_COMMAND "" # No install + ) + + ExternalProject_Get_Property(AwsSdkCpp SOURCE_DIR) + include_directories( + ${SOURCE_DIR}/aws-cpp-sdk-core/include/ + ${SOURCE_DIR}/aws-cpp-sdk-s3/include/ + ) + + ExternalProject_Get_Property(AwsSdkCpp BINARY_DIR) + if (STATIC_AWS_CLIENT) + set(AWSSDK_LINK_LIBRARIES + ${BINARY_DIR}/aws-cpp-sdk-s3/libaws-cpp-sdk-s3.a + ${BINARY_DIR}/aws-cpp-sdk-core/libaws-cpp-sdk-core.a + ${BINARY_DIR}/.deps/install/lib/libaws-c-event-stream.a + ${BINARY_DIR}/.deps/install/lib/libaws-checksums.a + ${BINARY_DIR}/.deps/install/lib/libaws-c-common.a + curl + crypto + ) + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + list(APPEND AWSSDK_LINK_LIBRARIES + gcc # for "undefined reference to `__cpu_model'" on Ubuntu 16.04 + ) + endif() + else() + set(AWSSDK_LINK_LIBRARIES + ${BINARY_DIR}/aws-cpp-sdk-core/libaws-cpp-sdk-core.so + ${BINARY_DIR}/aws-cpp-sdk-s3/libaws-cpp-sdk-s3.so + ) + endif() +endif() + set(COMMON_SOURCES ${CMAKE_SOURCE_DIR}/../Common/IStoragePlugin.h @@ -62,7 +141,7 @@ ${CMAKE_SOURCE_DIR}/../Common/StoragePlugin.cpp ${COMMON_SOURCES} - ) + ) set_target_properties(OrthancAwsS3Storage PROPERTIES VERSION ${PLUGIN_VERSION} @@ -71,7 +150,7 @@ target_link_libraries(OrthancAwsS3Storage PRIVATE - cryptopp-static + ${CRYPTOPP_LIBRARIES} ${AWSSDK_LINK_LIBRARIES} ) @@ -87,7 +166,13 @@ target_link_libraries(UnitTests PRIVATE - cryptopp-static ${GOOGLE_TEST_LIBRARIES} + ${CRYPTOPP_LIBRARIES} ${AWSSDK_LINK_LIBRARIES} ) + + +if (NOT USE_VCPKG_PACKAGES) + add_dependencies(OrthancAwsS3Storage AwsSdkCpp) + add_dependencies(UnitTests AwsSdkCpp) +endif()
--- a/Azure/CMakeLists.txt Fri Oct 09 10:16:50 2020 +0200 +++ b/Azure/CMakeLists.txt Fri Oct 09 10:23:27 2020 +0200 @@ -17,6 +17,9 @@ set(ENABLE_GOOGLE_TEST ON) set(ORTHANC_FRAMEWORK_PLUGIN ON) +set(ENABLE_MODULE_IMAGES OFF) +set(ENABLE_MODULE_JOBS OFF) +set(ENABLE_MODULE_DICOM OFF) include(${ORTHANC_FRAMEWORK_ROOT}/../Resources/CMake/OrthancFrameworkConfiguration.cmake) include(${ORTHANC_FRAMEWORK_ROOT}/../../OrthancServer/Plugins/Samples/Common/OrthancPluginsExports.cmake)
--- a/Common/EncryptionHelpers.cpp Fri Oct 09 10:16:50 2020 +0200 +++ b/Common/EncryptionHelpers.cpp Fri Oct 09 10:23:27 2020 +0200 @@ -21,17 +21,19 @@ #include <boost/lexical_cast.hpp> #include <iostream> -#include "cryptopp/cryptlib.h" -#include "cryptopp/modes.h" -#include "cryptopp/hex.h" -#include "cryptopp/gcm.h" -#include "cryptopp/files.h" + +#include <cryptopp/cryptlib.h> +#include <cryptopp/modes.h> +#include <cryptopp/hex.h> +#include <cryptopp/gcm.h> +#include <cryptopp/files.h> +#include <cryptopp/filters.h> const std::string EncryptionHelpers::HEADER_VERSION = "A1"; using namespace CryptoPP; -std::string EncryptionHelpers::ToHexString(const byte* block, size_t size) +std::string EncryptionHelpers::ToHexString(const void* block, size_t size) { std::string blockAsString = std::string(reinterpret_cast<const char*>(block), size); @@ -275,7 +277,7 @@ try { GCM<AES>::Encryption e; - e.SetKeyWithIV(dataKey, dataKey.size(), iv, sizeof(iv)); + e.SetKeyWithIV(dataKey, dataKey.size(), iv, iv.size()); // the output text starts with the unencrypted prefix output = prefix; @@ -323,7 +325,7 @@ // std::cout << ToHexString(iv) << std::endl; GCM<AES>::Decryption d; - d.SetKeyWithIV(dataKey, sizeof(dataKey), iv, sizeof(iv)); + d.SetKeyWithIV(dataKey, dataKey.size(), iv, iv.size()); try { AuthenticatedDecryptionFilter df(d, NULL,
--- a/Common/EncryptionHelpers.h Fri Oct 09 10:16:50 2020 +0200 +++ b/Common/EncryptionHelpers.h Fri Oct 09 10:23:27 2020 +0200 @@ -19,7 +19,7 @@ #include <memory.h> #include <cryptopp/secblock.h> -#include "cryptopp/osrng.h" +#include <cryptopp/osrng.h> #include <boost/thread/mutex.hpp> #include <MultiThreading/Semaphore.h> @@ -100,7 +100,7 @@ public: - static std::string ToHexString(const CryptoPP::byte* block, size_t size); + static std::string ToHexString(const void* block, size_t size); static std::string ToHexString(const std::string& block); static std::string ToHexString(const CryptoPP::SecByteBlock& block); static std::string ToString(const CryptoPP::SecByteBlock& block);
--- a/Common/StoragePlugin.cpp Fri Oct 09 10:16:50 2020 +0200 +++ b/Common/StoragePlugin.cpp Fri Oct 09 10:23:27 2020 +0200 @@ -301,59 +301,67 @@ return -1; } - plugin.reset(StoragePluginFactory::CreateStoragePlugin(orthancConfig)); - - if (plugin.get() == nullptr) - { - return -1; - } - - const char* pluginSectionName = plugin->GetConfigurationSectionName(); - static const char* const ENCRYPTION_SECTION = "StorageEncryption"; - - if (orthancConfig.IsSection(pluginSectionName)) + try { - OrthancPlugins::OrthancConfiguration pluginSection; - orthancConfig.GetSection(pluginSection, pluginSectionName); + plugin.reset(StoragePluginFactory::CreateStoragePlugin(orthancConfig)); - migrationFromFileSystemEnabled = pluginSection.GetBooleanValue("MigrationFromFileSystemEnabled", false); - - if (migrationFromFileSystemEnabled) + if (plugin.get() == nullptr) { - fileSystemRootPath = orthancConfig.GetStringValue("StorageDirectory", "OrthancStorageNotDefined"); - OrthancPlugins::LogWarning(std::string(StoragePluginFactory::GetStoragePluginName()) + ": migration from file system enabled, source: " + fileSystemRootPath); - } - - objectsRootPath = pluginSection.GetStringValue("RootPath", std::string()); - - if (objectsRootPath.size() >= 1 && objectsRootPath[0] == '/') - { - OrthancPlugins::LogError(std::string(StoragePluginFactory::GetStoragePluginName()) + ": The RootPath shall not start with a '/': " + objectsRootPath); return -1; } - plugin->SetRootPath(objectsRootPath); + const char* pluginSectionName = plugin->GetConfigurationSectionName(); + static const char* const ENCRYPTION_SECTION = "StorageEncryption"; - if (pluginSection.IsSection(ENCRYPTION_SECTION)) + if (orthancConfig.IsSection(pluginSectionName)) { - OrthancPlugins::OrthancConfiguration cryptoSection; - pluginSection.GetSection(cryptoSection, ENCRYPTION_SECTION); + OrthancPlugins::OrthancConfiguration pluginSection; + orthancConfig.GetSection(pluginSection, pluginSectionName); + + migrationFromFileSystemEnabled = pluginSection.GetBooleanValue("MigrationFromFileSystemEnabled", false); + + if (migrationFromFileSystemEnabled) + { + fileSystemRootPath = orthancConfig.GetStringValue("StorageDirectory", "OrthancStorageNotDefined"); + OrthancPlugins::LogWarning(std::string(StoragePluginFactory::GetStoragePluginName()) + ": migration from file system enabled, source: " + fileSystemRootPath); + } + + objectsRootPath = pluginSection.GetStringValue("RootPath", std::string()); + + if (objectsRootPath.size() >= 1 && objectsRootPath[0] == '/') + { + OrthancPlugins::LogError(std::string(StoragePluginFactory::GetStoragePluginName()) + ": The RootPath shall not start with a '/': " + objectsRootPath); + return -1; + } - crypto.reset(EncryptionConfigurator::CreateEncryptionHelpers(cryptoSection)); - cryptoEnabled = crypto.get() != nullptr; + plugin->SetRootPath(objectsRootPath); + + if (pluginSection.IsSection(ENCRYPTION_SECTION)) + { + OrthancPlugins::OrthancConfiguration cryptoSection; + pluginSection.GetSection(cryptoSection, ENCRYPTION_SECTION); + + crypto.reset(EncryptionConfigurator::CreateEncryptionHelpers(cryptoSection)); + cryptoEnabled = crypto.get() != nullptr; + } + + if (cryptoEnabled) + { + OrthancPlugins::LogWarning(std::string(StoragePluginFactory::GetStoragePluginName()) + ": client-side encryption is enabled"); + } + else + { + OrthancPlugins::LogWarning(std::string(StoragePluginFactory::GetStoragePluginName()) + ": client-side encryption is disabled"); + } } - if (cryptoEnabled) - { - OrthancPlugins::LogWarning(std::string(StoragePluginFactory::GetStoragePluginName()) + ": client-side encryption is enabled"); - } - else - { - OrthancPlugins::LogWarning(std::string(StoragePluginFactory::GetStoragePluginName()) + ": client-side encryption is disabled"); - } + OrthancPluginRegisterStorageArea(context, StorageCreate, StorageRead, StorageRemove); } - - OrthancPluginRegisterStorageArea(context, StorageCreate, StorageRead, StorageRemove); + catch (Orthanc::OrthancException& e) + { + LOG(ERROR) << "Exception while creating the object storage plugin: " << e.What(); + return -1; + } return 0; }
--- a/Google/CMakeLists.txt Fri Oct 09 10:16:50 2020 +0200 +++ b/Google/CMakeLists.txt Fri Oct 09 10:23:27 2020 +0200 @@ -17,6 +17,9 @@ set(ENABLE_GOOGLE_TEST ON) set(ORTHANC_FRAMEWORK_PLUGIN ON) +set(ENABLE_MODULE_IMAGES OFF) +set(ENABLE_MODULE_JOBS OFF) +set(ENABLE_MODULE_DICOM OFF) include(${ORTHANC_FRAMEWORK_ROOT}/../Resources/CMake/OrthancFrameworkConfiguration.cmake) include(${ORTHANC_FRAMEWORK_ROOT}/../../OrthancServer/Plugins/Samples/Common/OrthancPluginsExports.cmake)
--- a/NEWS Fri Oct 09 10:16:50 2020 +0200 +++ b/NEWS Fri Oct 09 10:23:27 2020 +0200 @@ -1,6 +1,10 @@ Pending changes in the mainline =============================== +* Allow compilation of the AWS S3 plugin without vcpkg +* Added "VirtualAddressing" configuration option in the AWS S3 plugin (for compatibility with minio) + + 2020-09-07 - v 1.1.0 ====================
--- a/UnitTestsSources/EncryptionTests.cpp Fri Oct 09 10:16:50 2020 +0200 +++ b/UnitTestsSources/EncryptionTests.cpp Fri Oct 09 10:23:27 2020 +0200 @@ -32,8 +32,8 @@ ASSERT_NE(key1, key2); - ASSERT_EQ(32, key1.size()); // right now, we work with 256bits key - ASSERT_EQ(32*2, EncryptionHelpers::ToHexString(key1).size()); + ASSERT_EQ(32u, key1.size()); // right now, we work with 256bits key + ASSERT_EQ(32u * 2u, EncryptionHelpers::ToHexString(key1).size()); } TEST(EncryptionHelpers, EncryptDecryptSimpleText)