Mercurial > hg > orthanc
changeset 5024:c2ebc47f4f18 delayed-deletion
wip: adding DelayedDeletion plugin
author | Alain Mazy <am@osimis.io> |
---|---|
date | Mon, 20 Jun 2022 16:53:21 +0200 |
parents | 559b35d18ef7 |
children | 6fd815fae50f |
files | NEWS OrthancFramework/Resources/CMake/OrthancFrameworkParameters.cmake OrthancServer/CMakeLists.txt OrthancServer/Plugins/Samples/ConnectivityChecks/CMakeLists.txt OrthancServer/Plugins/Samples/DelayedDeletion/CMakeLists.txt OrthancServer/Plugins/Samples/DelayedDeletion/LargeDeleteJob.cpp OrthancServer/Plugins/Samples/DelayedDeletion/LargeDeleteJob.h OrthancServer/Plugins/Samples/DelayedDeletion/PendingDeletionsDatabase.cpp OrthancServer/Plugins/Samples/DelayedDeletion/PendingDeletionsDatabase.h OrthancServer/Plugins/Samples/DelayedDeletion/Plugin.cpp OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp |
diffstat | 11 files changed, 1027 insertions(+), 7 deletions(-) [+] |
line wrap: on
line diff
--- a/NEWS Wed Jun 15 15:40:07 2022 +0200 +++ b/NEWS Mon Jun 20 16:53:21 2022 +0200 @@ -17,6 +17,14 @@ DicomControlUserConnection::SetupPresentationContexts() * Improved HttpClient error logging (add method + url) + +REST API +-------- + +* API version upgraded to 18 +* /system is now reporting "DatabaseServerIdentifier" + + Version 1.11.0 (2022-05-09) ===========================
--- a/OrthancFramework/Resources/CMake/OrthancFrameworkParameters.cmake Wed Jun 15 15:40:07 2022 +0200 +++ b/OrthancFramework/Resources/CMake/OrthancFrameworkParameters.cmake Mon Jun 20 16:53:21 2022 +0200 @@ -38,7 +38,7 @@ # Version of the Orthanc API, can be retrieved from "/system" URI in # order to check whether new URI endpoints are available even if using # the mainline version of Orthanc -set(ORTHANC_API_VERSION "17") +set(ORTHANC_API_VERSION "18") #####################################################################
--- a/OrthancServer/CMakeLists.txt Wed Jun 15 15:40:07 2022 +0200 +++ b/OrthancServer/CMakeLists.txt Mon Jun 20 16:53:21 2022 +0200 @@ -60,6 +60,7 @@ SET(BUILD_SERVE_FOLDERS ON CACHE BOOL "Whether to build the ServeFolders plugin") SET(BUILD_CONNECTIVITY_CHECKS ON CACHE BOOL "Whether to build the ConnectivityChecks plugin") SET(BUILD_HOUSEKEEPER ON CACHE BOOL "Whether to build the Housekeeper plugin") +SET(BUILD_DELAYED_DELETION ON CACHE BOOL "Whether to build the DelayedDeletion plugin") SET(ENABLE_PLUGINS ON CACHE BOOL "Enable plugins") SET(UNIT_TESTS_WITH_HTTP_CONNEXIONS ON CACHE BOOL "Allow unit tests to make HTTP requests") @@ -536,24 +537,28 @@ endif() +if (ENABLE_PLUGINS AND (BUILD_DELAYED_DELETION OR BUILD_CONNECTIVITY_CHECKS)) + include(ExternalProject) + +endif() + ##################################################################### ## Build the "ConnectivityChecks" plugin ##################################################################### if (ENABLE_PLUGINS AND BUILD_CONNECTIVITY_CHECKS) - include(ExternalProject) - set(Flags) + set(ConnectivityChecksFlags) if (CMAKE_TOOLCHAIN_FILE) # Take absolute path to the toolchain get_filename_component(TMP ${CMAKE_TOOLCHAIN_FILE} REALPATH BASE ${CMAKE_SOURCE_DIR}) - list(APPEND Flags -DCMAKE_TOOLCHAIN_FILE=${TMP}) + list(APPEND ConnectivityChecksFlags -DCMAKE_TOOLCHAIN_FILE=${TMP}) endif() if ("${CMAKE_SYSTEM_VERSION}" STREQUAL "LinuxStandardBase") - list(APPEND Flags + list(APPEND ConnectivityChecksFlags -DLSB_CC=${CMAKE_LSB_CC} -DLSB_CXX=${CMAKE_LSB_CXX} ) @@ -566,6 +571,9 @@ # that are too long on our Visual Studio 2008 CIS BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/ConnectivityChecks-build" + # this helps triggering build when changing the external project + BUILD_ALWAYS 1 + CMAKE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR} @@ -574,7 +582,7 @@ -DALLOW_DOWNLOADS=${ALLOW_DOWNLOADS} -DUSE_SYSTEM_BOOST=${USE_SYSTEM_BOOST} -DUSE_LEGACY_JSONCPP=${USE_LEGACY_JSONCPP} - ${Flags} + ${ConnectivityChecksFlags} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} @@ -605,6 +613,75 @@ endif() +##################################################################### +## Build the "DelayedDeletion" plugin +##################################################################### + +if (ENABLE_PLUGINS AND BUILD_DELAYED_DELETION) + + set(DelayedDeletionFlags) + + if (CMAKE_TOOLCHAIN_FILE) + # Take absolute path to the toolchain + get_filename_component(TMP ${CMAKE_TOOLCHAIN_FILE} REALPATH BASE ${CMAKE_SOURCE_DIR}) + list(APPEND DelayedDeletionFlags -DCMAKE_TOOLCHAIN_FILE=${TMP}) + endif() + + if ("${CMAKE_SYSTEM_VERSION}" STREQUAL "LinuxStandardBase") + list(APPEND DelayedDeletionFlags + -DLSB_CC=${CMAKE_LSB_CC} + -DLSB_CXX=${CMAKE_LSB_CXX} + ) + endif() + + externalproject_add(DelayedDeletion + SOURCE_DIR "${CMAKE_SOURCE_DIR}/Plugins/Samples/DelayedDeletion" + + # We explicitly provide a build directory, in order to avoid paths + # that are too long on our Visual Studio 2008 CIS + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/DelayedDeletion-build" + + # this helps triggering build when changing the external project + BUILD_ALWAYS 1 + + CMAKE_ARGS + -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR} + -DPLUGIN_VERSION=${ORTHANC_VERSION} + -DSTATIC_BUILD=${STATIC_BUILD} + -DALLOW_DOWNLOADS=${ALLOW_DOWNLOADS} + -DUSE_SYSTEM_BOOST=${USE_SYSTEM_BOOST} + -DUSE_LEGACY_JSONCPP=${USE_LEGACY_JSONCPP} + ${DelayedDeletionFlags} + + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} + -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} + ) + + if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + if (MSVC) + set(Prefix "") + else() + set(Prefix "lib") # MinGW + endif() + + install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/${Prefix}DelayedDeletion.dll + DESTINATION "lib") + else() + list(GET CMAKE_FIND_LIBRARY_PREFIXES 0 Prefix) + list(GET CMAKE_FIND_LIBRARY_SUFFIXES 0 Suffix) + install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/${Prefix}DelayedDeletion${Suffix} + ${CMAKE_CURRENT_BINARY_DIR}/${Prefix}DelayedDeletion${Suffix}.${ORTHANC_VERSION} + DESTINATION "share/orthanc/plugins") + endif() +endif() + ##################################################################### ## Build the "Housekeeper" plugin
--- a/OrthancServer/Plugins/Samples/ConnectivityChecks/CMakeLists.txt Wed Jun 15 15:40:07 2022 +0200 +++ b/OrthancServer/Plugins/Samples/ConnectivityChecks/CMakeLists.txt Mon Jun 20 16:53:21 2022 +0200 @@ -36,7 +36,7 @@ execute_process( COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/../../../../OrthancFramework/Resources/WindowsResources.py - ${PLUGIN_VERSION} ConnectivityChecks ConnectivityChecks.dll "Orthanc plugin to serve additional folders" + ${PLUGIN_VERSION} ConnectivityChecks ConnectivityChecks.dll "Orthanc plugin to show connectivity status" ERROR_VARIABLE Failure OUTPUT_FILE ${AUTOGENERATED_DIR}/ConnectivityChecks.rc )
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancServer/Plugins/Samples/DelayedDeletion/CMakeLists.txt Mon Jun 20 16:53:21 2022 +0200 @@ -0,0 +1,90 @@ +# Orthanc - A Lightweight, RESTful DICOM Store +# Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics +# Department, University Hospital of Liege, Belgium +# Copyright (C) 2017-2022 Osimis S.A., Belgium +# Copyright (C) 2021-2022 Sebastien Jodogne, ICTEAM UCLouvain, Belgium +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +cmake_minimum_required(VERSION 2.8) +cmake_policy(SET CMP0058 NEW) + +project(DelayedDeletion) + +SET(PLUGIN_NAME "delayed-deletion" CACHE STRING "Name of the plugin") +SET(PLUGIN_VERSION "mainline" CACHE STRING "Version of the plugin") + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../../OrthancFramework/Resources/CMake/OrthancFrameworkParameters.cmake) + +set(ENABLE_SQLITE ON) +set(ENABLE_MODULE_IMAGES OFF) +set(ENABLE_MODULE_JOBS OFF) +set(ENABLE_MODULE_DICOM OFF) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../../OrthancFramework/Resources/CMake/OrthancFrameworkConfiguration.cmake) + +if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + execute_process( + COMMAND + ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/../../../../OrthancFramework/Resources/WindowsResources.py + ${PLUGIN_VERSION} DelayedDeletion DelayedDeletion.dll "Orthanc plugin to delay deletion of files" + ERROR_VARIABLE Failure + OUTPUT_FILE ${AUTOGENERATED_DIR}/DelayedDeletion.rc + ) + + if (Failure) + message(FATAL_ERROR "Error while computing the version information: ${Failure}") + endif() + + list(APPEND ADDITIONAL_RESOURCES ${AUTOGENERATED_DIR}/DelayedDeletion.rc) +endif() + +add_definitions( + -DHAS_ORTHANC_EXCEPTION=1 + -DORTHANC_PLUGIN_NAME="${PLUGIN_NAME}" + -DORTHANC_PLUGIN_VERSION="${PLUGIN_VERSION}" + -DORTHANC_ENABLE_LOGGING=1 + -DORTHANC_ENABLE_LOGGING_PLUGIN=1 + -DORTHANC_ENABLE_PLUGINS=1 + -DORTHANC_BUILDING_SERVER_LIBRARY=0 + ) + +include_directories( + ${CMAKE_SOURCE_DIR}/../../Include/ + ${CMAKE_SOURCE_DIR}/../../../../OrthancFramework/Sources/ + ) + +add_library(DelayedDeletion SHARED + ${ADDITIONAL_RESOURCES} + ${AUTOGENERATED_SOURCES} + ${ORTHANC_CORE_SOURCES} + ${CMAKE_SOURCE_DIR}/../../../Plugins/Samples/DelayedDeletion/Plugin.cpp + ${CMAKE_SOURCE_DIR}/../../../Plugins/Samples/DelayedDeletion/LargeDeleteJob.cpp + ${CMAKE_SOURCE_DIR}/../../../Plugins/Samples/DelayedDeletion/PendingDeletionsDatabase.cpp + ${CMAKE_SOURCE_DIR}/../../../Plugins/Engine/PluginsEnumerations.cpp + ${CMAKE_SOURCE_DIR}/../../../Plugins/Samples/Common/OrthancPluginCppWrapper.cpp + Plugin.cpp + ) + +set_target_properties( + DelayedDeletion PROPERTIES + VERSION ${PLUGIN_VERSION} + SOVERSION ${PLUGIN_VERSION} + ) + +install( + TARGETS DelayedDeletion + DESTINATION . + )
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancServer/Plugins/Samples/DelayedDeletion/LargeDeleteJob.cpp Mon Jun 20 16:53:21 2022 +0200 @@ -0,0 +1,301 @@ +#include "LargeDeleteJob.h" + +#include "../../../../OrthancFramework/Sources/Logging.h" +#include "../../../../OrthancFramework/Sources/OrthancException.h" + +#include <json/reader.h> + +void LargeDeleteJob::UpdateDeleteProgress() +{ + size_t total = 2 * resources_.size() + instances_.size() + series_.size(); + + float progress; + if (total == 0) + { + progress = 1; + } + else + { + progress = (static_cast<float>(posResources_ + posInstances_ + posSeries_ + posDelete_) / + static_cast<float>(total)); + } + + UpdateProgress(progress); +} + + +void LargeDeleteJob::ScheduleChildrenResources(std::vector<std::string>& target, + const std::string& uri) +{ + Json::Value items; + if (OrthancPlugins::RestApiGet(items, uri, false)) + { + if (items.type() != Json::arrayValue) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); + } + + for (Json::Value::ArrayIndex i = 0; i < items.size(); i++) + { + if (items[i].type() != Json::objectValue || + !items[i].isMember("ID") || + items[i]["ID"].type() != Json::stringValue) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); + } + else + { + target.push_back(items[i]["ID"].asString()); + } + } + } +} + + +void LargeDeleteJob::ScheduleResource(Orthanc::ResourceType level, + const std::string& id) +{ +#if 0 + // Instance-level granularity => very slow! + switch (level) + { + case Orthanc::ResourceType_Patient: + ScheduleChildrenResources(instances_, "/patients/" + id + "/instances"); + break; + + case Orthanc::ResourceType_Study: + ScheduleChildrenResources(instances_, "/studies/" + id + "/instances"); + break; + + case Orthanc::ResourceType_Series: + ScheduleChildrenResources(instances_, "/series/" + id + "/instances"); + break; + + case Orthanc::ResourceType_Instance: + instances_.push_back(id); + break; + + default: + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); + } +#else + /** + * Series-level granularity => looks like a good compromise between + * having the Orthanc mutex locked during all the study, and very + * slow instance-level granularity. + **/ + switch (level) + { + case Orthanc::ResourceType_Patient: + ScheduleChildrenResources(series_, "/patients/" + id + "/series"); + break; + + case Orthanc::ResourceType_Study: + ScheduleChildrenResources(series_, "/studies/" + id + "/series"); + break; + + case Orthanc::ResourceType_Series: + series_.push_back(id); + break; + + case Orthanc::ResourceType_Instance: + instances_.push_back(id); + break; + + default: + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); + } +#endif +} + + +void LargeDeleteJob::DeleteResource(Orthanc::ResourceType level, + const std::string& id) +{ + std::string uri; + switch (level) + { + case Orthanc::ResourceType_Patient: + uri = "/patients/" + id; + break; + + case Orthanc::ResourceType_Study: + uri = "/studies/" + id; + break; + + case Orthanc::ResourceType_Series: + uri = "/series/" + id; + break; + + case Orthanc::ResourceType_Instance: + uri = "/instances/" + id; + break; + + default: + throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); + } + + OrthancPlugins::RestApiDelete(uri, false); +} + + +LargeDeleteJob::LargeDeleteJob(const std::vector<std::string>& resources, + const std::vector<Orthanc::ResourceType>& levels) : + OrthancJob("LargeDelete"), + resources_(resources), + levels_(levels), + posResources_(0), + posInstances_(0), + posDelete_(0) +{ + if (resources.size() != levels.size()) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); + } +} + + +OrthancPluginJobStepStatus LargeDeleteJob::Step() +{ + if (posResources_ == 0) + { + if (resources_.size() == 1) + { + // LOG(WARNING) << "LargeDeleteJob has started on resource: " << resources_[0]; + } + else + { + // LOG(WARNING) << "LargeDeleteJob has started"; + } + } + + if (posResources_ < resources_.size()) + { + // First step: Discovering all the instances of the resources + + ScheduleResource(levels_[posResources_], resources_[posResources_]); + + posResources_ += 1; + UpdateDeleteProgress(); + return OrthancPluginJobStepStatus_Continue; + } + else if (posInstances_ < instances_.size()) + { + // Second step: Deleting the instances one by one + + DeleteResource(Orthanc::ResourceType_Instance, instances_[posInstances_]); + + posInstances_ += 1; + UpdateDeleteProgress(); + return OrthancPluginJobStepStatus_Continue; + } + else if (posSeries_ < series_.size()) + { + // Third step: Deleting the series one by one + + DeleteResource(Orthanc::ResourceType_Series, series_[posSeries_]); + + posSeries_ += 1; + UpdateDeleteProgress(); + return OrthancPluginJobStepStatus_Continue; + } + else if (posDelete_ < resources_.size()) + { + // Fourth step: Make sure the resources where fully deleted + // (instances might have been received since the beginning of + // the job) + + DeleteResource(levels_[posDelete_], resources_[posDelete_]); + + posDelete_ += 1; + UpdateDeleteProgress(); + return OrthancPluginJobStepStatus_Continue; + } + else + { + if (resources_.size() == 1) + { + // LOG(WARNING) << "LargeDeleteJob has completed on resource: " << resources_[0]; + } + else + { + // LOG(WARNING) << "LargeDeleteJob has completed"; + } + + UpdateProgress(1); + return OrthancPluginJobStepStatus_Success; + } +} + + +void LargeDeleteJob::Reset() +{ + posResources_ = 0; + posInstances_ = 0; + posDelete_ = 0; + instances_.clear(); +} + + +void LargeDeleteJob::RestHandler(OrthancPluginRestOutput* output, + const char* url, + const OrthancPluginHttpRequest* request) +{ + static const char* KEY_RESOURCES = "Resources"; + + if (request->method != OrthancPluginHttpMethod_Post) + { + OrthancPluginSendMethodNotAllowed(OrthancPlugins::GetGlobalContext(), output, "POST"); + return; + } + + Json::Value body; + Json::Reader reader; + if (!reader.parse(reinterpret_cast<const char*>(request->body), + reinterpret_cast<const char*>(request->body) + request->bodySize, body)) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat, "JSON body is expected"); + } + + if (body.type() != Json::objectValue) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat, + "Expected a JSON object in the body"); + } + + if (!body.isMember(KEY_RESOURCES) || + body[KEY_RESOURCES].type() != Json::arrayValue) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat, + "The JSON object must contain an array in \"" + + std::string(KEY_RESOURCES) + "\""); + } + + std::vector<std::string> resources; + std::vector<Orthanc::ResourceType> levels; + + resources.reserve(body.size()); + levels.reserve(body.size()); + + const Json::Value& arr = body[KEY_RESOURCES]; + for (Json::Value::ArrayIndex i = 0; i < arr.size(); i++) + { + if (arr[i].type() != Json::arrayValue || + arr[i].size() != 2u || + arr[i][0].type() != Json::stringValue || + arr[i][1].type() != Json::stringValue) + { + throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat, + "Each entry must be an array containing 2 strings, " + "the resource level and its ID"); + } + else + { + levels.push_back(Orthanc::StringToResourceType(arr[i][0].asCString())); + resources.push_back(arr[i][1].asString()); + } + } + + OrthancPlugins::OrthancJob::SubmitFromRestApiPost( + output, body, new LargeDeleteJob(resources, levels)); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancServer/Plugins/Samples/DelayedDeletion/LargeDeleteJob.h Mon Jun 20 16:53:21 2022 +0200 @@ -0,0 +1,46 @@ +#pragma once + +#include "../Common/OrthancPluginCppWrapper.h" +#include "../../../../OrthancFramework/Sources/Enumerations.h" +#include "../../../../OrthancFramework/Sources/Compatibility.h" + + +class LargeDeleteJob : public OrthancPlugins::OrthancJob +{ +private: + std::vector<std::string> resources_; + std::vector<Orthanc::ResourceType> levels_; + std::vector<std::string> instances_; + std::vector<std::string> series_; + size_t posResources_; + size_t posInstances_; + size_t posSeries_; + size_t posDelete_; + + void UpdateDeleteProgress(); + + void ScheduleChildrenResources(std::vector<std::string>& target, + const std::string& uri); + + void ScheduleResource(Orthanc::ResourceType level, + const std::string& id); + + void DeleteResource(Orthanc::ResourceType level, + const std::string& id); + +public: + LargeDeleteJob(const std::vector<std::string>& resources, + const std::vector<Orthanc::ResourceType>& levels); + + virtual OrthancPluginJobStepStatus Step() ORTHANC_OVERRIDE; + + virtual void Stop(OrthancPluginJobStopReason reason) ORTHANC_OVERRIDE + { + } + + virtual void Reset() ORTHANC_OVERRIDE; + + static void RestHandler(OrthancPluginRestOutput* output, + const char* url, + const OrthancPluginHttpRequest* request); +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancServer/Plugins/Samples/DelayedDeletion/PendingDeletionsDatabase.cpp Mon Jun 20 16:53:21 2022 +0200 @@ -0,0 +1,112 @@ +#include "PendingDeletionsDatabase.h" + +#include "../../../../OrthancFramework/Sources/SQLite/Statement.h" +#include "../../../../OrthancFramework/Sources/SQLite/Transaction.h" + +void PendingDeletionsDatabase::Setup() +{ + // Performance tuning of SQLite with PRAGMAs + // http://www.sqlite.org/pragma.html + db_.Execute("PRAGMA SYNCHRONOUS=NORMAL;"); + db_.Execute("PRAGMA JOURNAL_MODE=WAL;"); + db_.Execute("PRAGMA LOCKING_MODE=EXCLUSIVE;"); + db_.Execute("PRAGMA WAL_AUTOCHECKPOINT=1000;"); + + { + Orthanc::SQLite::Transaction t(db_); + t.Begin(); + + if (!db_.DoesTableExist("Pending")) + { + db_.Execute("CREATE TABLE Pending(uuid TEXT, type INTEGER)"); + } + + t.Commit(); + } +} + + +PendingDeletionsDatabase::PendingDeletionsDatabase(const std::string& path) +{ + db_.Open(path); + Setup(); +} + + +void PendingDeletionsDatabase::Enqueue(const std::string& uuid, + Orthanc::FileContentType type) +{ + boost::mutex::scoped_lock lock(mutex_); + + Orthanc::SQLite::Transaction t(db_); + t.Begin(); + + { + Orthanc::SQLite::Statement s(db_, SQLITE_FROM_HERE, "INSERT INTO Pending VALUES(?, ?)"); + s.BindString(0, uuid); + s.BindInt(1, type); + s.Run(); + } + + t.Commit(); +} + + +bool PendingDeletionsDatabase::Dequeue(std::string& uuid, + Orthanc::FileContentType& type) +{ + bool ok = false; + + boost::mutex::scoped_lock lock(mutex_); + + Orthanc::SQLite::Transaction t(db_); + t.Begin(); + + { + Orthanc::SQLite::Statement s(db_, SQLITE_FROM_HERE, "SELECT uuid, type FROM Pending LIMIT 1"); + + if (s.Step()) + { + uuid = s.ColumnString(0); + type = static_cast<Orthanc::FileContentType>(s.ColumnInt(1)); + + Orthanc::SQLite::Statement s(db_, SQLITE_FROM_HERE, "DELETE FROM Pending WHERE uuid=?"); + s.BindString(0, uuid); + s.Run(); + + ok = true; + } + } + + t.Commit(); + + return ok; +} + + +unsigned int PendingDeletionsDatabase::GetSize() +{ + boost::mutex::scoped_lock lock(mutex_); + + unsigned int value = 0; + + Orthanc::SQLite::Transaction t(db_); + t.Begin(); + + { + Orthanc::SQLite::Statement s(db_, SQLITE_FROM_HERE, "SELECT COUNT(*) FROM Pending"); + + if (s.Step()) + { + int tmp = s.ColumnInt(0); + if (tmp > 0) + { + value = static_cast<unsigned int>(tmp); + } + } + } + + t.Commit(); + + return value; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancServer/Plugins/Samples/DelayedDeletion/PendingDeletionsDatabase.h Mon Jun 20 16:53:21 2022 +0200 @@ -0,0 +1,26 @@ +#pragma once + + +#include "../../../../OrthancFramework/Sources/SQLite/Connection.h" +#include <boost/thread/mutex.hpp> +#include <boost/noncopyable.hpp> + +class PendingDeletionsDatabase : public boost::noncopyable +{ +private: + boost::mutex mutex_; + Orthanc::SQLite::Connection db_; + + void Setup(); + +public: + PendingDeletionsDatabase(const std::string& path); + + void Enqueue(const std::string& uuid, + Orthanc::FileContentType type); + + bool Dequeue(std::string& uuid, + Orthanc::FileContentType& type); + + unsigned int GetSize(); +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancServer/Plugins/Samples/DelayedDeletion/Plugin.cpp Mon Jun 20 16:53:21 2022 +0200 @@ -0,0 +1,356 @@ +#include "PendingDeletionsDatabase.h" +#include "LargeDeleteJob.h" + +#include "../../../../OrthancFramework/Sources/FileStorage/FilesystemStorage.h" +#include "../../../../OrthancFramework/Sources/Logging.h" +#include "../../../../OrthancFramework/Sources/MultiThreading/SharedMessageQueue.h" +#include "../../../../OrthancServer/Plugins/Engine/PluginsEnumerations.h" + +#include <boost/thread.hpp> + + +#define ASYNCHRONOUS_SQLITE 0 + + +class PendingDeletion : public Orthanc::IDynamicObject +{ +private: + Orthanc::FileContentType type_; + std::string uuid_; + +public: + PendingDeletion(Orthanc::FileContentType type, + const std::string& uuid) : + type_(type), + uuid_(uuid) + { + } + + Orthanc::FileContentType GetType() const + { + return type_; + } + + const std::string& GetUuid() const + { + return uuid_; + } +}; + + + + + +static bool continue_; +static Orthanc::SharedMessageQueue queue_; +static std::unique_ptr<Orthanc::FilesystemStorage> storage_; +static std::unique_ptr<PendingDeletionsDatabase> db_; +static std::unique_ptr<boost::thread> databaseThread_; +static std::unique_ptr<boost::thread> deletionThread_; + + + +static OrthancPluginErrorCode StorageCreate(const char* uuid, + const void* content, + int64_t size, + OrthancPluginContentType type) +{ + try + { + storage_->Create(uuid, content, size, Orthanc::Plugins::Convert(type)); + return OrthancPluginErrorCode_Success; + } + catch (Orthanc::OrthancException& e) + { + return static_cast<OrthancPluginErrorCode>(e.GetErrorCode()); + } + catch (...) + { + return OrthancPluginErrorCode_StorageAreaPlugin; + } +} + + +static OrthancPluginErrorCode StorageReadWhole(OrthancPluginMemoryBuffer64* target, // Memory buffer where to store the content of the file. It must be allocated by the plugin using OrthancPluginCreateMemoryBuffer64(). The core of Orthanc will free it. + const char* uuid, + OrthancPluginContentType type) +{ + try + { + std::unique_ptr<Orthanc::IMemoryBuffer> buffer(storage_->Read(uuid, Orthanc::Plugins::Convert(type))); + + // copy from a buffer allocated on plugin's heap into a buffer allocated on core's heap + if (OrthancPluginCreateMemoryBuffer64(OrthancPlugins::GetGlobalContext(), target, buffer->GetSize()) != OrthancPluginErrorCode_Success) + { + OrthancPlugins::LogError("Delayed deletion plugin: error while reading object " + std::string(uuid) + ", cannot allocate memory of size " + boost::lexical_cast<std::string>(buffer->GetSize()) + " bytes"); + return OrthancPluginErrorCode_StorageAreaPlugin; + } + + memcpy(target->data, buffer->GetData(), buffer->GetSize()); + + return OrthancPluginErrorCode_Success; + } + catch (Orthanc::OrthancException& e) + { + return static_cast<OrthancPluginErrorCode>(e.GetErrorCode()); + } + catch (...) + { + return OrthancPluginErrorCode_StorageAreaPlugin; + } +} + + +static OrthancPluginErrorCode StorageReadRange(OrthancPluginMemoryBuffer64* target, // Memory buffer where to store the content of the range. The memory buffer is allocated and freed by Orthanc. The length of the range of interest corresponds to the size of this buffer. + const char* uuid, + OrthancPluginContentType type, + uint64_t rangeStart) +{ + try + { + std::unique_ptr<Orthanc::IMemoryBuffer> buffer(storage_->ReadRange(uuid, Orthanc::Plugins::Convert(type), rangeStart, rangeStart + target->size)); + + assert(buffer->GetSize() == target->size); + + memcpy(target->data, buffer->GetData(), buffer->GetSize()); + + return OrthancPluginErrorCode_Success; + } + catch (Orthanc::OrthancException& e) + { + return static_cast<OrthancPluginErrorCode>(e.GetErrorCode()); + } + catch (...) + { + return OrthancPluginErrorCode_StorageAreaPlugin; + } + + return OrthancPluginErrorCode_Success; +} + + +static OrthancPluginErrorCode StorageRemove(const char* uuid, + OrthancPluginContentType type) +{ + try + { +#if ASYNCHRONOUS_SQLITE == 1 + queue_.Enqueue(new PendingDeletion(Orthanc::Plugins::Convert(type), uuid)); +#else + db_->Enqueue(uuid, Orthanc::Plugins::Convert(type)); +#endif + + return OrthancPluginErrorCode_Success; + } + catch (Orthanc::OrthancException& e) + { + return static_cast<OrthancPluginErrorCode>(e.GetErrorCode()); + } + catch (...) + { + return OrthancPluginErrorCode_StorageAreaPlugin; + } +} + + +static void DatabaseWorker() +{ +#if ASYNCHRONOUS_SQLITE == 1 + while (continue_) + { + for (;;) + { + std::auto_ptr<Orthanc::IDynamicObject> obj(queue_.Dequeue(100)); + if (obj.get() == NULL) + { + break; + } + else + { + const PendingDeletion& deletion = dynamic_cast<const PendingDeletion&>(*obj); + db_->Enqueue(deletion.GetUuid(), deletion.GetType()); + } + } + } +#endif +} + + +static void DeletionWorker() +{ + while (continue_) + { + std::string uuid; + Orthanc::FileContentType type = Orthanc::FileContentType_Dicom; // Dummy initialization + + bool hasDeleted = false; + + while (db_->Dequeue(uuid, type)) + { + if (!hasDeleted) + { + LOG(INFO) << "TEST DELETION - Starting to process the pending deletions"; + } + + hasDeleted = true; + + try + { + LOG(INFO) << "TEST DELETION - Asynchronous removal of file: " << uuid; + storage_->Remove(uuid, type); + } + catch (Orthanc::OrthancException&) + { + LOG(ERROR) << "Cannot remove file: " << uuid; + } + } + + if (hasDeleted) + { + LOG(INFO) << "TEST DELETION - All the pending deletions have been completed"; + } + + boost::this_thread::sleep(boost::posix_time::milliseconds(100)); + } +} + + +OrthancPluginErrorCode OnChangeCallback(OrthancPluginChangeType changeType, + OrthancPluginResourceType resourceType, + const char* resourceId) +{ + switch (changeType) + { + case OrthancPluginChangeType_OrthancStarted: + assert(deletionThread_.get() == NULL && + databaseThread_.get() == NULL); + + LOG(WARNING) << "TEST DELETION - Starting the threads"; + continue_ = true; + deletionThread_.reset(new boost::thread(DeletionWorker)); + databaseThread_.reset(new boost::thread(DatabaseWorker)); + break; + + case OrthancPluginChangeType_OrthancStopped: + + if (deletionThread_.get() != NULL) + { + LOG(WARNING) << "TEST DELETION - Stopping the deletion thread"; + continue_ = false; + if (deletionThread_->joinable()) + { + deletionThread_->join(); + } + } + + if (databaseThread_.get() != NULL) + { + LOG(WARNING) << "TEST DELETION - Stopping the database thread"; + continue_ = false; + if (databaseThread_->joinable()) + { + databaseThread_->join(); + } + } + + break; + + default: + break; + } + + return OrthancPluginErrorCode_Success; +} + + + +void Statistics(OrthancPluginRestOutput* output, + const char* url, + const OrthancPluginHttpRequest* request) +{ + Json::Value stats; + OrthancPlugins::RestApiGet(stats, "/statistics", false); + + stats["PendingDeletions"] = db_->GetSize(); + + std::string s = stats.toStyledString(); + OrthancPluginAnswerBuffer(OrthancPlugins::GetGlobalContext(), output, s.c_str(), + s.size(), "application/json"); +} + + + +extern "C" +{ + ORTHANC_PLUGINS_API int32_t OrthancPluginInitialize(OrthancPluginContext* context) + { + /* Check the version of the Orthanc core */ + if (OrthancPluginCheckVersion(context) == 0) + { + char info[1024]; + sprintf(info, "Your version of Orthanc (%s) must be above %d.%d.%d to run this plugin", + context->orthancVersion, + ORTHANC_PLUGINS_MINIMAL_MAJOR_NUMBER, + ORTHANC_PLUGINS_MINIMAL_MINOR_NUMBER, + ORTHANC_PLUGINS_MINIMAL_REVISION_NUMBER); + OrthancPluginLogError(context, info); + return -1; + } + + Orthanc::Logging::InitializePluginContext(context); + + OrthancPlugins::SetGlobalContext(context); + OrthancPluginSetDescription(context, "Plugin removing files from storage asynchronously."); + + OrthancPlugins::OrthancConfiguration config; + + if (config.GetBooleanValue("DelayedDeletionEnabled", false)) + { + // Json::Value system; + // OrthancPlugins::RestApiGet(system, "/system", false); + // const std::string& databaseIdentifier = system["DatabaseIdentifier"].asString(); + std::string databaseServerIdentifier = config.GetDatabaseServerIdentifier(); + + std::string pathStorage = config.GetStringValue("StorageDirectory", "OrthancStorage"); + LOG(WARNING) << "DelayedDeletion - Path to the storage area: " << pathStorage; + + storage_.reset(new Orthanc::FilesystemStorage(pathStorage)); + + boost::filesystem::path p = boost::filesystem::path(pathStorage) / ("pending-deletions." + databaseServerIdentifier + ".db"); + LOG(WARNING) << "DelayedDeletion - Path to the SQLite database: " << p.string(); + + // This must run after the allocation of "storage_", to make sure + // that the folder actually exists + db_.reset(new PendingDeletionsDatabase(p.string())); + + OrthancPluginRegisterStorageArea2(context, StorageCreate, StorageReadWhole, StorageReadRange, StorageRemove); + + OrthancPluginRegisterOnChangeCallback(context, OnChangeCallback); + } + else + { + LOG(WARNING) << "DelayedDeletion - plugin is loaded but not enabled (check your \"DelayedDeletionEnabled\" configuration)"; + } + + OrthancPlugins::RegisterRestCallback<LargeDeleteJob::RestHandler>("/tools/large-delete", true); + OrthancPlugins::RegisterRestCallback<Statistics>("/statistics", true); + + return 0; + } + + ORTHANC_PLUGINS_API void OrthancPluginFinalize() + { + db_.reset(); + storage_.reset(); + } + + ORTHANC_PLUGINS_API const char* OrthancPluginGetName() + { + return ORTHANC_PLUGIN_NAME; + } + + ORTHANC_PLUGINS_API const char* OrthancPluginGetVersion() + { + return ORTHANC_PLUGIN_VERSION; + } +}
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp Wed Jun 15 15:40:07 2022 +0200 +++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp Mon Jun 20 16:53:21 2022 +0200 @@ -64,6 +64,7 @@ static const char* const CHECK_REVISIONS = "CheckRevisions"; static const char* const DATABASE_BACKEND_PLUGIN = "DatabaseBackendPlugin"; static const char* const DATABASE_VERSION = "DatabaseVersion"; + static const char* const DATABASE_SERVER_IDENTIFIER = "DatabaseServerIdentifier"; static const char* const DICOM_AET = "DicomAet"; static const char* const DICOM_PORT = "DicomPort"; static const char* const HTTP_PORT = "HttpPort"; @@ -87,6 +88,8 @@ .SetAnswerField(VERSION, RestApiCallDocumentation::Type_String, "Version of Orthanc") .SetAnswerField(DATABASE_VERSION, RestApiCallDocumentation::Type_Number, "Version of the database: https://book.orthanc-server.com/developers/db-versioning.html") + .SetAnswerField(DATABASE_SERVER_IDENTIFIER, RestApiCallDocumentation::Type_String, + "ID of the server in the database (when running multiple Orthanc on the same DB)") .SetAnswerField(IS_HTTP_SERVER_SECURE, RestApiCallDocumentation::Type_Boolean, "Whether the REST API is properly secured (assuming no reverse proxy is in use): https://book.orthanc-server.com/faq/security.html#securing-the-http-server") .SetAnswerField(STORAGE_AREA_PLUGIN, RestApiCallDocumentation::Type_String, @@ -133,6 +136,7 @@ result[STORAGE_COMPRESSION] = lock.GetConfiguration().GetBooleanParameter(STORAGE_COMPRESSION, false); // New in Orthanc 1.11.0 result[OVERWRITE_INSTANCES] = lock.GetConfiguration().GetBooleanParameter(OVERWRITE_INSTANCES, false); // New in Orthanc 1.11.0 result[INGEST_TRANSCODING] = lock.GetConfiguration().GetStringParameter(INGEST_TRANSCODING, ""); // New in Orthanc 1.11.0 + result[DATABASE_SERVER_IDENTIFIER] = lock.GetConfiguration().GetDatabaseServerIdentifier(); } result[STORAGE_AREA_PLUGIN] = Json::nullValue;