# HG changeset patch # User Alain Mazy # Date 1655825376 -7200 # Node ID eec3e4a91663e8e562590225189efa44d273bcae # Parent d6ed4c73c7190714c4f5f52b198cc8ddb437ff73 DelayedDeletion plugin: first version diff -r d6ed4c73c719 -r eec3e4a91663 NEWS --- a/NEWS Tue Jun 21 17:29:17 2022 +0200 +++ b/NEWS Tue Jun 21 17:29:36 2022 +0200 @@ -4,6 +4,8 @@ General ------- +* New sample plugin: "DelayedDeletion" that will delete files from disk + asynchronously to speed up deletion of large studies. * Lua: new "SetHttpTimeout" function * Lua: new "OnHeartBeat" callback called at regular interval provided that you have configured "LuaHeartBeatPeriod" > 0. @@ -25,6 +27,11 @@ * /system is now reporting "DatabaseServerIdentifier" * Added an Asynchronous mode to /modalities/../move. +Plugins +------- + +* New function in the SDK: "OrthancPluginGetDatabaseServerIdentifier" + Version 1.11.0 (2022-05-09) =========================== diff -r d6ed4c73c719 -r eec3e4a91663 OrthancFramework/Sources/Logging.cpp --- a/OrthancFramework/Sources/Logging.cpp Tue Jun 21 17:29:17 2022 +0200 +++ b/OrthancFramework/Sources/Logging.cpp Tue Jun 21 17:29:36 2022 +0200 @@ -698,6 +698,8 @@ boost::mutex::scoped_lock lock(loggingStreamsMutex_); loggingStreamsContext_.reset(NULL); pluginContext_ = reinterpret_cast(pluginContext); + + EnableInfoLevel(true); // allow the plugin to log at info level (but the Orthanc Core still decides of the level) } diff -r d6ed4c73c719 -r eec3e4a91663 OrthancServer/Plugins/Samples/DelayedDeletion/CMakeLists.txt --- a/OrthancServer/Plugins/Samples/DelayedDeletion/CMakeLists.txt Tue Jun 21 17:29:17 2022 +0200 +++ b/OrthancServer/Plugins/Samples/DelayedDeletion/CMakeLists.txt Tue Jun 21 17:29:36 2022 +0200 @@ -56,7 +56,6 @@ -DORTHANC_PLUGIN_NAME="${PLUGIN_NAME}" -DORTHANC_PLUGIN_VERSION="${PLUGIN_VERSION}" -DORTHANC_ENABLE_LOGGING=1 - -DORTHANC_ENABLE_LOGGING_PLUGIN=1 -DORTHANC_ENABLE_PLUGINS=1 -DORTHANC_BUILDING_SERVER_LIBRARY=0 ) @@ -71,7 +70,6 @@ ${AUTOGENERATED_SOURCES} ${ORTHANC_CORE_SOURCES} ${CMAKE_SOURCE_DIR}/../../../Plugins/Samples/DelayedDeletion/Plugin.cpp - ${CMAKE_SOURCE_DIR}/../../../Plugins/Samples/DelayedDeletion/LargeDeleteJob.cpp ${CMAKE_SOURCE_DIR}/../../../Plugins/Samples/DelayedDeletion/PendingDeletionsDatabase.cpp ${CMAKE_SOURCE_DIR}/../../../Plugins/Engine/PluginsEnumerations.cpp ${CMAKE_SOURCE_DIR}/../../../Plugins/Samples/Common/OrthancPluginCppWrapper.cpp diff -r d6ed4c73c719 -r eec3e4a91663 OrthancServer/Plugins/Samples/DelayedDeletion/Plugin.cpp --- a/OrthancServer/Plugins/Samples/DelayedDeletion/Plugin.cpp Tue Jun 21 17:29:17 2022 +0200 +++ b/OrthancServer/Plugins/Samples/DelayedDeletion/Plugin.cpp Tue Jun 21 17:29:36 2022 +0200 @@ -1,17 +1,14 @@ #include "PendingDeletionsDatabase.h" -#include "LargeDeleteJob.h" #include "../../../../OrthancFramework/Sources/FileStorage/FilesystemStorage.h" #include "../../../../OrthancFramework/Sources/Logging.h" #include "../../../../OrthancFramework/Sources/MultiThreading/SharedMessageQueue.h" #include "../../../../OrthancServer/Plugins/Engine/PluginsEnumerations.h" +#include "../../../../OrthancServer/Plugins/Samples/Common/OrthancPluginCppWrapper.h" #include -#define ASYNCHRONOUS_SQLITE 0 - - class PendingDeletion : public Orthanc::IDynamicObject { private: @@ -38,16 +35,14 @@ }; - - - -static bool continue_; +static const char* DELAYED_DELETION = "DelayedDeletion"; +static bool continue_ = false; static Orthanc::SharedMessageQueue queue_; static std::unique_ptr storage_; static std::unique_ptr db_; -static std::unique_ptr databaseThread_; static std::unique_ptr deletionThread_; - +static const char* databaseServerIdentifier_ = NULL; +static unsigned int throttleDelayMs_ = 0; static OrthancPluginErrorCode StorageCreate(const char* uuid, @@ -134,11 +129,8 @@ { try { -#if ASYNCHRONOUS_SQLITE == 1 - queue_.Enqueue(new PendingDeletion(Orthanc::Plugins::Convert(type), uuid)); -#else + LOG(INFO) << "DelayedDeletion - Scheduling delayed deletion of " << uuid; db_->Enqueue(uuid, Orthanc::Plugins::Convert(type)); -#endif return OrthancPluginErrorCode_Success; } @@ -152,32 +144,10 @@ } } - -static void DatabaseWorker() -{ -#if ASYNCHRONOUS_SQLITE == 1 - while (continue_) - { - for (;;) - { - std::auto_ptr obj(queue_.Dequeue(100)); - if (obj.get() == NULL) - { - break; - } - else - { - const PendingDeletion& deletion = dynamic_cast(*obj); - db_->Enqueue(deletion.GetUuid(), deletion.GetType()); - } - } - } -#endif -} - - static void DeletionWorker() { + static const unsigned int GRANULARITY = 100; // In milliseconds + while (continue_) { std::string uuid; @@ -185,32 +155,37 @@ bool hasDeleted = false; - while (db_->Dequeue(uuid, type)) + while (continue_ && db_->Dequeue(uuid, type)) { if (!hasDeleted) { - LOG(INFO) << "TEST DELETION - Starting to process the pending deletions"; + LOG(INFO) << "DelayedDeletion - Starting to process the pending deletions"; } hasDeleted = true; try { - LOG(INFO) << "TEST DELETION - Asynchronous removal of file: " << uuid; + LOG(INFO) << "DelayedDeletion - Asynchronous removal of file: " << uuid; storage_->Remove(uuid, type); + + if (throttleDelayMs_ > 0) + { + boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelayMs_)); + } } - catch (Orthanc::OrthancException&) + catch (Orthanc::OrthancException& ex) { - LOG(ERROR) << "Cannot remove file: " << uuid; + LOG(ERROR) << "DelayedDeletion - Cannot remove file: " << uuid << " " << ex.What(); } } if (hasDeleted) { - LOG(INFO) << "TEST DELETION - All the pending deletions have been completed"; + LOG(INFO) << "DelayedDeletion - All the pending deletions have been completed"; } - boost::this_thread::sleep(boost::posix_time::milliseconds(100)); + boost::this_thread::sleep(boost::posix_time::milliseconds(GRANULARITY)); } } @@ -222,20 +197,18 @@ switch (changeType) { case OrthancPluginChangeType_OrthancStarted: - assert(deletionThread_.get() == NULL && - databaseThread_.get() == NULL); + assert(deletionThread_.get() == NULL); - LOG(WARNING) << "TEST DELETION - Starting the threads"; + LOG(WARNING) << "DelayedDeletion - Starting the deletion thread"; continue_ = true; deletionThread_.reset(new boost::thread(DeletionWorker)); - databaseThread_.reset(new boost::thread(DatabaseWorker)); break; case OrthancPluginChangeType_OrthancStopped: if (deletionThread_.get() != NULL) { - LOG(WARNING) << "TEST DELETION - Stopping the deletion thread"; + LOG(WARNING) << "DelayedDeletion - Stopping the deletion thread"; continue_ = false; if (deletionThread_->joinable()) { @@ -243,16 +216,6 @@ } } - if (databaseThread_.get() != NULL) - { - LOG(WARNING) << "TEST DELETION - Stopping the database thread"; - continue_ = false; - if (databaseThread_->joinable()) - { - databaseThread_->join(); - } - } - break; default: @@ -264,16 +227,16 @@ -void Statistics(OrthancPluginRestOutput* output, +void GetPluginStatus(OrthancPluginRestOutput* output, const char* url, const OrthancPluginHttpRequest* request) { - Json::Value stats; - OrthancPlugins::RestApiGet(stats, "/statistics", false); - stats["PendingDeletions"] = db_->GetSize(); + Json::Value status; + status["FilesPendingDeletion"] = db_->GetSize(); + status["DatabaseServerIdentifier"] = databaseServerIdentifier_; - std::string s = stats.toStyledString(); + std::string s = status.toStyledString(); OrthancPluginAnswerBuffer(OrthancPlugins::GetGlobalContext(), output, s.c_str(), s.size(), "application/json"); } @@ -284,6 +247,10 @@ { ORTHANC_PLUGINS_API int32_t OrthancPluginInitialize(OrthancPluginContext* context) { + OrthancPlugins::SetGlobalContext(context); + Orthanc::Logging::InitializePluginContext(context); + + /* Check the version of the Orthanc core */ if (OrthancPluginCheckVersion(context) == 0) { @@ -297,44 +264,50 @@ return -1; } - Orthanc::Logging::InitializePluginContext(context); - - OrthancPlugins::SetGlobalContext(context); OrthancPluginSetDescription(context, "Plugin removing files from storage asynchronously."); - OrthancPlugins::OrthancConfiguration config; + OrthancPlugins::OrthancConfiguration orthancConfig; - if (config.GetBooleanValue("DelayedDeletionEnabled", false)) + if (!orthancConfig.IsSection(DELAYED_DELETION)) { - // Json::Value system; - // OrthancPlugins::RestApiGet(system, "/system", false); - // const std::string& databaseIdentifier = system["DatabaseIdentifier"].asString(); - std::string databaseServerIdentifier = config.GetDatabaseServerIdentifier(); + LOG(WARNING) << "DelayedDeletion - plugin is loaded but not enabled (no \"DelayedDeletion\" section found in configuration)"; + return 0; + } + + OrthancPlugins::OrthancConfiguration delayedDeletionConfig; + orthancConfig.GetSection(delayedDeletionConfig, DELAYED_DELETION); - std::string pathStorage = config.GetStringValue("StorageDirectory", "OrthancStorage"); + if (delayedDeletionConfig.GetBooleanValue("Enable", true)) + { + databaseServerIdentifier_ = OrthancPluginGetDatabaseServerIdentifier(context); + throttleDelayMs_ = delayedDeletionConfig.GetUnsignedIntegerValue("ThrottleDelayMs", 0); // delay in ms + + + std::string pathStorage = orthancConfig.GetStringValue("StorageDirectory", "OrthancStorage"); LOG(WARNING) << "DelayedDeletion - Path to the storage area: " << pathStorage; storage_.reset(new Orthanc::FilesystemStorage(pathStorage)); - boost::filesystem::path p = boost::filesystem::path(pathStorage) / ("pending-deletions." + databaseServerIdentifier + ".db"); - LOG(WARNING) << "DelayedDeletion - Path to the SQLite database: " << p.string(); + boost::filesystem::path defaultDbPath = boost::filesystem::path(pathStorage) / (std::string("pending-deletions.") + databaseServerIdentifier_ + ".db"); + std::string dbPath = delayedDeletionConfig.GetStringValue("Path", defaultDbPath.string()); + + LOG(WARNING) << "DelayedDeletion - Path to the SQLite database: " << dbPath; // This must run after the allocation of "storage_", to make sure // that the folder actually exists - db_.reset(new PendingDeletionsDatabase(p.string())); + db_.reset(new PendingDeletionsDatabase(dbPath)); OrthancPluginRegisterStorageArea2(context, StorageCreate, StorageReadWhole, StorageReadRange, StorageRemove); OrthancPluginRegisterOnChangeCallback(context, OnChangeCallback); + + OrthancPlugins::RegisterRestCallback(std::string("/plugins/") + ORTHANC_PLUGIN_NAME + "/status", true); } else { - LOG(WARNING) << "DelayedDeletion - plugin is loaded but not enabled (check your \"DelayedDeletionEnabled\" configuration)"; + LOG(WARNING) << "DelayedDeletion - plugin is loaded but disabled (check your \"DelayedDeletion.Enable\" configuration)"; } - OrthancPlugins::RegisterRestCallback("/tools/large-delete", true); - OrthancPlugins::RegisterRestCallback("/statistics", true); - return 0; }