Mercurial > hg > orthanc
changeset 4979:f316413027fd more-tags
renamed DbOptizer into Housekeeper + add Scheduler & triggers
| author | Alain Mazy <am@osimis.io> |
|---|---|
| date | Mon, 25 Apr 2022 15:30:51 +0200 |
| parents | 2cfa50d8eb60 |
| children | 8b95fc86b8d9 |
| files | NEWS OrthancServer/CMakeLists.txt OrthancServer/Plugins/Samples/DbOptimizer/Plugin.cpp OrthancServer/Plugins/Samples/Housekeeper/Plugin.cpp OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp |
| diffstat | 5 files changed, 699 insertions(+), 440 deletions(-) [+] |
line wrap: on
line diff
--- a/NEWS Wed Apr 20 14:36:47 2022 +0200 +++ b/NEWS Mon Apr 25 15:30:51 2022 +0200 @@ -6,12 +6,13 @@ * New configuration "ExtraMainDicomTags" to store more tags in the Index DB to speed up, e.g, building C-Find, dicom-web or tools/find answers -* New sample plugin: "DbOptimizer" that will re-construct the DB/Storage +* New sample plugin: "Housekeeper" that will re-construct the DB/Storage when it detects there is room for improvements, e.g: - if files were stored with a version of Orthanc prior to 1.9.1, the storage might still contain dicom-as-json files that are not needed - anymore - - if "ExtraMainDicomTags" has been changed. + anymore -> it will remove them + - if "ExtraMainDicomTags" has changed. + - if "StorageCompression" has chagned. * New configuration "Warnings" to enable/disable individual warnings that can be identified by a W0XX prefix in the logs. These warnings have been added: @@ -39,6 +40,7 @@ - /instances * new field "MainDicomTags" in the /system route response to list the tags that are saved in DB +* new field "StorageCompression" reported in the /system route response Version 1.10.1 (2022-03-23)
--- a/OrthancServer/CMakeLists.txt Wed Apr 20 14:36:47 2022 +0200 +++ b/OrthancServer/CMakeLists.txt Mon Apr 25 15:30:51 2022 +0200 @@ -59,7 +59,7 @@ SET(BUILD_RECOVER_COMPRESSED_FILE ON CACHE BOOL "Whether to build the companion tool to recover files compressed using Orthanc") SET(BUILD_SERVE_FOLDERS ON CACHE BOOL "Whether to build the ServeFolders plugin") SET(BUILD_CONNECTIVITY_CHECKS ON CACHE BOOL "Whether to build the ConnectivityChecks plugin") -SET(BUILD_DB_OPTIMIZER ON CACHE BOOL "Whether to build the DbOptimizer plugin") +SET(BUILD_HOUSEKEEPER ON CACHE BOOL "Whether to build the Housekeeper plugin") SET(ENABLE_PLUGINS ON CACHE BOOL "Enable plugins") SET(UNIT_TESTS_WITH_HTTP_CONNEXIONS ON CACHE BOOL "Allow unit tests to make HTTP requests") @@ -607,7 +607,7 @@ ##################################################################### -## Build the "DbOptimizer" plugin +## Build the "Housekeeper" plugin ##################################################################### if (ENABLE_PLUGINS AND BUILD_DB_OPTIMIZER) @@ -615,35 +615,35 @@ execute_process( COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/../OrthancFramework/Resources/WindowsResources.py - ${ORTHANC_VERSION} DbOptimizer DbOptimizer.dll "Sample Orthanc plugin to optimizer/clean the DB/Storage" + ${ORTHANC_VERSION} Housekeeper Housekeeper.dll "Sample Orthanc plugin to optimize/clean the DB/Storage" ERROR_VARIABLE Failure - OUTPUT_FILE ${AUTOGENERATED_DIR}/DbOptimizer.rc + OUTPUT_FILE ${AUTOGENERATED_DIR}/Housekeeper.rc ) if (Failure) message(FATAL_ERROR "Error while computing the version information: ${Failure}") endif() -list(APPEND DB_OPTIMIZER_RESOURCES ${AUTOGENERATED_DIR}/DbOptimizer.rc) +list(APPEND HOUSEKEEPER_RESOURCES ${AUTOGENERATED_DIR}/Housekeeper.rc) endif() -add_library(DbOptimizer SHARED -${CMAKE_SOURCE_DIR}/Plugins/Samples/DbOptimizer/Plugin.cpp -${DB_OPTIMIZER_RESOURCES} +add_library(Housekeeper SHARED +${CMAKE_SOURCE_DIR}/Plugins/Samples/Housekeeper/Plugin.cpp +${HOUSEKEEPER_RESOURCES} ) -target_link_libraries(DbOptimizer +target_link_libraries(Housekeeper ThirdPartyPlugins ) set_target_properties( - DbOptimizer PROPERTIES + Housekeeper PROPERTIES VERSION ${ORTHANC_VERSION} SOVERSION ${ORTHANC_VERSION} ) install( -TARGETS DbOptimizer +TARGETS Housekeeper RUNTIME DESTINATION lib # Destination for Windows LIBRARY DESTINATION share/orthanc/plugins # Destination for Linux )
--- a/OrthancServer/Plugins/Samples/DbOptimizer/Plugin.cpp Wed Apr 20 14:36:47 2022 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,426 +0,0 @@ -/** - * Orthanc - A Lightweight, RESTful DICOM Store - * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics - * Department, University Hospital of Liege, Belgium - * Copyright (C) 2017-2022 Osimis S.A., Belgium - * Copyright (C) 2021-2022 Sebastien Jodogne, ICTEAM UCLouvain, Belgium - * - * This program is free software: you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - **/ - - -#include "../../../../OrthancFramework/Sources/Compatibility.h" -#include "../Common/OrthancPluginCppWrapper.h" - -#include <boost/thread.hpp> -#include <json/value.h> -#include <json/writer.h> -#include <string.h> -#include <iostream> -#include <algorithm> -#include <map> - -static int globalPropertyId_ = 0; -static bool force_ = false; -static uint throttleDelay_ = 0; -static std::unique_ptr<boost::thread> workerThread_; -static bool workerThreadShouldStop = false; - -struct DbConfiguration -{ - std::string orthancVersion; - std::map<OrthancPluginResourceType, std::string> mainDicomTagsSignature; - - DbConfiguration() - { - } - - bool IsDefined() const - { - return !orthancVersion.empty() && mainDicomTagsSignature.size() == 4; - } - - void Clear() - { - orthancVersion.clear(); - mainDicomTagsSignature.clear(); - } - - void ToJson(Json::Value& target) - { - if (!IsDefined()) - { - target = Json::nullValue; - } - else - { - Json::Value signatures; - - target = Json::objectValue; - - // default main dicom tags signature are the one from Orthanc 1.4.2 (last time the list was changed): - signatures["Patient"] = mainDicomTagsSignature[OrthancPluginResourceType_Patient]; - signatures["Study"] = mainDicomTagsSignature[OrthancPluginResourceType_Study]; - signatures["Series"] = mainDicomTagsSignature[OrthancPluginResourceType_Series]; - signatures["Instance"] = mainDicomTagsSignature[OrthancPluginResourceType_Instance]; - - target["MainDicomTagsSignature"] = signatures; - target["OrthancVersion"] = orthancVersion; - } - } - - void FromJson(Json::Value& source) - { - if (!source.isNull()) - { - orthancVersion = source["OrthancVersion"].asString(); - - const Json::Value& signatures = source["MainDicomTagsSignature"]; - mainDicomTagsSignature[OrthancPluginResourceType_Patient] = signatures["Patient"].asString(); - mainDicomTagsSignature[OrthancPluginResourceType_Study] = signatures["Study"].asString(); - mainDicomTagsSignature[OrthancPluginResourceType_Series] = signatures["Series"].asString(); - mainDicomTagsSignature[OrthancPluginResourceType_Instance] = signatures["Instance"].asString(); - } - } -}; - -struct PluginStatus -{ - int statusVersion; - int64_t lastProcessedChange; - int64_t lastChangeToProcess; - - DbConfiguration currentlyProcessingConfiguration; // last configuration being processed (has not reached last change yet) - DbConfiguration lastProcessedConfiguration; // last configuration that has been fully processed (till last change) - - PluginStatus() - : statusVersion(1), - lastProcessedChange(-1), - lastChangeToProcess(-1) - { - } - - void ToJson(Json::Value& target) - { - target = Json::objectValue; - - target["Version"] = statusVersion; - target["LastProcessedChange"] = Json::Value::Int64(lastProcessedChange); - target["LastChangeToProcess"] = Json::Value::Int64(lastChangeToProcess); - - currentlyProcessingConfiguration.ToJson(target["CurrentlyProcessingConfiguration"]); - lastProcessedConfiguration.ToJson(target["LastProcessedConfiguration"]); - } - - void FromJson(Json::Value& source) - { - statusVersion = source["Version"].asInt(); - lastProcessedChange = source["LastProcessedChange"].asInt64(); - lastChangeToProcess = source["LastChangeToProcess"].asInt64(); - - Json::Value& current = source["CurrentlyProcessingConfiguration"]; - Json::Value& last = source["LastProcessedConfiguration"]; - - currentlyProcessingConfiguration.FromJson(current); - lastProcessedConfiguration.FromJson(last); - } -}; - - -static void ReadStatusFromDb(PluginStatus& pluginStatus) -{ - OrthancPlugins::OrthancString globalPropertyContent; - - globalPropertyContent.Assign(OrthancPluginGetGlobalProperty(OrthancPlugins::GetGlobalContext(), - globalPropertyId_, - "")); - - if (!globalPropertyContent.IsNullOrEmpty()) - { - Json::Value jsonStatus; - globalPropertyContent.ToJson(jsonStatus); - pluginStatus.FromJson(jsonStatus); - } - else - { - // default config - pluginStatus.statusVersion = 1; - pluginStatus.lastProcessedChange = -1; - pluginStatus.lastChangeToProcess = -1; - - pluginStatus.currentlyProcessingConfiguration.orthancVersion = "1.9.0"; // when we don't know, we assume some files were stored with Orthanc 1.9.0 (last version saving the dicom-as-json files) - - // default main dicom tags signature are the one from Orthanc 1.4.2 (last time the list was changed): - pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Patient] = "0010,0010;0010,0020;0010,0030;0010,0040;0010,1000"; - pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Study] = "0008,0020;0008,0030;0008,0050;0008,0080;0008,0090;0008,1030;0020,000d;0020,0010;0032,1032;0032,1060"; - pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Series] = "0008,0021;0008,0031;0008,0060;0008,0070;0008,1010;0008,103e;0008,1070;0018,0010;0018,0015;0018,0024;0018,1030;0018,1090;0018,1400;0020,000e;0020,0011;0020,0037;0020,0105;0020,1002;0040,0254;0054,0081;0054,0101;0054,1000"; - pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Instance] = "0008,0012;0008,0013;0008,0018;0020,0012;0020,0013;0020,0032;0020,0037;0020,0100;0020,4000;0028,0008;0054,1330"; - } -} - -static void SaveStatusInDb(PluginStatus& pluginStatus) -{ - Json::Value jsonStatus; - pluginStatus.ToJson(jsonStatus); - - Json::StreamWriterBuilder builder; - builder.settings_["indentation"] = " "; - std::string serializedStatus = Json::writeString(builder, jsonStatus); - - OrthancPluginSetGlobalProperty(OrthancPlugins::GetGlobalContext(), - globalPropertyId_, - serializedStatus.c_str()); -} - -static void GetCurrentDbConfiguration(DbConfiguration& configuration) -{ - Json::Value signatures; - Json::Value systemInfo; - - OrthancPlugins::RestApiGet(systemInfo, "/system", false); - configuration.mainDicomTagsSignature[OrthancPluginResourceType_Patient] = systemInfo["MainDicomTags"]["Patient"].asString(); - configuration.mainDicomTagsSignature[OrthancPluginResourceType_Study] = systemInfo["MainDicomTags"]["Study"].asString(); - configuration.mainDicomTagsSignature[OrthancPluginResourceType_Series] = systemInfo["MainDicomTags"]["Series"].asString(); - configuration.mainDicomTagsSignature[OrthancPluginResourceType_Instance] = systemInfo["MainDicomTags"]["Instance"].asString(); - - configuration.orthancVersion = OrthancPlugins::GetGlobalContext()->orthancVersion; -} - -static bool NeedsProcessing(const DbConfiguration& current, const DbConfiguration& last) -{ - if (!last.IsDefined()) - { - return true; - } - - const char* lastVersion = last.orthancVersion.c_str(); - const std::map<OrthancPluginResourceType, std::string>& lastTags = last.mainDicomTagsSignature; - const std::map<OrthancPluginResourceType, std::string>& currentTags = current.mainDicomTagsSignature; - bool needsProcessing = false; - - if (!OrthancPlugins::CheckMinimalVersion(lastVersion, 1, 9, 1)) - { - OrthancPlugins::LogWarning("DbOptimizer: your storage might still contain some dicom-as-json files -> will reconstruct DB"); - needsProcessing = true; - } - - if (lastTags.at(OrthancPluginResourceType_Patient) != currentTags.at(OrthancPluginResourceType_Patient)) - { - OrthancPlugins::LogWarning("DbOptimizer: Patient main dicom tags have changed, -> will reconstruct DB"); - needsProcessing = true; - } - - if (lastTags.at(OrthancPluginResourceType_Study) != currentTags.at(OrthancPluginResourceType_Study)) - { - OrthancPlugins::LogWarning("DbOptimizer: Study main dicom tags have changed, -> will reconstruct DB"); - needsProcessing = true; - } - - if (lastTags.at(OrthancPluginResourceType_Series) != currentTags.at(OrthancPluginResourceType_Series)) - { - OrthancPlugins::LogWarning("DbOptimizer: Series main dicom tags have changed, -> will reconstruct DB"); - needsProcessing = true; - } - - if (lastTags.at(OrthancPluginResourceType_Instance) != currentTags.at(OrthancPluginResourceType_Instance)) - { - OrthancPlugins::LogWarning("DbOptimizer: Instance main dicom tags have changed, -> will reconstruct DB"); - needsProcessing = true; - } - - return needsProcessing; -} - -static bool ProcessChanges(PluginStatus& pluginStatus, const DbConfiguration& currentDbConfiguration) -{ - Json::Value changes; - - pluginStatus.currentlyProcessingConfiguration = currentDbConfiguration; - - OrthancPlugins::RestApiGet(changes, "/changes?since=" + boost::lexical_cast<std::string>(pluginStatus.lastProcessedChange) + "&limit=100", false); - - for (Json::ArrayIndex i = 0; i < changes["Changes"].size(); i++) - { - const Json::Value& change = changes["Changes"][i]; - int64_t seq = change["Seq"].asInt64(); - - if (change["ChangeType"] == "NewStudy") // some StableStudy might be missing if orthanc was shutdown during a StableAge -> consider only the NewStudy events that can not be missed - { - Json::Value result; - OrthancPlugins::RestApiPost(result, "/studies/" + change["ID"].asString() + "/reconstruct", std::string(""), false); - boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelay_*1000)); - } - - if (seq >= pluginStatus.lastChangeToProcess) // we are done ! - { - return true; - } - - pluginStatus.lastProcessedChange = seq; - } - - return false; -} - - -static void WorkerThread() -{ - PluginStatus pluginStatus; - DbConfiguration currentDbConfiguration; - - OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "Starting DB optimizer worker thread"); - - ReadStatusFromDb(pluginStatus); - GetCurrentDbConfiguration(currentDbConfiguration); - - if (!NeedsProcessing(currentDbConfiguration, pluginStatus.lastProcessedConfiguration)) - { - OrthancPlugins::LogWarning("DbOptimizer: everything has been processed already !"); - return; - } - - if (force_ || NeedsProcessing(currentDbConfiguration, pluginStatus.currentlyProcessingConfiguration)) - { - if (force_) - { - OrthancPlugins::LogWarning("DbOptimizer: forcing execution -> will reconstruct DB"); - } - else - { - OrthancPlugins::LogWarning("DbOptimizer: the DB configuration has changed since last run, will reprocess the whole DB !"); - } - - Json::Value changes; - OrthancPlugins::RestApiGet(changes, "/changes?last", false); - - pluginStatus.lastProcessedChange = 0; - pluginStatus.lastChangeToProcess = changes["Last"].asInt64(); // the last change is the last change at the time we start. We assume that every new ingested file will be constructed correctly - } - else - { - OrthancPlugins::LogWarning("DbOptimizer: the DB configuration has not changed since last run, will continue processing changes"); - } - - bool completed = pluginStatus.lastChangeToProcess == 0; // if the DB is empty at start, no need to process anyting - while (!workerThreadShouldStop && !completed) - { - completed = ProcessChanges(pluginStatus, currentDbConfiguration); - SaveStatusInDb(pluginStatus); - - if (!completed) - { - OrthancPlugins::LogInfo("DbOptimizer: processed changes " + - boost::lexical_cast<std::string>(pluginStatus.lastProcessedChange) + - " / " + boost::lexical_cast<std::string>(pluginStatus.lastChangeToProcess)); - - boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelay_*100)); // wait 1/10 of the delay between changes - } - } - - if (completed) - { - pluginStatus.lastProcessedConfiguration = currentDbConfiguration; - pluginStatus.currentlyProcessingConfiguration.Clear(); - - pluginStatus.lastProcessedChange = -1; - pluginStatus.lastChangeToProcess = -1; - - SaveStatusInDb(pluginStatus); - - OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "DbOptimizer: finished processing all changes"); - } -} - -extern "C" -{ - OrthancPluginErrorCode OnChangeCallback(OrthancPluginChangeType changeType, - OrthancPluginResourceType resourceType, - const char* resourceId) - { - switch (changeType) - { - case OrthancPluginChangeType_OrthancStarted: - { - OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "Starting DB Optmizer worker thread"); - workerThread_.reset(new boost::thread(WorkerThread)); - return OrthancPluginErrorCode_Success; - } - case OrthancPluginChangeType_OrthancStopped: - { - if (workerThread_ && workerThread_->joinable()) - { - workerThreadShouldStop = true; - workerThread_->join(); - } - } - default: - return OrthancPluginErrorCode_Success; - } - } - - ORTHANC_PLUGINS_API int32_t OrthancPluginInitialize(OrthancPluginContext* c) - { - OrthancPlugins::SetGlobalContext(c); - - /* Check the version of the Orthanc core */ - if (OrthancPluginCheckVersion(c) == 0) - { - OrthancPlugins::ReportMinimalOrthancVersion(ORTHANC_PLUGINS_MINIMAL_MAJOR_NUMBER, - ORTHANC_PLUGINS_MINIMAL_MINOR_NUMBER, - ORTHANC_PLUGINS_MINIMAL_REVISION_NUMBER); - return -1; - } - - OrthancPlugins::LogWarning("DB Optimizer plugin is initializing"); - OrthancPluginSetDescription(c, "Optimizes your DB and storage."); - - OrthancPlugins::OrthancConfiguration configuration; - - OrthancPlugins::OrthancConfiguration dbOptimizer; - configuration.GetSection(dbOptimizer, "DbOptimizer"); - - bool enabled = dbOptimizer.GetBooleanValue("Enable", false); - if (enabled) - { - globalPropertyId_ = dbOptimizer.GetIntegerValue("GlobalPropertyId", 1025); - force_ = dbOptimizer.GetBooleanValue("Force", false); - throttleDelay_ = dbOptimizer.GetUnsignedIntegerValue("ThrottleDelay", 0); - OrthancPluginRegisterOnChangeCallback(c, OnChangeCallback); - } - else - { - OrthancPlugins::LogWarning("DB Optimizer plugin is disabled by the configuration file"); - } - - return 0; - } - - - ORTHANC_PLUGINS_API void OrthancPluginFinalize() - { - OrthancPlugins::LogWarning("DB Optimizer plugin is finalizing"); - } - - - ORTHANC_PLUGINS_API const char* OrthancPluginGetName() - { - return "db-optimizer"; - } - - - ORTHANC_PLUGINS_API const char* OrthancPluginGetVersion() - { - return DB_OPTIMIZER_VERSION; - } -}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancServer/Plugins/Samples/Housekeeper/Plugin.cpp Mon Apr 25 15:30:51 2022 +0200 @@ -0,0 +1,679 @@ +/** + * Orthanc - A Lightweight, RESTful DICOM Store + * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics + * Department, University Hospital of Liege, Belgium + * Copyright (C) 2017-2022 Osimis S.A., Belgium + * Copyright (C) 2021-2022 Sebastien Jodogne, ICTEAM UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +#include "../../../../OrthancFramework/Sources/Compatibility.h" +#include "../Common/OrthancPluginCppWrapper.h" + +#include <boost/thread.hpp> +#include <boost/algorithm/string.hpp> +#include <json/value.h> +#include <json/writer.h> +#include <string.h> +#include <iostream> +#include <algorithm> +#include <map> +#include <list> +#include <time.h> + +static int globalPropertyId_ = 0; +static bool force_ = false; +static uint throttleDelay_ = 0; +static std::unique_ptr<boost::thread> workerThread_; +static bool workerThreadShouldStop_ = false; +static bool triggerOnStorageCompressionChange_ = true; +static bool triggerOnMainDicomTagsChange_ = true; +static bool triggerOnUnnecessaryDicomAsJsonFiles_ = true; + + +struct RunningPeriod +{ + int fromHour_; + int toHour_; + int weekday_; + + RunningPeriod(const std::string& weekday, const std::string& period) + { + if (weekday == "Monday") + { + weekday_ = 1; + } + else if (weekday == "Tuesday") + { + weekday_ = 2; + } + else if (weekday == "Wednesday") + { + weekday_ = 3; + } + else if (weekday == "Thursday") + { + weekday_ = 4; + } + else if (weekday == "Friday") + { + weekday_ = 5; + } + else if (weekday == "Saturday") + { + weekday_ = 6; + } + else if (weekday == "Sunday") + { + weekday_ = 0; + } + else + { + OrthancPlugins::LogWarning("Housekeeper: invalid schedule: unknown 'day': " + weekday); + ORTHANC_PLUGINS_THROW_EXCEPTION(BadFileFormat); + } + + std::vector<std::string> hours; + boost::split(hours, period, boost::is_any_of("-")); + + fromHour_ = boost::lexical_cast<int>(hours[0]); + toHour_ = boost::lexical_cast<int>(hours[1]); + } + + bool isInPeriod() const + { + time_t now = time(NULL); + tm* nowLocalTime = localtime(&now); + + if (nowLocalTime->tm_wday != weekday_) + { + return false; + } + + if (nowLocalTime->tm_hour >= fromHour_ && nowLocalTime->tm_hour < toHour_) + { + return true; + } + + return false; + } +}; + +struct RunningPeriods +{ + std::list<RunningPeriod> runningPeriods_; + + void load(const Json::Value& scheduleConfiguration) + { +// "Monday": ["0-6", "20-24"], + + Json::Value::Members names = scheduleConfiguration.getMemberNames(); + + for (Json::Value::Members::const_iterator it = names.begin(); + it != names.end(); it++) + { + for (Json::Value::ArrayIndex i = 0; i < scheduleConfiguration[*it].size(); i++) + { + runningPeriods_.push_back(RunningPeriod(*it, scheduleConfiguration[*it][i].asString())); + } + } + } + + bool isInPeriod() + { + if (runningPeriods_.size() == 0) + { + return true; // if no config: always run + } + + for (std::list<RunningPeriod>::const_iterator it = runningPeriods_.begin(); + it != runningPeriods_.end(); it++) + { + if (it->isInPeriod()) + { + return true; + } + } + return false; + } +}; + +RunningPeriods runningPeriods_; + +struct DbConfiguration +{ + std::string orthancVersion; + std::map<OrthancPluginResourceType, std::string> mainDicomTagsSignature; + bool storageCompressionEnabled; + + DbConfiguration() + : storageCompressionEnabled(false) + { + } + + bool IsDefined() const + { + return !orthancVersion.empty() && mainDicomTagsSignature.size() == 4; + } + + void Clear() + { + orthancVersion.clear(); + mainDicomTagsSignature.clear(); + } + + void ToJson(Json::Value& target) + { + if (!IsDefined()) + { + target = Json::nullValue; + } + else + { + Json::Value signatures; + + target = Json::objectValue; + + // default main dicom tags signature are the one from Orthanc 1.4.2 (last time the list was changed): + signatures["Patient"] = mainDicomTagsSignature[OrthancPluginResourceType_Patient]; + signatures["Study"] = mainDicomTagsSignature[OrthancPluginResourceType_Study]; + signatures["Series"] = mainDicomTagsSignature[OrthancPluginResourceType_Series]; + signatures["Instance"] = mainDicomTagsSignature[OrthancPluginResourceType_Instance]; + + target["MainDicomTagsSignature"] = signatures; + target["OrthancVersion"] = orthancVersion; + target["StorageCompressionEnabled"] = storageCompressionEnabled; + } + } + + void FromJson(Json::Value& source) + { + if (!source.isNull()) + { + orthancVersion = source["OrthancVersion"].asString(); + + const Json::Value& signatures = source["MainDicomTagsSignature"]; + mainDicomTagsSignature[OrthancPluginResourceType_Patient] = signatures["Patient"].asString(); + mainDicomTagsSignature[OrthancPluginResourceType_Study] = signatures["Study"].asString(); + mainDicomTagsSignature[OrthancPluginResourceType_Series] = signatures["Series"].asString(); + mainDicomTagsSignature[OrthancPluginResourceType_Instance] = signatures["Instance"].asString(); + + storageCompressionEnabled = source["StorageCompressionEnabled"].asBool(); + } + } +}; + +struct PluginStatus +{ + int statusVersion; + int64_t lastProcessedChange; + int64_t lastChangeToProcess; + + DbConfiguration currentlyProcessingConfiguration; // last configuration being processed (has not reached last change yet) + DbConfiguration lastProcessedConfiguration; // last configuration that has been fully processed (till last change) + + PluginStatus() + : statusVersion(1), + lastProcessedChange(-1), + lastChangeToProcess(-1) + { + } + + void ToJson(Json::Value& target) + { + target = Json::objectValue; + + target["Version"] = statusVersion; + target["LastProcessedChange"] = Json::Value::Int64(lastProcessedChange); + target["LastChangeToProcess"] = Json::Value::Int64(lastChangeToProcess); + + currentlyProcessingConfiguration.ToJson(target["CurrentlyProcessingConfiguration"]); + lastProcessedConfiguration.ToJson(target["LastProcessedConfiguration"]); + } + + void FromJson(Json::Value& source) + { + statusVersion = source["Version"].asInt(); + lastProcessedChange = source["LastProcessedChange"].asInt64(); + lastChangeToProcess = source["LastChangeToProcess"].asInt64(); + + Json::Value& current = source["CurrentlyProcessingConfiguration"]; + Json::Value& last = source["LastProcessedConfiguration"]; + + currentlyProcessingConfiguration.FromJson(current); + lastProcessedConfiguration.FromJson(last); + } +}; + + +static void ReadStatusFromDb(PluginStatus& pluginStatus) +{ + OrthancPlugins::OrthancString globalPropertyContent; + + globalPropertyContent.Assign(OrthancPluginGetGlobalProperty(OrthancPlugins::GetGlobalContext(), + globalPropertyId_, + "")); + + if (!globalPropertyContent.IsNullOrEmpty()) + { + Json::Value jsonStatus; + globalPropertyContent.ToJson(jsonStatus); + pluginStatus.FromJson(jsonStatus); + } + else + { + // default config + pluginStatus.statusVersion = 1; + pluginStatus.lastProcessedChange = -1; + pluginStatus.lastChangeToProcess = -1; + + pluginStatus.currentlyProcessingConfiguration.orthancVersion = "1.9.0"; // when we don't know, we assume some files were stored with Orthanc 1.9.0 (last version saving the dicom-as-json files) + + // default main dicom tags signature are the one from Orthanc 1.4.2 (last time the list was changed): + pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Patient] = "0010,0010;0010,0020;0010,0030;0010,0040;0010,1000"; + pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Study] = "0008,0020;0008,0030;0008,0050;0008,0080;0008,0090;0008,1030;0020,000d;0020,0010;0032,1032;0032,1060"; + pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Series] = "0008,0021;0008,0031;0008,0060;0008,0070;0008,1010;0008,103e;0008,1070;0018,0010;0018,0015;0018,0024;0018,1030;0018,1090;0018,1400;0020,000e;0020,0011;0020,0037;0020,0105;0020,1002;0040,0254;0054,0081;0054,0101;0054,1000"; + pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Instance] = "0008,0012;0008,0013;0008,0018;0020,0012;0020,0013;0020,0032;0020,0037;0020,0100;0020,4000;0028,0008;0054,1330"; + } +} + +static void SaveStatusInDb(PluginStatus& pluginStatus) +{ + Json::Value jsonStatus; + pluginStatus.ToJson(jsonStatus); + + Json::StreamWriterBuilder builder; + builder.settings_["indentation"] = " "; + std::string serializedStatus = Json::writeString(builder, jsonStatus); + + OrthancPluginSetGlobalProperty(OrthancPlugins::GetGlobalContext(), + globalPropertyId_, + serializedStatus.c_str()); +} + +static void GetCurrentDbConfiguration(DbConfiguration& configuration) +{ + Json::Value signatures; + Json::Value systemInfo; + + OrthancPlugins::RestApiGet(systemInfo, "/system", false); + configuration.mainDicomTagsSignature[OrthancPluginResourceType_Patient] = systemInfo["MainDicomTags"]["Patient"].asString(); + configuration.mainDicomTagsSignature[OrthancPluginResourceType_Study] = systemInfo["MainDicomTags"]["Study"].asString(); + configuration.mainDicomTagsSignature[OrthancPluginResourceType_Series] = systemInfo["MainDicomTags"]["Series"].asString(); + configuration.mainDicomTagsSignature[OrthancPluginResourceType_Instance] = systemInfo["MainDicomTags"]["Instance"].asString(); + configuration.storageCompressionEnabled = systemInfo["StorageCompression"].asBool(); + + configuration.orthancVersion = OrthancPlugins::GetGlobalContext()->orthancVersion; +} + +static bool NeedsProcessing(const DbConfiguration& current, const DbConfiguration& last) +{ + if (!last.IsDefined()) + { + return true; + } + + const char* lastVersion = last.orthancVersion.c_str(); + const std::map<OrthancPluginResourceType, std::string>& lastTags = last.mainDicomTagsSignature; + const std::map<OrthancPluginResourceType, std::string>& currentTags = current.mainDicomTagsSignature; + bool needsProcessing = false; + + if (!OrthancPlugins::CheckMinimalVersion(lastVersion, 1, 9, 1)) + { + if (triggerOnUnnecessaryDicomAsJsonFiles_) + { + OrthancPlugins::LogWarning("Housekeeper: your storage might still contain some dicom-as-json files -> will perform housekeeping"); + needsProcessing = true; + } + else + { + OrthancPlugins::LogWarning("Housekeeper: your storage might still contain some dicom-as-json files but the trigger has been disabled"); + } + } + + if (lastTags.at(OrthancPluginResourceType_Patient) != currentTags.at(OrthancPluginResourceType_Patient)) + { + if (triggerOnMainDicomTagsChange_) + { + OrthancPlugins::LogWarning("Housekeeper: Patient main dicom tags have changed, -> will perform housekeeping"); + needsProcessing = true; + } + else + { + OrthancPlugins::LogWarning("Housekeeper: Patient main dicom tags have changed but the trigger is disabled"); + } + } + + if (lastTags.at(OrthancPluginResourceType_Study) != currentTags.at(OrthancPluginResourceType_Study)) + { + if (triggerOnMainDicomTagsChange_) + { + OrthancPlugins::LogWarning("Housekeeper: Study main dicom tags have changed, -> will perform housekeeping"); + needsProcessing = true; + } + else + { + OrthancPlugins::LogWarning("Housekeeper: Study main dicom tags have changed but the trigger is disabled"); + } + } + + if (lastTags.at(OrthancPluginResourceType_Series) != currentTags.at(OrthancPluginResourceType_Series)) + { + if (triggerOnMainDicomTagsChange_) + { + OrthancPlugins::LogWarning("Housekeeper: Series main dicom tags have changed, -> will perform housekeeping"); + needsProcessing = true; + } + else + { + OrthancPlugins::LogWarning("Housekeeper: Series main dicom tags have changed but the trigger is disabled"); + } + } + + if (lastTags.at(OrthancPluginResourceType_Instance) != currentTags.at(OrthancPluginResourceType_Instance)) + { + if (triggerOnMainDicomTagsChange_) + { + OrthancPlugins::LogWarning("Housekeeper: Instance main dicom tags have changed, -> will perform housekeeping"); + needsProcessing = true; + } + else + { + OrthancPlugins::LogWarning("Housekeeper: Instance main dicom tags have changed but the trigger is disabled"); + } + } + + if (current.storageCompressionEnabled != last.storageCompressionEnabled) + { + if (triggerOnStorageCompressionChange_) + { + if (current.storageCompressionEnabled) + { + OrthancPlugins::LogWarning("Housekeeper: storage compression is now enabled -> will perform housekeeping"); + } + else + { + OrthancPlugins::LogWarning("Housekeeper: storage compression is now disabled -> will perform housekeeping"); + } + + needsProcessing = true; + } + else + { + OrthancPlugins::LogWarning("Housekeeper: storage compression has changed but the trigger is disabled"); + } + } + + return needsProcessing; +} + +static bool ProcessChanges(PluginStatus& pluginStatus, const DbConfiguration& currentDbConfiguration) +{ + Json::Value changes; + + pluginStatus.currentlyProcessingConfiguration = currentDbConfiguration; + + OrthancPlugins::RestApiGet(changes, "/changes?since=" + boost::lexical_cast<std::string>(pluginStatus.lastProcessedChange) + "&limit=100", false); + + for (Json::ArrayIndex i = 0; i < changes["Changes"].size(); i++) + { + const Json::Value& change = changes["Changes"][i]; + int64_t seq = change["Seq"].asInt64(); + + if (change["ChangeType"] == "NewStudy") // some StableStudy might be missing if orthanc was shutdown during a StableAge -> consider only the NewStudy events that can not be missed + { + Json::Value result; + OrthancPlugins::RestApiPost(result, "/studies/" + change["ID"].asString() + "/reconstruct", std::string(""), false); + boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelay_ * 1000)); + } + + if (seq >= pluginStatus.lastChangeToProcess) // we are done ! + { + return true; + } + + pluginStatus.lastProcessedChange = seq; + } + + return false; +} + + +static void WorkerThread() +{ + PluginStatus pluginStatus; + DbConfiguration currentDbConfiguration; + + OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "Starting Housekeeper worker thread"); + + ReadStatusFromDb(pluginStatus); + GetCurrentDbConfiguration(currentDbConfiguration); + + if (!NeedsProcessing(currentDbConfiguration, pluginStatus.lastProcessedConfiguration)) + { + OrthancPlugins::LogWarning("Housekeeper: everything has been processed already !"); + return; + } + + if (force_ || NeedsProcessing(currentDbConfiguration, pluginStatus.currentlyProcessingConfiguration)) + { + if (force_) + { + OrthancPlugins::LogWarning("Housekeeper: forcing execution -> will perform housekeeping"); + } + else + { + OrthancPlugins::LogWarning("Housekeeper: the DB configuration has changed since last run, will reprocess the whole DB !"); + } + + Json::Value changes; + OrthancPlugins::RestApiGet(changes, "/changes?last", false); + + pluginStatus.lastProcessedChange = 0; + pluginStatus.lastChangeToProcess = changes["Last"].asInt64(); // the last change is the last change at the time we start. We assume that every new ingested file will be constructed correctly + } + else + { + OrthancPlugins::LogWarning("Housekeeper: the DB configuration has not changed since last run, will continue processing changes"); + } + + bool completed = pluginStatus.lastChangeToProcess == 0; // if the DB is empty at start, no need to process anyting + bool loggedNotRightPeriodChangeMessage = false; + + while (!workerThreadShouldStop_ && !completed) + { + if (runningPeriods_.isInPeriod()) + { + completed = ProcessChanges(pluginStatus, currentDbConfiguration); + SaveStatusInDb(pluginStatus); + + if (!completed) + { + OrthancPlugins::LogInfo("Housekeeper: processed changes " + + boost::lexical_cast<std::string>(pluginStatus.lastProcessedChange) + + " / " + boost::lexical_cast<std::string>(pluginStatus.lastChangeToProcess)); + + boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelay_ * 100)); // wait 1/10 of the delay between changes + } + + loggedNotRightPeriodChangeMessage = false; + } + else + { + if (!loggedNotRightPeriodChangeMessage) + { + OrthancPlugins::LogInfo("Housekeeper: entering quiet period"); + loggedNotRightPeriodChangeMessage = true; + } + } + } + + if (completed) + { + pluginStatus.lastProcessedConfiguration = currentDbConfiguration; + pluginStatus.currentlyProcessingConfiguration.Clear(); + + pluginStatus.lastProcessedChange = -1; + pluginStatus.lastChangeToProcess = -1; + + SaveStatusInDb(pluginStatus); + + OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "Housekeeper: finished processing all changes"); + } +} + +extern "C" +{ + OrthancPluginErrorCode OnChangeCallback(OrthancPluginChangeType changeType, + OrthancPluginResourceType resourceType, + const char* resourceId) + { + switch (changeType) + { + case OrthancPluginChangeType_OrthancStarted: + { + workerThread_.reset(new boost::thread(WorkerThread)); + return OrthancPluginErrorCode_Success; + } + case OrthancPluginChangeType_OrthancStopped: + { + if (workerThread_ && workerThread_->joinable()) + { + workerThreadShouldStop_ = true; + workerThread_->join(); + } + } + default: + return OrthancPluginErrorCode_Success; + } + } + + ORTHANC_PLUGINS_API int32_t OrthancPluginInitialize(OrthancPluginContext* c) + { + OrthancPlugins::SetGlobalContext(c); + + /* Check the version of the Orthanc core */ + if (OrthancPluginCheckVersion(c) == 0) + { + OrthancPlugins::ReportMinimalOrthancVersion(ORTHANC_PLUGINS_MINIMAL_MAJOR_NUMBER, + ORTHANC_PLUGINS_MINIMAL_MINOR_NUMBER, + ORTHANC_PLUGINS_MINIMAL_REVISION_NUMBER); + return -1; + } + + OrthancPlugins::LogWarning("Housekeeper plugin is initializing"); + OrthancPluginSetDescription(c, "Optimizes your DB and storage."); + + OrthancPlugins::OrthancConfiguration configuration; + + OrthancPlugins::OrthancConfiguration housekeeper; + configuration.GetSection(housekeeper, "Housekeeper"); + + bool enabled = housekeeper.GetBooleanValue("Enable", false); + if (enabled) + { + /* + { + "Housekeeper": { + + // Enables/disables the plugin + "Enable": false, + + // the Global Prooperty ID in which the plugin progress + // is stored. Must be > 1024 and must not be used by + // another plugin + "GlobalPropertyId": 1025, + + // Forces execution even if the plugin did not detect + // any changes in configuration + "Force": false, + + // Delay (in seconds) between reconstruction of 2 studies + // This avoids overloading Orthanc with the housekeeping + // process and leaves room for other operations. + "ThrottleDelay": 5, + + // Runs the plugin only at certain period of time. + // If not specified, the plugin runs all the time + // Examples: + // to run between 0AM and 6AM everyday + every night + // from 8PM to 12PM and 24h a day on the weekend: + // "Schedule": { + // "Monday": ["0-6", "20-24"], + // "Tuesday": ["0-6", "20-24"], + // "Wednesday": ["0-6", "20-24"], + // "Thursday": ["0-6", "20-24"], + // "Friday": ["0-6", "20-24"], + // "Saturday": ["0-24"], + // "Sunday": ["0-24"] + // }, + + // configure events that can trigger a housekeeping processing + "Triggers" : { + "StorageCompressionChange": true, + "MainDicomTagsChange": true, + "UnnecessaryDicomAsJsonFiles": true + } + + } + } + */ + + + globalPropertyId_ = housekeeper.GetIntegerValue("GlobalPropertyId", 1025); + force_ = housekeeper.GetBooleanValue("Force", false); + throttleDelay_ = housekeeper.GetUnsignedIntegerValue("ThrottleDelay", 5); + + if (housekeeper.GetJson().isMember("Triggers")) + { + triggerOnStorageCompressionChange_ = housekeeper.GetBooleanValue("StorageCompressionChange", true); + triggerOnMainDicomTagsChange_ = housekeeper.GetBooleanValue("MainDicomTagsChange", true); + triggerOnUnnecessaryDicomAsJsonFiles_ = housekeeper.GetBooleanValue("UnnecessaryDicomAsJsonFiles", true); + } + + if (housekeeper.GetJson().isMember("Schedule")) + { + runningPeriods_.load(housekeeper.GetJson()["Schedule"]); + } + + OrthancPluginRegisterOnChangeCallback(c, OnChangeCallback); + } + else + { + OrthancPlugins::LogWarning("Housekeeper plugin is disabled by the configuration file"); + } + + return 0; + } + + + ORTHANC_PLUGINS_API void OrthancPluginFinalize() + { + OrthancPlugins::LogWarning("Housekeeper plugin is finalizing"); + } + + + ORTHANC_PLUGINS_API const char* OrthancPluginGetName() + { + return "housekeeper"; + } + + + ORTHANC_PLUGINS_API const char* OrthancPluginGetVersion() + { + return DB_OPTIMIZER_VERSION; + } +}
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp Wed Apr 20 14:36:47 2022 +0200 +++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp Mon Apr 25 15:30:51 2022 +0200 @@ -73,6 +73,7 @@ static const char* const STORAGE_AREA_PLUGIN = "StorageAreaPlugin"; static const char* const VERSION = "Version"; static const char* const MAIN_DICOM_TAGS = "MainDicomTags"; + static const char* const STORAGE_COMPRESSION = "StorageCompression"; if (call.IsDocumentation()) { @@ -101,6 +102,8 @@ "Whether Orthanc handle revisions of metadata and attachments to deal with multiple writers (new in Orthanc 1.9.2)") .SetAnswerField(MAIN_DICOM_TAGS, RestApiCallDocumentation::Type_JsonObject, "The list of MainDicomTags saved in DB for each resource level (new in Orthanc 1.11.0)") + .SetAnswerField(STORAGE_COMPRESSION, RestApiCallDocumentation::Type_Boolean, + "Whether storage compression is enabled (new in Orthanc 1.11.0)") .SetHttpGetSample("https://demo.orthanc-server.com/system", true); return; } @@ -121,6 +124,7 @@ result[HTTP_PORT] = lock.GetConfiguration().GetUnsignedIntegerParameter(HTTP_PORT, 8042); result[NAME] = lock.GetConfiguration().GetStringParameter(NAME, ""); result[CHECK_REVISIONS] = lock.GetConfiguration().GetBooleanParameter(CHECK_REVISIONS, false); // New in Orthanc 1.9.2 + result[STORAGE_COMPRESSION] = lock.GetConfiguration().GetBooleanParameter(STORAGE_COMPRESSION, false); // New in Orthanc 1.11.0 } result[STORAGE_AREA_PLUGIN] = Json::nullValue;