Mercurial > hg > orthanc
changeset 4988:8fba26292a9f
Housekeeper plugin: finalizing + integration tests ok
author | Alain Mazy <am@osimis.io> |
---|---|
date | Sat, 30 Apr 2022 19:39:40 +0200 |
parents | a25e74fad379 |
children | 24ef02dc7a7a |
files | NEWS OrthancServer/Plugins/Samples/Housekeeper/Plugin.cpp OrthancServer/Sources/Database/StatelessDatabaseOperations.cpp OrthancServer/Sources/Database/StatelessDatabaseOperations.h OrthancServer/Sources/DicomInstanceToStore.cpp OrthancServer/Sources/DicomInstanceToStore.h OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp OrthancServer/Sources/ServerContext.cpp OrthancServer/Sources/ServerContext.h OrthancServer/Sources/ServerIndex.cpp OrthancServer/Sources/ServerIndex.h OrthancServer/Sources/ServerToolbox.cpp OrthancServer/Sources/ServerToolbox.h |
diffstat | 14 files changed, 411 insertions(+), 168 deletions(-) [+] |
line wrap: on
line diff
--- a/NEWS Tue Apr 26 16:14:49 2022 +0200 +++ b/NEWS Sat Apr 30 19:39:40 2022 +0200 @@ -12,7 +12,7 @@ the storage might still contain dicom-as-json files that are not needed anymore -> it will remove them - if "ExtraMainDicomTags" has changed. - - if "StorageCompression" has chagned. + - if "StorageCompression" or "IngestTranscoding" has chagned. * New configuration "Warnings" to enable/disable individual warnings that can be identified by a W0XX prefix in the logs. These warnings have been added: @@ -38,9 +38,20 @@ - /studies, /studies/../series, /studies/../instances - /series, /series/../instances - /instances -* new field "MainDicomTags" in the /system route response to list the tags that - are saved in DB -* new field "StorageCompression" reported in the /system route response +* /reconstruct routes: + - new options "ReconstructFiles" (false by default to keep backward compatibility) to + potentialy compress/uncompress the files or transcode them if "StorageCompression" + or "IngestTranscoding" has changed since the file has been ingested. + POSSIBLE BREAKING-CHANGES: + - the /reconstruct routes now preserve all metadata + - the /reconstruct routes now skip the IncomingInstanceFilter + - the /reconstruct routes won't generate new events like NewStudy, StableStudy, ... + therefore, the corresponding callbacks won't be called anymore + - the /reconstruct routes won't affect the patient recycling anymore +* new fields reported in the /system route: + - "MainDicomTags" to list the tags that are saved in DB + - "StorageCompression", "OverwriteInstances", "IngestTranscoding" reported from the + configuration file Version 1.10.1 (2022-03-23)
--- a/OrthancServer/Plugins/Samples/Housekeeper/Plugin.cpp Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Plugins/Samples/Housekeeper/Plugin.cpp Sat Apr 30 19:39:40 2022 +0200 @@ -25,6 +25,7 @@ #include <boost/thread.hpp> #include <boost/algorithm/string.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> #include <json/value.h> #include <json/writer.h> #include <string.h> @@ -42,6 +43,7 @@ static bool triggerOnStorageCompressionChange_ = true; static bool triggerOnMainDicomTagsChange_ = true; static bool triggerOnUnnecessaryDicomAsJsonFiles_ = true; +static bool triggerOnIngestTranscodingChange_ = true; struct RunningPeriod @@ -112,6 +114,7 @@ } }; + struct RunningPeriods { std::list<RunningPeriod> runningPeriods_; @@ -153,6 +156,7 @@ RunningPeriods runningPeriods_; + struct DbConfiguration { std::string orthancVersion; @@ -160,6 +164,7 @@ std::string studiesMainDicomTagsSignature; std::string seriesMainDicomTagsSignature; std::string instancesMainDicomTagsSignature; + std::string ingestTranscoding; bool storageCompressionEnabled; DbConfiguration() @@ -179,6 +184,7 @@ studiesMainDicomTagsSignature.clear(); seriesMainDicomTagsSignature.clear(); instancesMainDicomTagsSignature.clear(); + ingestTranscoding.clear(); } void ToJson(Json::Value& target) @@ -202,6 +208,7 @@ target["MainDicomTagsSignature"] = signatures; target["OrthancVersion"] = orthancVersion; target["StorageCompressionEnabled"] = storageCompressionEnabled; + target["IngestTranscoding"] = ingestTranscoding; } } @@ -218,15 +225,18 @@ instancesMainDicomTagsSignature = signatures["Instance"].asString(); storageCompressionEnabled = source["StorageCompressionEnabled"].asBool(); + ingestTranscoding = source["IngestTranscoding"].asString(); } } }; + struct PluginStatus { int statusVersion; int64_t lastProcessedChange; int64_t lastChangeToProcess; + boost::posix_time::ptime lastTimeStarted; DbConfiguration currentlyProcessingConfiguration; // last configuration being processed (has not reached last change yet) DbConfiguration lastProcessedConfiguration; // last configuration that has been fully processed (till last change) @@ -234,7 +244,8 @@ PluginStatus() : statusVersion(1), lastProcessedChange(-1), - lastChangeToProcess(-1) + lastChangeToProcess(-1), + lastTimeStarted(boost::date_time::special_values::not_a_date_time) { } @@ -245,6 +256,15 @@ target["Version"] = statusVersion; target["LastProcessedChange"] = Json::Value::Int64(lastProcessedChange); target["LastChangeToProcess"] = Json::Value::Int64(lastChangeToProcess); + + if (lastTimeStarted == boost::posix_time::special_values::not_a_date_time) + { + target["LastTimeStarted"] = Json::Value::null; + } + else + { + target["LastTimeStarted"] = boost::posix_time::to_iso_string(lastTimeStarted); + } currentlyProcessingConfiguration.ToJson(target["CurrentlyProcessingConfiguration"]); lastProcessedConfiguration.ToJson(target["LastProcessedConfiguration"]); @@ -255,6 +275,14 @@ statusVersion = source["Version"].asInt(); lastProcessedChange = source["LastProcessedChange"].asInt64(); lastChangeToProcess = source["LastChangeToProcess"].asInt64(); + if (source["LastTimeStarted"].isNull()) + { + lastTimeStarted = boost::posix_time::special_values::not_a_date_time; + } + else + { + lastTimeStarted = boost::posix_time::from_iso_string(source["LastTimeStarted"].asString()); + } Json::Value& current = source["CurrentlyProcessingConfiguration"]; Json::Value& last = source["LastProcessedConfiguration"]; @@ -264,9 +292,13 @@ } }; +static PluginStatus pluginStatus_; +static boost::recursive_mutex pluginStatusMutex_; -static void ReadStatusFromDb(PluginStatus& pluginStatus) +static void ReadStatusFromDb() { + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); + OrthancPlugins::OrthancString globalPropertyContent; globalPropertyContent.Assign(OrthancPluginGetGlobalProperty(OrthancPlugins::GetGlobalContext(), @@ -277,29 +309,32 @@ { Json::Value jsonStatus; globalPropertyContent.ToJson(jsonStatus); - pluginStatus.FromJson(jsonStatus); + pluginStatus_.FromJson(jsonStatus); } else { // default config - pluginStatus.statusVersion = 1; - pluginStatus.lastProcessedChange = -1; - pluginStatus.lastChangeToProcess = -1; + pluginStatus_.statusVersion = 1; + pluginStatus_.lastProcessedChange = -1; + pluginStatus_.lastChangeToProcess = -1; + pluginStatus_.lastTimeStarted = boost::date_time::special_values::not_a_date_time; - pluginStatus.currentlyProcessingConfiguration.orthancVersion = "1.9.0"; // when we don't know, we assume some files were stored with Orthanc 1.9.0 (last version saving the dicom-as-json files) + pluginStatus_.lastProcessedConfiguration.orthancVersion = "1.9.0"; // when we don't know, we assume some files were stored with Orthanc 1.9.0 (last version saving the dicom-as-json files) // default main dicom tags signature are the one from Orthanc 1.4.2 (last time the list was changed): - pluginStatus.currentlyProcessingConfiguration.patientsMainDicomTagsSignature = "0010,0010;0010,0020;0010,0030;0010,0040;0010,1000"; - pluginStatus.currentlyProcessingConfiguration.studiesMainDicomTagsSignature = "0008,0020;0008,0030;0008,0050;0008,0080;0008,0090;0008,1030;0020,000d;0020,0010;0032,1032;0032,1060"; - pluginStatus.currentlyProcessingConfiguration.seriesMainDicomTagsSignature = "0008,0021;0008,0031;0008,0060;0008,0070;0008,1010;0008,103e;0008,1070;0018,0010;0018,0015;0018,0024;0018,1030;0018,1090;0018,1400;0020,000e;0020,0011;0020,0037;0020,0105;0020,1002;0040,0254;0054,0081;0054,0101;0054,1000"; - pluginStatus.currentlyProcessingConfiguration.instancesMainDicomTagsSignature = "0008,0012;0008,0013;0008,0018;0020,0012;0020,0013;0020,0032;0020,0037;0020,0100;0020,4000;0028,0008;0054,1330"; + pluginStatus_.lastProcessedConfiguration.patientsMainDicomTagsSignature = "0010,0010;0010,0020;0010,0030;0010,0040;0010,1000"; + pluginStatus_.lastProcessedConfiguration.studiesMainDicomTagsSignature = "0008,0020;0008,0030;0008,0050;0008,0080;0008,0090;0008,1030;0020,000d;0020,0010;0032,1032;0032,1060"; + pluginStatus_.lastProcessedConfiguration.seriesMainDicomTagsSignature = "0008,0021;0008,0031;0008,0060;0008,0070;0008,1010;0008,103e;0008,1070;0018,0010;0018,0015;0018,0024;0018,1030;0018,1090;0018,1400;0020,000e;0020,0011;0020,0037;0020,0105;0020,1002;0040,0254;0054,0081;0054,0101;0054,1000"; + pluginStatus_.lastProcessedConfiguration.instancesMainDicomTagsSignature = "0008,0012;0008,0013;0008,0018;0020,0012;0020,0013;0020,0032;0020,0037;0020,0100;0020,4000;0028,0008;0054,1330"; } } -static void SaveStatusInDb(PluginStatus& pluginStatus) +static void SaveStatusInDb() { + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); + Json::Value jsonStatus; - pluginStatus.ToJson(jsonStatus); + pluginStatus_.ToJson(jsonStatus); Json::StreamWriterBuilder builder; builder.settings_["indentation"] = " "; @@ -321,26 +356,29 @@ configuration.seriesMainDicomTagsSignature = systemInfo["MainDicomTags"]["Series"].asString(); configuration.instancesMainDicomTagsSignature = systemInfo["MainDicomTags"]["Instance"].asString(); configuration.storageCompressionEnabled = systemInfo["StorageCompression"].asBool(); + configuration.ingestTranscoding = systemInfo["IngestTranscoding"].asString(); configuration.orthancVersion = OrthancPlugins::GetGlobalContext()->orthancVersion; } -static bool NeedsProcessing(const DbConfiguration& current, const DbConfiguration& last) +static void CheckNeedsProcessing(bool& needsReconstruct, bool& needsReingest, const DbConfiguration& current, const DbConfiguration& last) { + needsReconstruct = false; + needsReingest = false; + if (!last.IsDefined()) { - return true; + return; } const char* lastVersion = last.orthancVersion.c_str(); - bool needsProcessing = false; if (!OrthancPlugins::CheckMinimalVersion(lastVersion, 1, 9, 1)) { if (triggerOnUnnecessaryDicomAsJsonFiles_) { OrthancPlugins::LogWarning("Housekeeper: your storage might still contain some dicom-as-json files -> will perform housekeeping"); - needsProcessing = true; + needsReconstruct = true; // the default reconstruct removes the dicom-as-json } else { @@ -353,7 +391,7 @@ if (triggerOnMainDicomTagsChange_) { OrthancPlugins::LogWarning("Housekeeper: Patient main dicom tags have changed, -> will perform housekeeping"); - needsProcessing = true; + needsReconstruct = true; } else { @@ -366,7 +404,7 @@ if (triggerOnMainDicomTagsChange_) { OrthancPlugins::LogWarning("Housekeeper: Study main dicom tags have changed, -> will perform housekeeping"); - needsProcessing = true; + needsReconstruct = true; } else { @@ -379,7 +417,7 @@ if (triggerOnMainDicomTagsChange_) { OrthancPlugins::LogWarning("Housekeeper: Series main dicom tags have changed, -> will perform housekeeping"); - needsProcessing = true; + needsReconstruct = true; } else { @@ -392,7 +430,7 @@ if (triggerOnMainDicomTagsChange_) { OrthancPlugins::LogWarning("Housekeeper: Instance main dicom tags have changed, -> will perform housekeeping"); - needsProcessing = true; + needsReconstruct = true; } else { @@ -413,7 +451,7 @@ OrthancPlugins::LogWarning("Housekeeper: storage compression is now disabled -> will perform housekeeping"); } - needsProcessing = true; + needsReingest = true; } else { @@ -421,16 +459,33 @@ } } - return needsProcessing; + if (current.ingestTranscoding != last.ingestTranscoding) + { + if (triggerOnIngestTranscodingChange_) + { + OrthancPlugins::LogWarning("Housekeeper: ingest transcoding has changed -> will perform housekeeping"); + + needsReingest = true; + } + else + { + OrthancPlugins::LogWarning("Housekeeper: ingest transcoding has changed but the trigger is disabled"); + } + } + } -static bool ProcessChanges(PluginStatus& pluginStatus, const DbConfiguration& currentDbConfiguration) +static bool ProcessChanges(bool needsReconstruct, bool needsReingest, const DbConfiguration& currentDbConfiguration) { Json::Value changes; - pluginStatus.currentlyProcessingConfiguration = currentDbConfiguration; + { + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); - OrthancPlugins::RestApiGet(changes, "/changes?since=" + boost::lexical_cast<std::string>(pluginStatus.lastProcessedChange) + "&limit=100", false); + pluginStatus_.currentlyProcessingConfiguration = currentDbConfiguration; + + OrthancPlugins::RestApiGet(changes, "/changes?since=" + boost::lexical_cast<std::string>(pluginStatus_.lastProcessedChange) + "&limit=100", false); + } for (Json::ArrayIndex i = 0; i < changes["Changes"].size(); i++) { @@ -440,16 +495,29 @@ if (change["ChangeType"] == "NewStudy") // some StableStudy might be missing if orthanc was shutdown during a StableAge -> consider only the NewStudy events that can not be missed { Json::Value result; - OrthancPlugins::RestApiPost(result, "/studies/" + change["ID"].asString() + "/reconstruct", std::string(""), false); - boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelay_ * 1000)); + Json::Value request; + if (needsReingest) + { + request["ReconstructFiles"] = true; + } + OrthancPlugins::RestApiPost(result, "/studies/" + change["ID"].asString() + "/reconstruct", request, false); } - if (seq >= pluginStatus.lastChangeToProcess) // we are done ! { - return true; + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); + + pluginStatus_.lastProcessedChange = seq; + + if (seq >= pluginStatus_.lastChangeToProcess) // we are done ! + { + return true; + } } - pluginStatus.lastProcessedChange = seq; + if (change["ChangeType"] == "NewStudy") + { + boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelay_ * 1000)); + } } return false; @@ -458,21 +526,31 @@ static void WorkerThread() { - PluginStatus pluginStatus; DbConfiguration currentDbConfiguration; OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "Starting Housekeeper worker thread"); - ReadStatusFromDb(pluginStatus); + ReadStatusFromDb(); + GetCurrentDbConfiguration(currentDbConfiguration); - if (!NeedsProcessing(currentDbConfiguration, pluginStatus.lastProcessedConfiguration)) + bool needsReconstruct = false; + bool needsReingest = false; + + { + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); + CheckNeedsProcessing(needsReconstruct, needsReingest, currentDbConfiguration, pluginStatus_.lastProcessedConfiguration); + } + + bool needsProcessing = needsReconstruct || needsReingest; + + if (!needsProcessing) { OrthancPlugins::LogWarning("Housekeeper: everything has been processed already !"); return; } - if (force_ || NeedsProcessing(currentDbConfiguration, pluginStatus.currentlyProcessingConfiguration)) + if (force_ || needsProcessing) { if (force_) { @@ -486,29 +564,41 @@ Json::Value changes; OrthancPlugins::RestApiGet(changes, "/changes?last", false); - pluginStatus.lastProcessedChange = 0; - pluginStatus.lastChangeToProcess = changes["Last"].asInt64(); // the last change is the last change at the time we start. We assume that every new ingested file will be constructed correctly + { + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); + + pluginStatus_.lastProcessedChange = 0; + pluginStatus_.lastChangeToProcess = changes["Last"].asInt64(); // the last change is the last change at the time we start. We assume that every new ingested file will be constructed correctly + pluginStatus_.lastTimeStarted = boost::posix_time::microsec_clock::universal_time(); + } } else { OrthancPlugins::LogWarning("Housekeeper: the DB configuration has not changed since last run, will continue processing changes"); } - bool completed = pluginStatus.lastChangeToProcess == 0; // if the DB is empty at start, no need to process anyting + bool completed = false; + { + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); + completed = pluginStatus_.lastChangeToProcess == 0; // if the DB is empty at start, no need to process anyting + } + bool loggedNotRightPeriodChangeMessage = false; while (!workerThreadShouldStop_ && !completed) { if (runningPeriods_.isInPeriod()) { - completed = ProcessChanges(pluginStatus, currentDbConfiguration); - SaveStatusInDb(pluginStatus); + completed = ProcessChanges(needsReconstruct, needsReingest, currentDbConfiguration); + SaveStatusInDb(); if (!completed) { + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); + OrthancPlugins::LogInfo("Housekeeper: processed changes " + - boost::lexical_cast<std::string>(pluginStatus.lastProcessedChange) + - " / " + boost::lexical_cast<std::string>(pluginStatus.lastChangeToProcess)); + boost::lexical_cast<std::string>(pluginStatus_.lastProcessedChange) + + " / " + boost::lexical_cast<std::string>(pluginStatus_.lastChangeToProcess)); boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelay_ * 100)); // wait 1/10 of the delay between changes } @@ -527,13 +617,15 @@ if (completed) { - pluginStatus.lastProcessedConfiguration = currentDbConfiguration; - pluginStatus.currentlyProcessingConfiguration.Clear(); + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); + + pluginStatus_.lastProcessedConfiguration = currentDbConfiguration; + pluginStatus_.currentlyProcessingConfiguration.Clear(); - pluginStatus.lastProcessedChange = -1; - pluginStatus.lastChangeToProcess = -1; + pluginStatus_.lastProcessedChange = -1; + pluginStatus_.lastChangeToProcess = -1; - SaveStatusInDb(pluginStatus); + SaveStatusInDb(); OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "Housekeeper: finished processing all changes"); } @@ -541,6 +633,28 @@ extern "C" { + OrthancPluginErrorCode GetPluginStatus(OrthancPluginRestOutput* output, + const char* url, + const OrthancPluginHttpRequest* request) + { + if (request->method != OrthancPluginHttpMethod_Get) + { + OrthancPlugins::AnswerMethodNotAllowed(output, "GET"); + } + else + { + boost::recursive_mutex::scoped_lock lock(pluginStatusMutex_); + + Json::Value status; + pluginStatus_.ToJson(status); + + OrthancPlugins::AnswerJson(status, output); + } + + return OrthancPluginErrorCode_Success; + } + + OrthancPluginErrorCode OnChangeCallback(OrthancPluginChangeType changeType, OrthancPluginResourceType resourceType, const char* resourceId) @@ -581,10 +695,10 @@ OrthancPlugins::LogWarning("Housekeeper plugin is initializing"); OrthancPluginSetDescription(c, "Optimizes your DB and storage."); - OrthancPlugins::OrthancConfiguration configuration; + OrthancPlugins::OrthancConfiguration orthancConfiguration; OrthancPlugins::OrthancConfiguration housekeeper; - configuration.GetSection(housekeeper, "Housekeeper"); + orthancConfiguration.GetSection(housekeeper, "Housekeeper"); bool enabled = housekeeper.GetBooleanValue("Enable", false); if (enabled) @@ -644,8 +758,10 @@ if (housekeeper.GetJson().isMember("Triggers")) { triggerOnStorageCompressionChange_ = housekeeper.GetBooleanValue("StorageCompressionChange", true); + triggerOnMainDicomTagsChange_ = housekeeper.GetBooleanValue("MainDicomTagsChange", true); triggerOnUnnecessaryDicomAsJsonFiles_ = housekeeper.GetBooleanValue("UnnecessaryDicomAsJsonFiles", true); + triggerOnIngestTranscodingChange_ = housekeeper.GetBooleanValue("IngestTranscodingChange", true); } if (housekeeper.GetJson().isMember("Schedule")) @@ -654,6 +770,7 @@ } OrthancPluginRegisterOnChangeCallback(c, OnChangeCallback); + OrthancPluginRegisterRestCallback(c, "/housekeeper/status", GetPluginStatus); } else {
--- a/OrthancServer/Sources/Database/StatelessDatabaseOperations.cpp Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/Database/StatelessDatabaseOperations.cpp Sat Apr 30 19:39:40 2022 +0200 @@ -2850,7 +2850,8 @@ bool hasPixelDataOffset, uint64_t pixelDataOffset, uint64_t maximumStorageSize, - unsigned int maximumPatients) + unsigned int maximumPatients, + bool isReconstruct) { class Operations : public IReadWriteOperations { @@ -2868,6 +2869,7 @@ uint64_t pixelDataOffset_; uint64_t maximumStorageSize_; unsigned int maximumPatientCount_; + bool isReconstruct_; // Auto-computed fields bool hasExpectedInstances_; @@ -2955,7 +2957,8 @@ bool hasPixelDataOffset, uint64_t pixelDataOffset, uint64_t maximumStorageSize, - unsigned int maximumPatientCount) : + unsigned int maximumPatientCount, + bool isReconstruct) : storeStatus_(StoreStatus_Failure), instanceMetadata_(instanceMetadata), dicomSummary_(dicomSummary), @@ -2968,7 +2971,8 @@ hasPixelDataOffset_(hasPixelDataOffset), pixelDataOffset_(pixelDataOffset), maximumStorageSize_(maximumStorageSize), - maximumPatientCount_(maximumPatientCount) + maximumPatientCount_(maximumPatientCount), + isReconstruct_(isReconstruct) { hasExpectedInstances_ = ComputeExpectedNumberOfInstances(expectedInstances_, dicomSummary); @@ -3022,31 +3026,33 @@ } - // Warn about the creation of new resources. The order must be - // from instance to patient. - - // NB: In theory, could be sped up by grouping the underlying - // calls to "transaction.LogChange()". However, this would only have an - // impact when new patient/study/series get created, which - // occurs far less often that creating new instances. The - // positive impact looks marginal in practice. - transaction.LogChange(instanceId, ChangeType_NewInstance, ResourceType_Instance, hashInstance_); - - if (status.isNewSeries_) + if (!isReconstruct_) // don't signal new resources if this is a reconstruction { - transaction.LogChange(status.seriesId_, ChangeType_NewSeries, ResourceType_Series, hashSeries_); - } - - if (status.isNewStudy_) - { - transaction.LogChange(status.studyId_, ChangeType_NewStudy, ResourceType_Study, hashStudy_); - } - - if (status.isNewPatient_) - { - transaction.LogChange(status.patientId_, ChangeType_NewPatient, ResourceType_Patient, hashPatient_); - } - + // Warn about the creation of new resources. The order must be + // from instance to patient. + + // NB: In theory, could be sped up by grouping the underlying + // calls to "transaction.LogChange()". However, this would only have an + // impact when new patient/study/series get created, which + // occurs far less often that creating new instances. The + // positive impact looks marginal in practice. + transaction.LogChange(instanceId, ChangeType_NewInstance, ResourceType_Instance, hashInstance_); + + if (status.isNewSeries_) + { + transaction.LogChange(status.seriesId_, ChangeType_NewSeries, ResourceType_Series, hashSeries_); + } + + if (status.isNewStudy_) + { + transaction.LogChange(status.studyId_, ChangeType_NewStudy, ResourceType_Study, hashStudy_); + } + + if (status.isNewPatient_) + { + transaction.LogChange(status.patientId_, ChangeType_NewPatient, ResourceType_Patient, hashPatient_); + } + } // Ensure there is enough room in the storage for the new instance uint64_t instanceSize = 0; @@ -3056,9 +3062,11 @@ instanceSize += it->GetCompressedSize(); } - transaction.Recycle(maximumStorageSize_, maximumPatientCount_, - instanceSize, hashPatient_ /* don't consider the current patient for recycling */); - + if (!isReconstruct_) // reconstruction should not affect recycling + { + transaction.Recycle(maximumStorageSize_, maximumPatientCount_, + instanceSize, hashPatient_ /* don't consider the current patient for recycling */); + } // Attach the files to the newly created instance for (Attachments::const_iterator it = attachments_.begin(); @@ -3070,33 +3078,9 @@ { ResourcesContent content(true /* new resource, metadata can be set */); - - // Populate the tags of the newly-created resources - - content.AddResource(instanceId, ResourceType_Instance, dicomSummary_); - content.AddMetadata(instanceId, MetadataType_MainDicomTagsSignature, DicomMap::GetMainDicomTagsSignature(ResourceType_Instance)); // New in Orthanc 1.11.0 - - if (status.isNewSeries_) - { - content.AddResource(status.seriesId_, ResourceType_Series, dicomSummary_); - content.AddMetadata(status.seriesId_, MetadataType_MainDicomTagsSignature, DicomMap::GetMainDicomTagsSignature(ResourceType_Series)); // New in Orthanc 1.11.0 - } - - if (status.isNewStudy_) - { - content.AddResource(status.studyId_, ResourceType_Study, dicomSummary_); - content.AddMetadata(status.studyId_, MetadataType_MainDicomTagsSignature, DicomMap::GetMainDicomTagsSignature(ResourceType_Study)); // New in Orthanc 1.11.0 - } - - if (status.isNewPatient_) - { - content.AddResource(status.patientId_, ResourceType_Patient, dicomSummary_); - content.AddMetadata(status.patientId_, MetadataType_MainDicomTagsSignature, DicomMap::GetMainDicomTagsSignature(ResourceType_Patient)); // New in Orthanc 1.11.0 - } - - - // Attach the user-specified metadata - + + + // Attach the user-specified metadata (in case of reconstruction, metadata_ contains all past metadata, including the system ones we want to keep) for (MetadataMap::const_iterator it = metadata_.begin(); it != metadata_.end(); ++it) { @@ -3124,7 +3108,29 @@ } } - + // Populate the tags of the newly-created resources + + content.AddResource(instanceId, ResourceType_Instance, dicomSummary_); + SetInstanceMetadata(content, instanceMetadata_, instanceId, MetadataType_MainDicomTagsSignature, DicomMap::GetMainDicomTagsSignature(ResourceType_Instance)); // New in Orthanc 1.11.0 + + if (status.isNewSeries_) + { + content.AddResource(status.seriesId_, ResourceType_Series, dicomSummary_); + content.AddMetadata(status.seriesId_, MetadataType_MainDicomTagsSignature, DicomMap::GetMainDicomTagsSignature(ResourceType_Series)); // New in Orthanc 1.11.0 + } + + if (status.isNewStudy_) + { + content.AddResource(status.studyId_, ResourceType_Study, dicomSummary_); + content.AddMetadata(status.studyId_, MetadataType_MainDicomTagsSignature, DicomMap::GetMainDicomTagsSignature(ResourceType_Study)); // New in Orthanc 1.11.0 + } + + if (status.isNewPatient_) + { + content.AddResource(status.patientId_, ResourceType_Patient, dicomSummary_); + content.AddMetadata(status.patientId_, MetadataType_MainDicomTagsSignature, DicomMap::GetMainDicomTagsSignature(ResourceType_Patient)); // New in Orthanc 1.11.0 + } + // Attach the auto-computed metadata for the patient/study/series levels std::string now = SystemToolbox::GetNowIsoString(true /* use UTC time (not local time) */); content.AddMetadata(status.seriesId_, MetadataType_LastUpdate, now); @@ -3144,17 +3150,6 @@ origin_.GetRemoteAetC()); } - - // Attach the auto-computed metadata for the instance level, - // reflecting these additions into the input metadata map - SetInstanceMetadata(content, instanceMetadata_, instanceId, - MetadataType_Instance_ReceptionDate, now); - SetInstanceMetadata(content, instanceMetadata_, instanceId, MetadataType_RemoteAet, - origin_.GetRemoteAetC()); - SetInstanceMetadata(content, instanceMetadata_, instanceId, MetadataType_Instance_Origin, - EnumerationToString(origin_.GetRequestOrigin())); - - if (hasTransferSyntax_) { // New in Orthanc 1.2.0 @@ -3163,7 +3158,17 @@ GetTransferSyntaxUid(transferSyntax_)); } - { + if (!isReconstruct_) // don't change origin metadata + { + // Attach the auto-computed metadata for the instance level, + // reflecting these additions into the input metadata map + SetInstanceMetadata(content, instanceMetadata_, instanceId, + MetadataType_Instance_ReceptionDate, now); + SetInstanceMetadata(content, instanceMetadata_, instanceId, MetadataType_RemoteAet, + origin_.GetRemoteAetC()); + SetInstanceMetadata(content, instanceMetadata_, instanceId, MetadataType_Instance_Origin, + EnumerationToString(origin_.GetRequestOrigin())); + std::string s; if (origin_.LookupRemoteIp(s)) @@ -3270,7 +3275,7 @@ Operations operations(instanceMetadata, dicomSummary, attachments, metadata, origin, overwrite, hasTransferSyntax, transferSyntax, hasPixelDataOffset, - pixelDataOffset, maximumStorageSize, maximumPatients); + pixelDataOffset, maximumStorageSize, maximumPatients, isReconstruct); Apply(operations); return operations.GetStoreStatus(); }
--- a/OrthancServer/Sources/Database/StatelessDatabaseOperations.h Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/Database/StatelessDatabaseOperations.h Sat Apr 30 19:39:40 2022 +0200 @@ -659,7 +659,8 @@ bool hasPixelDataOffset, uint64_t pixelDataOffset, uint64_t maximumStorageSize, - unsigned int maximumPatients); + unsigned int maximumPatients, + bool isReconstruct); StoreStatus AddAttachment(int64_t& newRevision /*out*/, const FileInfo& attachment,
--- a/OrthancServer/Sources/DicomInstanceToStore.cpp Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/DicomInstanceToStore.cpp Sat Apr 30 19:39:40 2022 +0200 @@ -300,4 +300,14 @@ { return GetParsedDicomFile().DecodeFrame(frame); } + + void DicomInstanceToStore::CopyMetadata(const DicomInstanceToStore::MetadataMap& metadata) + { + for (MetadataMap::const_iterator it = metadata.begin(); + it != metadata.end(); ++it) + { + AddMetadata(it->first.first, it->first.second, it->second); + } + } + }
--- a/OrthancServer/Sources/DicomInstanceToStore.h Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/DicomInstanceToStore.h Sat Apr 30 19:39:40 2022 +0200 @@ -94,6 +94,8 @@ metadata_[std::make_pair(level, metadata)] = value; } + void CopyMetadata(const MetadataMap& metadata); + bool LookupTransferSyntax(DicomTransferSyntax& result) const; virtual ParsedDicomFile& GetParsedDicomFile() const = 0;
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp Sat Apr 30 19:39:40 2022 +0200 @@ -57,6 +57,7 @@ static const std::string CHECK_REVISIONS = "CheckRevisions"; static const char* const IGNORE_LENGTH = "ignore-length"; +static const char* const RECONSTRUCT_FILES = "ReconstructFiles"; namespace Orthanc @@ -3379,6 +3380,32 @@ call.GetOutput().AnswerBuffer("", MimeType_PlainText); } + void DocumentReconstructFilesField(RestApiPostCall& call) + { + call.GetDocumentation() + .SetRequestField(RECONSTRUCT_FILES, RestApiCallDocumentation::Type_Boolean, + "Also reconstruct the files of the resources (e.g: apply IngestTranscoding, StorageCompression). " + "'false' by default. (New in Orthanc 1.11.0)", false); + } + + bool GetReconstructFilesField(RestApiPostCall& call) + { + bool reconstructFiles = false; + Json::Value request; + + if (call.GetBodySize() > 0 && call.ParseJsonRequest(request) && request.isMember(RECONSTRUCT_FILES)) // allow "" payload to keep backward compatibility + { + if (!request[RECONSTRUCT_FILES].isBool()) + { + throw OrthancException(ErrorCode_BadFileFormat, + "The field " + std::string(RECONSTRUCT_FILES) + " must contain a Boolean"); + } + + reconstructFiles = request[RECONSTRUCT_FILES].asBool(); + } + + return reconstructFiles; + } template <enum ResourceType type> static void ReconstructResource(RestApiPostCall& call) @@ -3388,18 +3415,20 @@ const std::string resource = GetResourceTypeText(type, false /* plural */, false /* lower case */); call.GetDocumentation() .SetTag(GetResourceTypeText(type, true /* plural */, true /* upper case */)) - .SetSummary("Reconstruct tags of " + resource) - .SetDescription("Reconstruct the main DICOM tags of the " + resource + " whose Orthanc identifier is provided " + .SetSummary("Reconstruct tags & optionally files of " + resource) + .SetDescription("Reconstruct the main DICOM tags in DB of the " + resource + " whose Orthanc identifier is provided " "in the URL. This is useful if child studies/series/instances have inconsistent values for " "higher-level tags, in order to force Orthanc to use the value from the resource of interest. " "Beware that this is a time-consuming operation, as all the children DICOM instances will be " "parsed again, and the Orthanc index will be updated accordingly.") .SetUriArgument("id", "Orthanc identifier of the " + resource + " of interest"); + DocumentReconstructFilesField(call); + return; } ServerContext& context = OrthancRestApi::GetContext(call); - ServerToolbox::ReconstructResource(context, call.GetUriComponent("id", "")); + ServerToolbox::ReconstructResource(context, call.GetUriComponent("id", ""), GetReconstructFilesField(call)); call.GetOutput().AnswerBuffer("", MimeType_PlainText); } @@ -3414,7 +3443,11 @@ .SetDescription("Reconstruct the index of all the tags of all the DICOM instances that are stored in Orthanc. " "This is notably useful after the deletion of resources whose children resources have inconsistent " "values with their sibling resources. Beware that this is a highly time-consuming operation, " - "as all the DICOM instances will be parsed again, and as all the Orthanc index will be regenerated."); + "as all the DICOM instances will be parsed again, and as all the Orthanc index will be regenerated. " + "If you have a large database to process, it is advised to use the Housekeeper plugin to perform " + "this action resource by resource"); + DocumentReconstructFilesField(call); + return; } @@ -3422,11 +3455,12 @@ std::list<std::string> studies; context.GetIndex().GetAllUuids(studies, ResourceType_Study); + bool reconstructFiles = GetReconstructFilesField(call); for (std::list<std::string>::const_iterator study = studies.begin(); study != studies.end(); ++study) { - ServerToolbox::ReconstructResource(context, *study); + ServerToolbox::ReconstructResource(context, *study, reconstructFiles); } call.GetOutput().AnswerBuffer("", MimeType_PlainText);
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp Sat Apr 30 19:39:40 2022 +0200 @@ -74,6 +74,8 @@ static const char* const VERSION = "Version"; static const char* const MAIN_DICOM_TAGS = "MainDicomTags"; static const char* const STORAGE_COMPRESSION = "StorageCompression"; + static const char* const OVERWRITE_INSTANCES = "OverwriteInstances"; + static const char* const INGEST_TRANSCODING = "IngestTranscoding"; if (call.IsDocumentation()) { @@ -104,6 +106,10 @@ "The list of MainDicomTags saved in DB for each resource level (new in Orthanc 1.11.0)") .SetAnswerField(STORAGE_COMPRESSION, RestApiCallDocumentation::Type_Boolean, "Whether storage compression is enabled (new in Orthanc 1.11.0)") + .SetAnswerField(OVERWRITE_INSTANCES, RestApiCallDocumentation::Type_Boolean, + "Whether instances are overwritten when re-ingested (new in Orthanc 1.11.0)") + .SetAnswerField(INGEST_TRANSCODING, RestApiCallDocumentation::Type_String, + "Whether instances are transcoded when ingested into Orthanc (`""` if no transcoding is performed) (new in Orthanc 1.11.0)") .SetHttpGetSample("https://demo.orthanc-server.com/system", true); return; } @@ -125,6 +131,8 @@ result[NAME] = lock.GetConfiguration().GetStringParameter(NAME, ""); result[CHECK_REVISIONS] = lock.GetConfiguration().GetBooleanParameter(CHECK_REVISIONS, false); // New in Orthanc 1.9.2 result[STORAGE_COMPRESSION] = lock.GetConfiguration().GetBooleanParameter(STORAGE_COMPRESSION, false); // New in Orthanc 1.11.0 + result[OVERWRITE_INSTANCES] = lock.GetConfiguration().GetBooleanParameter(OVERWRITE_INSTANCES, false); // New in Orthanc 1.11.0 + result[INGEST_TRANSCODING] = lock.GetConfiguration().GetStringParameter(INGEST_TRANSCODING, ""); // New in Orthanc 1.11.0 } result[STORAGE_AREA_PLUGIN] = Json::nullValue;
--- a/OrthancServer/Sources/ServerContext.cpp Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/ServerContext.cpp Sat Apr 30 19:39:40 2022 +0200 @@ -495,7 +495,8 @@ ServerContext::StoreResult ServerContext::StoreAfterTranscoding(std::string& resultPublicId, DicomInstanceToStore& dicom, - StoreInstanceMode mode) + StoreInstanceMode mode, + bool isReconstruct) { bool overwrite; switch (mode) @@ -544,6 +545,7 @@ // Test if the instance must be filtered out StoreResult result; + if (!isReconstruct) // skip all filters if this is a reconstruction { boost::shared_lock<boost::shared_mutex> lock(listenersMutex_); @@ -615,7 +617,7 @@ InstanceMetadata instanceMetadata; result.SetStatus(index_.Store( instanceMetadata, summary, attachments, dicom.GetMetadata(), dicom.GetOrigin(), overwrite, - hasTransferSyntax, transferSyntax, hasPixelDataOffset, pixelDataOffset)); + hasTransferSyntax, transferSyntax, hasPixelDataOffset, pixelDataOffset, isReconstruct)); // Only keep the metadata for the "instance" level dicom.ClearMetadata(); @@ -636,41 +638,47 @@ } } - switch (result.GetStatus()) + if (!isReconstruct) // skip logs in case of reconstruction { - case StoreStatus_Success: - LOG(INFO) << "New instance stored"; - break; + switch (result.GetStatus()) + { + case StoreStatus_Success: + LOG(INFO) << "New instance stored"; + break; - case StoreStatus_AlreadyStored: - LOG(INFO) << "Already stored"; - break; + case StoreStatus_AlreadyStored: + LOG(INFO) << "Already stored"; + break; - case StoreStatus_Failure: - LOG(ERROR) << "Store failure"; - break; + case StoreStatus_Failure: + LOG(ERROR) << "Store failure"; + break; - default: - // This should never happen - break; + default: + // This should never happen + break; + } } - if (result.GetStatus() == StoreStatus_Success || - result.GetStatus() == StoreStatus_AlreadyStored) + if (!isReconstruct) // skip all signals if this is a reconstruction { - boost::shared_lock<boost::shared_mutex> lock(listenersMutex_); - - for (ServerListeners::iterator it = listeners_.begin(); it != listeners_.end(); ++it) + if (result.GetStatus() == StoreStatus_Success || + result.GetStatus() == StoreStatus_AlreadyStored) { - try + boost::shared_lock<boost::shared_mutex> lock(listenersMutex_); + + for (ServerListeners::iterator it = listeners_.begin(); it != listeners_.end(); ++it) { - it->GetListener().SignalStoredInstance(resultPublicId, dicom, simplifiedTags); - } - catch (OrthancException& e) - { - LOG(ERROR) << "Error in the " << it->GetDescription() - << " callback while receiving an instance: " << e.What() - << " (code " << e.GetErrorCode() << ")"; + try + { + it->GetListener().SignalStoredInstance(resultPublicId, dicom, simplifiedTags); + } + catch (OrthancException& e) + { + LOG(ERROR) << "Error in the " << it->GetDescription() + << " callback while receiving an instance: " << e.What() + << " (code " << e.GetErrorCode() << ")"; + } } } } @@ -743,10 +751,19 @@ } #endif + return TranscodeAndStore(resultPublicId, dicom, mode); + } + + ServerContext::StoreResult ServerContext::TranscodeAndStore(std::string& resultPublicId, + DicomInstanceToStore* dicom, + StoreInstanceMode mode, + bool isReconstruct) + { + if (!isIngestTranscoding_) { // No automated transcoding. This was the only path in Orthanc <= 1.6.1. - return StoreAfterTranscoding(resultPublicId, *dicom, mode); + return StoreAfterTranscoding(resultPublicId, *dicom, mode, isReconstruct); } else { @@ -782,7 +799,7 @@ if (!transcode) { // No transcoding - return StoreAfterTranscoding(resultPublicId, *dicom, mode); + return StoreAfterTranscoding(resultPublicId, *dicom, mode, isReconstruct); } else { @@ -801,7 +818,12 @@ std::unique_ptr<DicomInstanceToStore> toStore(DicomInstanceToStore::CreateFromParsedDicomFile(*tmp)); toStore->SetOrigin(dicom->GetOrigin()); - StoreResult result = StoreAfterTranscoding(resultPublicId, *toStore, mode); + if (isReconstruct) // the initial instance to store already has its own metadata + { + toStore->CopyMetadata(dicom->GetMetadata()); + } + + StoreResult result = StoreAfterTranscoding(resultPublicId, *toStore, mode, isReconstruct); assert(resultPublicId == tmp->GetHasher().HashInstance()); return result; @@ -809,7 +831,7 @@ else { // Cannot transcode => store the original file - return StoreAfterTranscoding(resultPublicId, *dicom, mode); + return StoreAfterTranscoding(resultPublicId, *dicom, mode, isReconstruct); } } }
--- a/OrthancServer/Sources/ServerContext.h Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/ServerContext.h Sat Apr 30 19:39:40 2022 +0200 @@ -261,7 +261,8 @@ StoreResult StoreAfterTranscoding(std::string& resultPublicId, DicomInstanceToStore& dicom, - StoreInstanceMode mode); + StoreInstanceMode mode, + bool isReconstruct); void PublishDicomCacheMetrics(); @@ -335,6 +336,11 @@ DicomInstanceToStore& dicom, StoreInstanceMode mode); + StoreResult TranscodeAndStore(std::string& resultPublicId, + DicomInstanceToStore* dicom, + StoreInstanceMode mode, + bool isReconstruct = false); + void AnswerAttachment(RestApiOutput& output, const std::string& resourceId, FileContentType content);
--- a/OrthancServer/Sources/ServerIndex.cpp Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/ServerIndex.cpp Sat Apr 30 19:39:40 2022 +0200 @@ -518,7 +518,8 @@ bool hasTransferSyntax, DicomTransferSyntax transferSyntax, bool hasPixelDataOffset, - uint64_t pixelDataOffset) + uint64_t pixelDataOffset, + bool isReconstruct) { uint64_t maximumStorageSize; unsigned int maximumPatients; @@ -531,7 +532,7 @@ return StatelessDatabaseOperations::Store( instanceMetadata, dicomSummary, attachments, metadata, origin, overwrite, hasTransferSyntax, - transferSyntax, hasPixelDataOffset, pixelDataOffset, maximumStorageSize, maximumPatients); + transferSyntax, hasPixelDataOffset, pixelDataOffset, maximumStorageSize, maximumPatients, isReconstruct); }
--- a/OrthancServer/Sources/ServerIndex.h Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/ServerIndex.h Sat Apr 30 19:39:40 2022 +0200 @@ -84,7 +84,8 @@ bool hasTransferSyntax, DicomTransferSyntax transferSyntax, bool hasPixelDataOffset, - uint64_t pixelDataOffset); + uint64_t pixelDataOffset, + bool isResonstruct); StoreStatus AddAttachment(int64_t& newRevision /*out*/, const FileInfo& attachment,
--- a/OrthancServer/Sources/ServerToolbox.cpp Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/ServerToolbox.cpp Sat Apr 30 19:39:40 2022 +0200 @@ -254,7 +254,8 @@ void ReconstructResource(ServerContext& context, - const std::string& resource) + const std::string& resource, + bool reconstructFiles) { LOG(WARNING) << "Reconstructing resource " << resource; @@ -271,7 +272,30 @@ -1 /* dummy revision */, "" /* dummy MD5 */); context.GetIndex().ReconstructInstance(locker.GetDicom()); + + if (reconstructFiles) + { + // preserve metadata from old resource + typedef std::map<MetadataType, std::string> InstanceMetadata; + InstanceMetadata instanceMetadata; + + std::string resultPublicId; // ignored + std::unique_ptr<DicomInstanceToStore> dicomInstancetoStore(DicomInstanceToStore::CreateFromParsedDicomFile(locker.GetDicom())); + + context.GetIndex().GetAllMetadata(instanceMetadata, *it, ResourceType_Instance); + + for (InstanceMetadata::const_iterator itm = instanceMetadata.begin(); + itm != instanceMetadata.end(); ++itm) + { + dicomInstancetoStore->AddMetadata(ResourceType_Instance, itm->first, itm->second); + } + + context.TranscodeAndStore(resultPublicId, dicomInstancetoStore.get(), StoreInstanceMode_OverwriteDuplicate, true); + } } } } } + + +todo: add a status route for the plugin !!! \ No newline at end of file
--- a/OrthancServer/Sources/ServerToolbox.h Tue Apr 26 16:14:49 2022 +0200 +++ b/OrthancServer/Sources/ServerToolbox.h Sat Apr 30 19:39:40 2022 +0200 @@ -54,6 +54,7 @@ std::string NormalizeIdentifier(const std::string& value); void ReconstructResource(ServerContext& context, - const std::string& resource); + const std::string& resource, + bool reconstructFiles); } }