Mercurial > hg > orthanc
changeset 5311:79fa77e9fa0d am-experimental
merge default -> am-experimental
| author | Alain Mazy <am@osimis.io> |
|---|---|
| date | Fri, 16 Jun 2023 09:26:33 +0200 |
| parents | 9504de20d43d (current diff) b5c502bcaf99 (diff) |
| children | |
| files | NEWS OrthancFramework/Sources/DicomParsing/FromDcmtkBridge.cpp |
| diffstat | 20 files changed, 171 insertions(+), 38 deletions(-) [+] |
line wrap: on
line diff
--- a/NEWS Thu May 25 17:19:52 2023 +0200 +++ b/NEWS Fri Jun 16 09:26:33 2023 +0200 @@ -4,7 +4,10 @@ REST API -------- +* API version upgraded to 21 * New URI /instances/{id}/file-until-pixel-data +* added a route to delete the output of an asynchronous job (right now only for archive jobs): + e.g. DELETE /jobs/../archive Maintenance ----------- @@ -18,6 +21,11 @@ * Fix orphan files remaining in storage when working with MaximumStorageSize (https://discourse.orthanc-server.org/t/issue-with-deleting-incoming-dicoms-when-maximumstoragesize-is-reached/3510) * When deleting a resource, its parents LastUpdate metadata are now updated. +* Reduced the memory usage when downloading archives when "ZipLoaderThreads" > 0. +* Disabled automatic conversion from YBR to RGB when transcoding JPEG to RAW transfer-syntax + (https://discourse.orthanc-server.org/t/orthanc-convert-ybr-to-rgb-but-does-not-change-metadata/3533). + This might have an impact on the image returned by /dicom-web/studies/../series/../instances/../frames/1; the + image format is now consistent with the PhotometricIntepretation DICOM Tag. * WIP: new dicomWeb Json format for some of the Rest API routes. * WIP: new 'include' get arguments for some of the Rest API routes to define
--- a/OrthancFramework/Sources/Cache/SharedArchive.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/Cache/SharedArchive.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -102,7 +102,7 @@ std::string SharedArchive::Add(IDynamicObject* obj) { - boost::mutex::scoped_lock lock(mutex_); + boost::recursive_mutex::scoped_lock lock(mutex_); if (archive_.size() == maxSize_) { @@ -122,7 +122,7 @@ void SharedArchive::Remove(const std::string& id) { - boost::mutex::scoped_lock lock(mutex_); + boost::recursive_mutex::scoped_lock lock(mutex_); RemoveInternal(id); } @@ -132,7 +132,7 @@ items.clear(); { - boost::mutex::scoped_lock lock(mutex_); + boost::recursive_mutex::scoped_lock lock(mutex_); for (Archive::const_iterator it = archive_.begin(); it != archive_.end(); ++it)
--- a/OrthancFramework/Sources/Cache/SharedArchive.h Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/Cache/SharedArchive.h Fri Jun 16 09:26:33 2023 +0200 @@ -44,9 +44,9 @@ private: typedef std::map<std::string, IDynamicObject*> Archive; - size_t maxSize_; - boost::mutex mutex_; - Archive archive_; + size_t maxSize_; + boost::recursive_mutex mutex_; + Archive archive_; LeastRecentlyUsedIndex<std::string> lru_; void RemoveInternal(const std::string& id); @@ -55,8 +55,8 @@ class ORTHANC_PUBLIC Accessor : public boost::noncopyable { private: - boost::mutex::scoped_lock lock_; - IDynamicObject* item_; + boost::recursive_mutex::scoped_lock lock_; + IDynamicObject* item_; public: Accessor(SharedArchive& that,
--- a/OrthancFramework/Sources/DicomParsing/FromDcmtkBridge.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/DicomParsing/FromDcmtkBridge.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -1994,7 +1994,14 @@ case EVR_OL: // other long (requires byte-swapping) #endif { - ok = element.putUint32(boost::lexical_cast<Uint32>(*decoded)).good(); + if (decoded->find('\\') != std::string::npos) + { + ok = element.putString(decoded->c_str()).good(); + } + else + { + ok = element.putUint32(boost::lexical_cast<Uint32>(*decoded)).good(); + } break; } @@ -2473,7 +2480,7 @@ #if ORTHANC_ENABLE_DCMTK_JPEG == 1 CLOG(INFO, DICOM) << "Registering JPEG codecs in DCMTK"; - DJDecoderRegistration::registerCodecs(); + DJDecoderRegistration::registerCodecs(EDC_never); // disable automatic conversion from YBR to RGB when transcoding JPEG to RAW transfer-syntax (https://discourse.orthanc-server.org/t/orthanc-convert-ybr-to-rgb-but-does-not-change-metadata/3533) # if ORTHANC_ENABLE_DCMTK_TRANSCODING == 1 DJEncoderRegistration::registerCodecs(); # endif
--- a/OrthancFramework/Sources/JobsEngine/IJob.h Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/JobsEngine/IJob.h Fri Jun 16 09:26:33 2023 +0200 @@ -62,5 +62,9 @@ MimeType& mime, std::string& filename, const std::string& key) = 0; + + // This function can only be called if the job has reached its + // "success" state + virtual bool DeleteOutput(const std::string& key) = 0; }; }
--- a/OrthancFramework/Sources/JobsEngine/JobsRegistry.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/JobsEngine/JobsRegistry.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -679,6 +679,33 @@ } } + bool JobsRegistry::DeleteJobOutput(const std::string& job, + const std::string& key) + { + boost::mutex::scoped_lock lock(mutex_); + CheckInvariants(); + + JobsIndex::const_iterator found = jobsIndex_.find(job); + + if (found == jobsIndex_.end()) + { + return false; + } + else + { + const JobHandler& handler = *found->second; + + if (handler.GetState() == JobState_Success) + { + return handler.GetJob().DeleteOutput(key); + } + else + { + return false; + } + } + } + void JobsRegistry::SubmitInternal(std::string& id, JobHandler* handler)
--- a/OrthancFramework/Sources/JobsEngine/JobsRegistry.h Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/JobsEngine/JobsRegistry.h Fri Jun 16 09:26:33 2023 +0200 @@ -153,6 +153,9 @@ const std::string& job, const std::string& key); + bool DeleteJobOutput(const std::string& job, + const std::string& key); + void Serialize(Json::Value& target); void Submit(std::string& id,
--- a/OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -447,13 +447,6 @@ return true; } - bool SequenceOfOperationsJob::GetOutput(std::string& output, - MimeType& mime, - std::string& filename, - const std::string& key) - { - return false; - } void SequenceOfOperationsJob::AwakeTrailingSleep() {
--- a/OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.h Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.h Fri Jun 16 09:26:33 2023 +0200 @@ -127,7 +127,15 @@ virtual bool GetOutput(std::string& output, MimeType& mime, std::string& filename, - const std::string& key) ORTHANC_OVERRIDE; + const std::string& key) ORTHANC_OVERRIDE + { + return false; + } + + virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE + { + return false; + } void AwakeTrailingSleep(); };
--- a/OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -269,14 +269,6 @@ return true; } - bool SetOfCommandsJob::GetOutput(std::string &output, - MimeType &mime, - std::string& filename, - const std::string &key) - { - return false; - } - SetOfCommandsJob::SetOfCommandsJob(ICommandUnserializer* unserializer, const Json::Value& source) :
--- a/OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.h Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.h Fri Jun 16 09:26:33 2023 +0200 @@ -106,6 +106,14 @@ virtual bool GetOutput(std::string& output, MimeType& mime, std::string& filename, - const std::string& key) ORTHANC_OVERRIDE; + const std::string& key) ORTHANC_OVERRIDE + { + return false; + } + + virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE + { + return false; + } }; }
--- a/OrthancFramework/UnitTestsSources/JobsTests.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancFramework/UnitTestsSources/JobsTests.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -131,6 +131,11 @@ { return false; } + + virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE + { + return false; + } };
--- a/OrthancServer/Plugins/Engine/PluginsJob.h Thu May 25 17:19:52 2023 +0200 +++ b/OrthancServer/Plugins/Engine/PluginsJob.h Fri Jun 16 09:26:33 2023 +0200 @@ -76,6 +76,12 @@ // TODO return false; } + + virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE + { + // TODO + return false; + } }; }
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -770,6 +770,36 @@ } + static void DeleteJobOutput(RestApiDeleteCall& call) + { + if (call.IsDocumentation()) + { + call.GetDocumentation() + .SetTag("Jobs") + .SetSummary("Delete a job output") + .SetDescription("Delete the output produced by a job. As of Orthanc 1.12.1, only the jobs that generate a " + "DICOMDIR media or a ZIP archive provide such an output (with `key` equals to `archive`).") + .SetUriArgument("id", "Identifier of the job of interest") + .SetUriArgument("key", "Name of the output of interest"); + return; + } + + std::string job = call.GetUriComponent("id", ""); + std::string key = call.GetUriComponent("key", ""); + + if (OrthancRestApi::GetContext(call).GetJobsEngine(). + GetRegistry().DeleteJobOutput(job, key)) + { + call.GetOutput().AnswerBuffer("", MimeType_PlainText); + } + else + { + throw OrthancException(ErrorCode_InexistentItem, + "Job has no such output: " + key); + } + } + + enum JobAction { JobAction_Cancel, @@ -1113,6 +1143,7 @@ Register("/jobs/{id}/resubmit", ApplyJobAction<JobAction_Resubmit>); Register("/jobs/{id}/resume", ApplyJobAction<JobAction_Resume>); Register("/jobs/{id}/{key}", GetJobOutput); + Register("/jobs/{id}/{key}", DeleteJobOutput); // New in Orthanc 1.9.0 Register("/tools/accepted-transfer-syntaxes", GetAcceptedTransferSyntaxes);
--- a/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -176,6 +176,7 @@ class ArchiveJob::ThreadedInstanceLoader : public ArchiveJob::InstanceLoader { Semaphore availableInstancesSemaphore_; + Semaphore bufferedInstancesSemaphore_; std::map<std::string, boost::shared_ptr<std::string> > availableInstances_; boost::mutex availableInstancesMutex_; SharedMessageQueue instancesToPreload_; @@ -185,7 +186,8 @@ public: ThreadedInstanceLoader(ServerContext& context, size_t threadCount, bool transcode, DicomTransferSyntax transferSyntax) : InstanceLoader(context, transcode, transferSyntax), - availableInstancesSemaphore_(0) + availableInstancesSemaphore_(0), + bufferedInstancesSemaphore_(3*threadCount) { for (size_t i = 0; i < threadCount; i++) { @@ -227,6 +229,9 @@ { return; } + + // wait for the consumers (zip writer), no need to accumulate instances in memory if loaders are faster than writers + that->bufferedInstancesSemaphore_.Acquire(); try { @@ -270,6 +275,7 @@ { // wait for an instance to be available but this might not be the one we are waiting for ! availableInstancesSemaphore_.Acquire(); + bufferedInstancesSemaphore_.Release(); // unlock the "flow" of loaders boost::shared_ptr<std::string> dicomContent; { @@ -1453,4 +1459,27 @@ return false; } } + + bool ArchiveJob::DeleteOutput(const std::string& key) + { + if (key == "archive" && + !mediaArchiveId_.empty()) + { + SharedArchive::Accessor accessor(context_.GetMediaArchive(), mediaArchiveId_); + + if (accessor.IsValid()) + { + context_.GetMediaArchive().Remove(mediaArchiveId_); + return true; + } + else + { + return false; + } + } + else + { + return false; + } + } }
--- a/OrthancServer/Sources/ServerJobs/ArchiveJob.h Thu May 25 17:19:52 2023 +0200 +++ b/OrthancServer/Sources/ServerJobs/ArchiveJob.h Fri Jun 16 09:26:33 2023 +0200 @@ -120,5 +120,7 @@ MimeType& mime, std::string& filename, const std::string& key) ORTHANC_OVERRIDE; + + virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE; }; }
--- a/OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -295,15 +295,6 @@ } - bool ThreadedSetOfInstancesJob::GetOutput(std::string &output, - MimeType &mime, - std::string& filename, - const std::string &key) - { - return false; - } - - size_t ThreadedSetOfInstancesJob::GetInstancesCount() const { boost::recursive_mutex::scoped_lock lock(mutex_);
--- a/OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.h Thu May 25 17:19:52 2023 +0200 +++ b/OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.h Fri Jun 16 09:26:33 2023 +0200 @@ -152,7 +152,15 @@ virtual bool GetOutput(std::string& output, MimeType& mime, std::string& filename, - const std::string& key) ORTHANC_OVERRIDE; + const std::string& key) ORTHANC_OVERRIDE + { + return false; + } + + virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE + { + return false; + } bool IsFailedInstance(const std::string& instance) const;
--- a/OrthancServer/UnitTestsSources/ServerJobsTests.cpp Thu May 25 17:19:52 2023 +0200 +++ b/OrthancServer/UnitTestsSources/ServerJobsTests.cpp Fri Jun 16 09:26:33 2023 +0200 @@ -134,6 +134,11 @@ { return false; } + + virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE + { + return false; + } };
--- a/TODO Thu May 25 17:19:52 2023 +0200 +++ b/TODO Fri Jun 16 09:26:33 2023 +0200 @@ -99,6 +99,8 @@ https://groups.google.com/g/orthanc-users/c/y3-xa_GcdLM/m/m0Kr5G5UPAAJ * (1) Specify the transfer syntax in /tools/create-dicom https://groups.google.com/g/orthanc-users/c/o15Dekecgds/m/xmPE2y3bAwAJ +* Support Palette PNG in /tools/create-dicom: + https://discourse.orthanc-server.org/t/404-on-tools-create-dicom-endpoint-with-specific-png/3562 * (1) In the /studies/{id}/anonymize route, add an option to remove secondary captures. They usually contains Patient info in the image. The SOPClassUID might be used to identify such secondary @@ -186,6 +188,8 @@ - ModalitiesInStudy - all computed counters at series/study/patient level - RequestAttributesSequence (sequence that must be included in all DicomWeb QIDO-RS for series) +* Investigate increase of CPU consumption while idle after anonymize/delete + (https://discourse.orthanc-server.org/t/onchange-callbacks-and-cpu-loads/3534) * Long-shot & not sure it is even feasible at all: try to reduce memory usage by implementing streaming when receiving DICOM instances from the Rest API or from DICOM and store files directly to disk as they @@ -322,6 +326,8 @@ * Add more complex testing scenarios like data-migration, change of configuration files, multiple orthanc interacting togethers with various config. This should probably look like the python toolbox tests ... + - add a test to validate Modalities and Peers stored in DB are not lost + while upgrading from one version to the other (Sylvain) ---------------------