# HG changeset patch # User Sebastien Jodogne # Date 1544095650 -3600 # Node ID ee0b2c5ad49bd289c3e15bc200e515bde321736e # Parent cb5d75143da03f126b7609d7cf20ac4ecd29a665# Parent e62e296a5714bff5cbf602af36345778eb529e41 merge diff -r e62e296a5714 -r ee0b2c5ad49b Core/Cache/SharedArchive.cpp --- a/Core/Cache/SharedArchive.cpp Thu Dec 06 11:16:23 2018 +0100 +++ b/Core/Cache/SharedArchive.cpp Thu Dec 06 12:27:30 2018 +0100 @@ -47,6 +47,8 @@ { delete it->second; archive_.erase(it); + + lru_.Invalidate(id); } } @@ -59,7 +61,7 @@ if (it == that.archive_.end()) { - throw OrthancException(ErrorCode_InexistentItem); + item_ = NULL; } else { @@ -69,6 +71,20 @@ } + IDynamicObject& SharedArchive::Accessor::GetItem() const + { + if (item_ == NULL) + { + // "IsValid()" should have been called + throw OrthancException(ErrorCode_BadSequenceOfCalls); + } + else + { + return *item_; + } + } + + SharedArchive::SharedArchive(size_t maxSize) : maxSize_(maxSize) { @@ -96,12 +112,12 @@ if (archive_.size() == maxSize_) { // The quota has been reached, remove the oldest element - std::string oldest = lru_.RemoveOldest(); - RemoveInternal(oldest); + RemoveInternal(lru_.GetOldest()); } std::string id = Toolbox::GenerateUuid(); RemoveInternal(id); // Should never be useful because of UUID + archive_[id] = obj; lru_.Add(id); @@ -113,7 +129,6 @@ { boost::mutex::scoped_lock lock(mutex_); RemoveInternal(id); - lru_.Invalidate(id); } @@ -121,14 +136,14 @@ { items.clear(); - boost::mutex::scoped_lock lock(mutex_); + { + boost::mutex::scoped_lock lock(mutex_); - for (Archive::const_iterator it = archive_.begin(); - it != archive_.end(); ++it) - { - items.push_back(it->first); + for (Archive::const_iterator it = archive_.begin(); + it != archive_.end(); ++it) + { + items.push_back(it->first); + } } } } - - diff -r e62e296a5714 -r ee0b2c5ad49b Core/Cache/SharedArchive.h --- a/Core/Cache/SharedArchive.h Thu Dec 06 11:16:23 2018 +0100 +++ b/Core/Cache/SharedArchive.h Thu Dec 06 12:27:30 2018 +0100 @@ -72,10 +72,12 @@ Accessor(SharedArchive& that, const std::string& id); - IDynamicObject& GetItem() const + bool IsValid() const { - return *item_; - } + return item_ != NULL; + } + + IDynamicObject& GetItem() const; }; diff -r e62e296a5714 -r ee0b2c5ad49b Core/Enumerations.cpp --- a/Core/Enumerations.cpp Thu Dec 06 11:16:23 2018 +0100 +++ b/Core/Enumerations.cpp Thu Dec 06 12:27:30 2018 +0100 @@ -54,6 +54,7 @@ static const char* const MIME_PLAIN_TEXT = "text/plain"; static const char* const MIME_WEB_ASSEMBLY = "application/wasm"; static const char* const MIME_XML_2 = "text/xml"; + static const char* const MIME_ZIP = "application/zip"; // This function is autogenerated by the script // "Resources/GenerateErrorCodes.py" @@ -1084,6 +1085,9 @@ case MimeType_Gif: return MIME_GIF; + case MimeType_Zip: + return MIME_ZIP; + default: throw OrthancException(ErrorCode_ParameterOutOfRange); } @@ -1672,6 +1676,10 @@ { return MimeType_Gif; } + else if (mime == MIME_ZIP) + { + return MimeType_Zip; + } else { throw OrthancException(ErrorCode_ParameterOutOfRange); diff -r e62e296a5714 -r ee0b2c5ad49b Core/Enumerations.h --- a/Core/Enumerations.h Thu Dec 06 11:16:23 2018 +0100 +++ b/Core/Enumerations.h Thu Dec 06 12:27:30 2018 +0100 @@ -100,7 +100,8 @@ MimeType_JavaScript, MimeType_Css, MimeType_WebAssembly, - MimeType_Gif + MimeType_Gif, + MimeType_Zip }; diff -r e62e296a5714 -r ee0b2c5ad49b Core/JobsEngine/IJob.h --- a/Core/JobsEngine/IJob.h Thu Dec 06 11:16:23 2018 +0100 +++ b/Core/JobsEngine/IJob.h Thu Dec 06 12:27:30 2018 +0100 @@ -65,5 +65,11 @@ virtual void GetPublicContent(Json::Value& value) = 0; virtual bool Serialize(Json::Value& value) = 0; + + // This function can only be called if the job has reached its + // "success" state + virtual bool GetOutput(std::string& output, + MimeType& mime, + const std::string& key) = 0; }; } diff -r e62e296a5714 -r ee0b2c5ad49b Core/JobsEngine/JobsRegistry.cpp --- a/Core/JobsEngine/JobsRegistry.cpp Thu Dec 06 11:16:23 2018 +0100 +++ b/Core/JobsEngine/JobsRegistry.cpp Thu Dec 06 12:27:30 2018 +0100 @@ -263,7 +263,7 @@ // as a "RunningJob" instance is running. We do not use a // mutex at the "JobHandler" level, as serialization would be // blocked while a step in the job is running. Instead, we - // save a snapshot of the serialized job. + // save a snapshot of the serialized job. (*) if (lastStatus_.HasSerialized()) { @@ -631,6 +631,36 @@ } + bool JobsRegistry::GetJobOutput(std::string& output, + MimeType& mime, + const std::string& job, + const std::string& key) + { + boost::mutex::scoped_lock lock(mutex_); + CheckInvariants(); + + JobsIndex::const_iterator found = jobsIndex_.find(job); + + if (found == jobsIndex_.end()) + { + return false; + } + else + { + const JobHandler& handler = *found->second; + + if (handler.GetState() == JobState_Success) + { + return handler.GetJob().GetOutput(output, mime, key); + } + else + { + return false; + } + } + } + + void JobsRegistry::SubmitInternal(std::string& id, JobHandler* handler) { diff -r e62e296a5714 -r ee0b2c5ad49b Core/JobsEngine/JobsRegistry.h --- a/Core/JobsEngine/JobsRegistry.h Thu Dec 06 11:16:23 2018 +0100 +++ b/Core/JobsEngine/JobsRegistry.h Thu Dec 06 12:27:30 2018 +0100 @@ -161,6 +161,11 @@ bool GetJobInfo(JobInfo& target, const std::string& id); + bool GetJobOutput(std::string& output, + MimeType& mime, + const std::string& job, + const std::string& key); + void Serialize(Json::Value& target); void Submit(std::string& id, diff -r e62e296a5714 -r ee0b2c5ad49b Core/JobsEngine/Operations/SequenceOfOperationsJob.h --- a/Core/JobsEngine/Operations/SequenceOfOperationsJob.h Thu Dec 06 11:16:23 2018 +0100 +++ b/Core/JobsEngine/Operations/SequenceOfOperationsJob.h Thu Dec 06 12:27:30 2018 +0100 @@ -147,6 +147,13 @@ virtual bool Serialize(Json::Value& value); + virtual bool GetOutput(std::string& output, + MimeType& mime, + const std::string& key) + { + return false; + } + void AwakeTrailingSleep() { operationAdded_.notify_one(); diff -r e62e296a5714 -r ee0b2c5ad49b Core/JobsEngine/SetOfCommandsJob.h --- a/Core/JobsEngine/SetOfCommandsJob.h Thu Dec 06 11:16:23 2018 +0100 +++ b/Core/JobsEngine/SetOfCommandsJob.h Thu Dec 06 12:27:30 2018 +0100 @@ -131,5 +131,12 @@ virtual void GetPublicContent(Json::Value& value); virtual bool Serialize(Json::Value& target); + + virtual bool GetOutput(std::string& output, + MimeType& mime, + const std::string& key) + { + return false; + } }; } diff -r e62e296a5714 -r ee0b2c5ad49b NEWS --- a/NEWS Thu Dec 06 11:16:23 2018 +0100 +++ b/NEWS Thu Dec 06 12:27:30 2018 +0100 @@ -21,6 +21,7 @@ -------- * API Version has been upgraded to 1.2 +* Asynchronous generation of ZIP archives and DICOM medias * New URI: "/studies/.../merge" to merge a study * New URI: "/studies/.../split" to split a study * POST-ing a DICOM file to "/instances" also answers the patient/study/series ID diff -r e62e296a5714 -r ee0b2c5ad49b OrthancServer/OrthancRestApi/OrthancRestApi.cpp --- a/OrthancServer/OrthancRestApi/OrthancRestApi.cpp Thu Dec 06 11:16:23 2018 +0100 +++ b/OrthancServer/OrthancRestApi/OrthancRestApi.cpp Thu Dec 06 12:27:30 2018 +0100 @@ -211,10 +211,11 @@ } - void OrthancRestApi::SubmitGenericJob(RestApiPostCall& call, + void OrthancRestApi::SubmitGenericJob(RestApiOutput& output, + ServerContext& context, IJob* job, - bool isDefaultSynchronous, - const Json::Value& body) const + bool synchronous, + int priority) { std::auto_ptr raii(job); @@ -223,41 +224,55 @@ throw OrthancException(ErrorCode_NullPointer); } - if (body.type() != Json::objectValue) - { - throw OrthancException(ErrorCode_BadFileFormat); - } - - if (IsSynchronousJobRequest(isDefaultSynchronous, body)) + if (synchronous) { Json::Value successContent; - if (context_.GetJobsEngine().GetRegistry().SubmitAndWait - (successContent, raii.release(), GetJobRequestPriority(body))) + if (context.GetJobsEngine().GetRegistry().SubmitAndWait + (successContent, raii.release(), priority)) { // Success in synchronous execution - call.GetOutput().AnswerJson(successContent); + output.AnswerJson(successContent); } else { // Error during synchronous execution - call.GetOutput().SignalError(HttpStatus_500_InternalServerError); + output.SignalError(HttpStatus_500_InternalServerError); } } else { // Asynchronous mode: Submit the job, but don't wait for its completion std::string id; - context_.GetJobsEngine().GetRegistry().Submit - (id, raii.release(), GetJobRequestPriority(body)); + context.GetJobsEngine().GetRegistry().Submit + (id, raii.release(), priority); Json::Value v; v["ID"] = id; v["Path"] = "/jobs/" + id; - call.GetOutput().AnswerJson(v); + output.AnswerJson(v); } } + void OrthancRestApi::SubmitGenericJob(RestApiPostCall& call, + IJob* job, + bool isDefaultSynchronous, + const Json::Value& body) const + { + std::auto_ptr raii(job); + + if (body.type() != Json::objectValue) + { + throw OrthancException(ErrorCode_BadFileFormat); + } + + bool synchronous = IsSynchronousJobRequest(isDefaultSynchronous, body); + int priority = GetJobRequestPriority(body); + + SubmitGenericJob(call.GetOutput(), context_, raii.release(), synchronous, priority); + } + + void OrthancRestApi::SubmitCommandsJob(RestApiPostCall& call, SetOfCommandsJob* job, bool isDefaultSynchronous, @@ -265,11 +280,6 @@ { std::auto_ptr raii(job); - if (job == NULL) - { - throw OrthancException(ErrorCode_NullPointer); - } - if (body.type() != Json::objectValue) { throw OrthancException(ErrorCode_BadFileFormat); diff -r e62e296a5714 -r ee0b2c5ad49b OrthancServer/OrthancRestApi/OrthancRestApi.h --- a/OrthancServer/OrthancRestApi/OrthancRestApi.h Thu Dec 06 11:16:23 2018 +0100 +++ b/OrthancServer/OrthancRestApi/OrthancRestApi.h Thu Dec 06 12:27:30 2018 +0100 @@ -108,6 +108,12 @@ static unsigned int GetJobRequestPriority(const Json::Value& body); + static void SubmitGenericJob(RestApiOutput& output, + ServerContext& context, + IJob* job, + bool synchronous, + int priority); + void SubmitGenericJob(RestApiPostCall& call, IJob* job, bool isDefaultSynchronous, diff -r e62e296a5714 -r ee0b2c5ad49b OrthancServer/OrthancRestApi/OrthancRestArchive.cpp --- a/OrthancServer/OrthancRestApi/OrthancRestArchive.cpp Thu Dec 06 11:16:23 2018 +0100 +++ b/OrthancServer/OrthancRestApi/OrthancRestArchive.cpp Thu Dec 06 12:27:30 2018 +0100 @@ -155,7 +155,7 @@ } else { - throw OrthancException(ErrorCode_NotImplemented); + OrthancRestApi::SubmitGenericJob(output, context, job.release(), false, priority); } } diff -r e62e296a5714 -r ee0b2c5ad49b OrthancServer/OrthancRestApi/OrthancRestSystem.cpp --- a/OrthancServer/OrthancRestApi/OrthancRestSystem.cpp Thu Dec 06 11:16:23 2018 +0100 +++ b/OrthancServer/OrthancRestApi/OrthancRestSystem.cpp Thu Dec 06 12:27:30 2018 +0100 @@ -325,6 +325,27 @@ } + static void GetJobOutput(RestApiGetCall& call) + { + std::string job = call.GetUriComponent("id", ""); + std::string key = call.GetUriComponent("key", ""); + + std::string value; + MimeType mime; + + if (OrthancRestApi::GetContext(call).GetJobsEngine(). + GetRegistry().GetJobOutput(value, mime, job, key)) + { + call.GetOutput().AnswerBuffer(value, mime); + } + else + { + throw OrthancException(ErrorCode_InexistentItem, + "Job has no such output: " + key); + } + } + + enum JobAction { JobAction_Cancel, @@ -392,5 +413,6 @@ Register("/jobs/{id}/pause", ApplyJobAction); Register("/jobs/{id}/resubmit", ApplyJobAction); Register("/jobs/{id}/resume", ApplyJobAction); + Register("/jobs/{id}/{key}", GetJobOutput); } } diff -r e62e296a5714 -r ee0b2c5ad49b OrthancServer/ServerContext.h --- a/OrthancServer/ServerContext.h Thu Dec 06 11:16:23 2018 +0100 +++ b/OrthancServer/ServerContext.h Thu Dec 06 12:27:30 2018 +0100 @@ -164,11 +164,13 @@ LuaScripting mainLua_; LuaScripting filterLua_; LuaServerListener luaListener_; - + std::auto_ptr mediaArchive_; + // The "JobsEngine" must be *after* "LuaScripting", as // "LuaScripting" embeds "LuaJobManager" that registers as an // observer to "SequenceOfOperationsJob", whose lifetime - // corresponds to that of "JobsEngine" + // corresponds to that of "JobsEngine". It must also be after + // "mediaArchive_", as jobs might access this archive. JobsEngine jobsEngine_; #if ORTHANC_ENABLE_PLUGINS == 1 @@ -189,8 +191,6 @@ std::string defaultLocalAet_; OrthancHttpHandler httpHandler_; - std::auto_ptr mediaArchive_; - public: class DicomCacheLocker : public boost::noncopyable { diff -r e62e296a5714 -r ee0b2c5ad49b OrthancServer/ServerJobs/ArchiveJob.cpp --- a/OrthancServer/ServerJobs/ArchiveJob.cpp Thu Dec 06 11:16:23 2018 +0100 +++ b/OrthancServer/ServerJobs/ArchiveJob.cpp Thu Dec 06 12:27:30 2018 +0100 @@ -47,7 +47,12 @@ static const uint64_t MEGA_BYTES = 1024 * 1024; static const uint64_t GIGA_BYTES = 1024 * 1024 * 1024; -static const char* MEDIA_IMAGES_FOLDER = "IMAGES"; + +static const char* const MEDIA_IMAGES_FOLDER = "IMAGES"; +static const char* const KEY_DESCRIPTION = "Description"; +static const char* const KEY_INSTANCES_COUNT = "InstancesCount"; +static const char* const KEY_UNCOMPRESSED_SIZE_MB = "UncompressedSizeMB"; + namespace Orthanc { @@ -791,6 +796,15 @@ { } + + ArchiveJob::~ArchiveJob() + { + if (!mediaArchiveId_.empty()) + { + context_.GetMediaArchive().Remove(mediaArchiveId_); + } + } + void ArchiveJob::SetSynchronousTarget(boost::shared_ptr& target) { @@ -798,7 +812,9 @@ { throw OrthancException(ErrorCode_NullPointer); } - else if (synchronousTarget_.get() != NULL) + else if (writer_.get() != NULL || // Already started + synchronousTarget_.get() != NULL || + asynchronousTarget_.get() != NULL) { throw OrthancException(ErrorCode_BadSequenceOfCalls); } @@ -809,15 +825,30 @@ } + void ArchiveJob::SetDescription(const std::string& description) + { + if (writer_.get() != NULL) // Already started + { + throw OrthancException(ErrorCode_BadSequenceOfCalls); + } + else + { + description_ = description; + } + } + + void ArchiveJob::AddResource(const std::string& publicId) { if (writer_.get() != NULL) // Already started { throw OrthancException(ErrorCode_BadSequenceOfCalls); } - - ResourceIdentifiers resource(context_.GetIndex(), publicId); - archive_->Add(context_.GetIndex(), resource); + else + { + ResourceIdentifiers resource(context_.GetIndex(), publicId); + archive_->Add(context_.GetIndex(), resource); + } } @@ -830,9 +861,16 @@ void ArchiveJob::Start() { + TemporaryFile* target = NULL; + if (synchronousTarget_.get() == NULL) { - throw OrthancException(ErrorCode_BadSequenceOfCalls); + asynchronousTarget_.reset(new TemporaryFile); + target = asynchronousTarget_.get(); + } + else + { + target = synchronousTarget_.get(); } if (writer_.get() != NULL) @@ -840,19 +878,59 @@ throw OrthancException(ErrorCode_BadSequenceOfCalls); } - writer_.reset(new ZipWriterIterator(*synchronousTarget_, context_, *archive_, + writer_.reset(new ZipWriterIterator(*target, context_, *archive_, isMedia_, enableExtendedSopClass_)); instancesCount_ = writer_->GetInstancesCount(); uncompressedSize_ = writer_->GetUncompressedSize(); } + + + namespace + { + class DynamicTemporaryFile : public IDynamicObject + { + private: + std::auto_ptr file_; + + public: + DynamicTemporaryFile(TemporaryFile* f) : file_(f) + { + if (f == NULL) + { + throw OrthancException(ErrorCode_NullPointer); + } + } + + const TemporaryFile& GetFile() const + { + assert(file_.get() != NULL); + return *file_; + } + }; + } + + void ArchiveJob::FinalizeTarget() + { + writer_.reset(); // Flush all the results + + if (asynchronousTarget_.get() != NULL) + { + // Asynchronous behavior: Move the resulting file into the media archive + mediaArchiveId_ = context_.GetMediaArchive().Add( + new DynamicTemporaryFile(asynchronousTarget_.release())); + } + } + + JobStepResult ArchiveJob::Step() { assert(writer_.get() != NULL); - if (synchronousTarget_.unique()) + if (synchronousTarget_.get() != NULL && + synchronousTarget_.unique()) { LOG(WARNING) << "A client has disconnected while creating an archive"; return JobStepResult::Failure(ErrorCode_NetworkProtocol); @@ -860,7 +938,7 @@ if (writer_->GetStepsCount() == 0) { - writer_.reset(); // Flush all the results + FinalizeTarget(); return JobStepResult::Success(); } else @@ -871,7 +949,7 @@ if (currentStep_ == writer_->GetStepsCount()) { - writer_.reset(); // Flush all the results + FinalizeTarget(); return JobStepResult::Success(); } else @@ -909,12 +987,41 @@ } } - + void ArchiveJob::GetPublicContent(Json::Value& value) { - value["Description"] = description_; - value["InstancesCount"] = instancesCount_; - value["UncompressedSizeMB"] = + value = Json::objectValue; + value[KEY_DESCRIPTION] = description_; + value[KEY_INSTANCES_COUNT] = instancesCount_; + value[KEY_UNCOMPRESSED_SIZE_MB] = static_cast(uncompressedSize_ / MEGA_BYTES); } + + + bool ArchiveJob::GetOutput(std::string& output, + MimeType& mime, + const std::string& key) + { + if (key == "archive" && + !mediaArchiveId_.empty()) + { + SharedArchive::Accessor accessor(context_.GetMediaArchive(), mediaArchiveId_); + + if (accessor.IsValid()) + { + const DynamicTemporaryFile& f = dynamic_cast(accessor.GetItem()); + f.GetFile().Read(output); + mime = MimeType_Zip; + return true; + } + else + { + return false; + } + } + else + { + return false; + } + } } diff -r e62e296a5714 -r ee0b2c5ad49b OrthancServer/ServerJobs/ArchiveJob.h --- a/OrthancServer/ServerJobs/ArchiveJob.h Thu Dec 06 11:16:23 2018 +0100 +++ b/OrthancServer/ServerJobs/ArchiveJob.h Thu Dec 06 12:27:30 2018 +0100 @@ -51,6 +51,7 @@ class ZipWriterIterator; boost::shared_ptr synchronousTarget_; + std::auto_ptr asynchronousTarget_; ServerContext& context_; boost::shared_ptr archive_; bool isMedia_; @@ -61,18 +62,20 @@ size_t currentStep_; unsigned int instancesCount_; uint64_t uncompressedSize_; + std::string mediaArchiveId_; + void FinalizeTarget(); + public: ArchiveJob(ServerContext& context, bool isMedia, bool enableExtendedSopClass); - - void SetSynchronousTarget(boost::shared_ptr& synchronousTarget); + + virtual ~ArchiveJob(); - void SetDescription(const std::string& description) - { - description_ = description; - } + void SetSynchronousTarget(boost::shared_ptr& synchronousTarget); + + void SetDescription(const std::string& description); const std::string& GetDescription() const { @@ -101,5 +104,9 @@ { return false; // Cannot serialize this kind of job } + + virtual bool GetOutput(std::string& output, + MimeType& mime, + const std::string& key); }; } diff -r e62e296a5714 -r ee0b2c5ad49b Plugins/Engine/PluginsJob.h --- a/Plugins/Engine/PluginsJob.h Thu Dec 06 11:16:23 2018 +0100 +++ b/Plugins/Engine/PluginsJob.h Thu Dec 06 12:27:30 2018 +0100 @@ -71,6 +71,14 @@ virtual void GetPublicContent(Json::Value& value); virtual bool Serialize(Json::Value& value); + + virtual bool GetOutput(std::string& output, + MimeType& mime, + const std::string& key) + { + // TODO + return false; + } }; } diff -r e62e296a5714 -r ee0b2c5ad49b UnitTestsSources/MemoryCacheTests.cpp --- a/UnitTestsSources/MemoryCacheTests.cpp Thu Dec 06 11:16:23 2018 +0100 +++ b/UnitTestsSources/MemoryCacheTests.cpp Thu Dec 06 12:27:30 2018 +0100 @@ -261,9 +261,25 @@ for (int i = 1; i < 100; i++) { a.Add(new S("Item " + boost::lexical_cast(i))); + // Continuously protect the two first items - try { Orthanc::SharedArchive::Accessor(a, first); } catch (Orthanc::OrthancException&) {} - try { Orthanc::SharedArchive::Accessor(a, second); } catch (Orthanc::OrthancException&) {} + { + Orthanc::SharedArchive::Accessor accessor(a, first); + ASSERT_TRUE(accessor.IsValid()); + ASSERT_EQ("First item", dynamic_cast(accessor.GetItem()).GetValue()); + } + + { + Orthanc::SharedArchive::Accessor accessor(a, second); + ASSERT_TRUE(accessor.IsValid()); + ASSERT_EQ("Second item", dynamic_cast(accessor.GetItem()).GetValue()); + } + + { + Orthanc::SharedArchive::Accessor accessor(a, "nope"); + ASSERT_FALSE(accessor.IsValid()); + ASSERT_THROW(accessor.GetItem(), Orthanc::OrthancException); + } } std::list i; diff -r e62e296a5714 -r ee0b2c5ad49b UnitTestsSources/MultiThreadingTests.cpp --- a/UnitTestsSources/MultiThreadingTests.cpp Thu Dec 06 11:16:23 2018 +0100 +++ b/UnitTestsSources/MultiThreadingTests.cpp Thu Dec 06 12:27:30 2018 +0100 @@ -143,6 +143,13 @@ { value["hello"] = "world"; } + + virtual bool GetOutput(std::string& output, + MimeType& mime, + const std::string& key) ORTHANC_OVERRIDE + { + return false; + } };