# HG changeset patch
# User Alain Mazy <am@osimis.io>
# Date 1686900393 -7200
# Node ID 79fa77e9fa0da50cc49aa8a856c8dbed096c9dba
# Parent  9504de20d43da94f0b0dd92c07ad22a1be9ba6c3# Parent  b5c502bcaf9970c18a7c332a4ac5abd4dfb8a2fb
merge default -> am-experimental

diff -r 9504de20d43d -r 79fa77e9fa0d NEWS
--- a/NEWS	Thu May 25 17:19:52 2023 +0200
+++ b/NEWS	Fri Jun 16 09:26:33 2023 +0200
@@ -4,7 +4,10 @@
 REST API
 --------
 
+* API version upgraded to 21
 * New URI /instances/{id}/file-until-pixel-data
+* added a route to delete the output of an asynchronous job (right now only for archive jobs):
+  e.g. DELETE /jobs/../archive
 
 Maintenance
 -----------
@@ -18,6 +21,11 @@
 * Fix orphan files remaining in storage when working with MaximumStorageSize
   (https://discourse.orthanc-server.org/t/issue-with-deleting-incoming-dicoms-when-maximumstoragesize-is-reached/3510)
 * When deleting a resource, its parents LastUpdate metadata are now updated.
+* Reduced the memory usage when downloading archives when "ZipLoaderThreads" > 0.
+* Disabled automatic conversion from YBR to RGB when transcoding JPEG to RAW transfer-syntax
+  (https://discourse.orthanc-server.org/t/orthanc-convert-ybr-to-rgb-but-does-not-change-metadata/3533).
+  This might have an impact on the image returned by /dicom-web/studies/../series/../instances/../frames/1; the
+  image format is now consistent with the PhotometricIntepretation DICOM Tag.
 
 * WIP: new dicomWeb Json format for some of the Rest API routes.
 * WIP: new 'include' get arguments for some of the Rest API routes to define
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/Cache/SharedArchive.cpp
--- a/OrthancFramework/Sources/Cache/SharedArchive.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/Cache/SharedArchive.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -102,7 +102,7 @@
 
   std::string SharedArchive::Add(IDynamicObject* obj)
   {
-    boost::mutex::scoped_lock lock(mutex_);
+    boost::recursive_mutex::scoped_lock lock(mutex_);
 
     if (archive_.size() == maxSize_)
     {
@@ -122,7 +122,7 @@
 
   void SharedArchive::Remove(const std::string& id)
   {
-    boost::mutex::scoped_lock lock(mutex_);
+    boost::recursive_mutex::scoped_lock lock(mutex_);
     RemoveInternal(id);      
   }
 
@@ -132,7 +132,7 @@
     items.clear();
 
     {
-      boost::mutex::scoped_lock lock(mutex_);
+      boost::recursive_mutex::scoped_lock lock(mutex_);
 
       for (Archive::const_iterator it = archive_.begin();
            it != archive_.end(); ++it)
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/Cache/SharedArchive.h
--- a/OrthancFramework/Sources/Cache/SharedArchive.h	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/Cache/SharedArchive.h	Fri Jun 16 09:26:33 2023 +0200
@@ -44,9 +44,9 @@
   private:
     typedef std::map<std::string, IDynamicObject*>  Archive;
 
-    size_t         maxSize_;
-    boost::mutex   mutex_;
-    Archive        archive_;
+    size_t                  maxSize_;
+    boost::recursive_mutex  mutex_;
+    Archive                 archive_;
     LeastRecentlyUsedIndex<std::string> lru_;
 
     void RemoveInternal(const std::string& id);
@@ -55,8 +55,8 @@
     class ORTHANC_PUBLIC Accessor : public boost::noncopyable
     {
     private:
-      boost::mutex::scoped_lock  lock_;
-      IDynamicObject*            item_;
+      boost::recursive_mutex::scoped_lock lock_;
+      IDynamicObject*                     item_;
 
     public:
       Accessor(SharedArchive& that,
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/DicomParsing/FromDcmtkBridge.cpp
--- a/OrthancFramework/Sources/DicomParsing/FromDcmtkBridge.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/DicomParsing/FromDcmtkBridge.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -1994,7 +1994,14 @@
         case EVR_OL:  // other long (requires byte-swapping)
 #endif
         {
-          ok = element.putUint32(boost::lexical_cast<Uint32>(*decoded)).good();
+          if (decoded->find('\\') != std::string::npos)
+          {
+            ok = element.putString(decoded->c_str()).good();
+          }
+          else
+          {
+            ok = element.putUint32(boost::lexical_cast<Uint32>(*decoded)).good();
+          }
           break;
         }
 
@@ -2473,7 +2480,7 @@
 
 #if ORTHANC_ENABLE_DCMTK_JPEG == 1
     CLOG(INFO, DICOM) << "Registering JPEG codecs in DCMTK";
-    DJDecoderRegistration::registerCodecs(); 
+    DJDecoderRegistration::registerCodecs(EDC_never);  // disable automatic conversion from YBR to RGB when transcoding JPEG to RAW transfer-syntax (https://discourse.orthanc-server.org/t/orthanc-convert-ybr-to-rgb-but-does-not-change-metadata/3533)
 # if ORTHANC_ENABLE_DCMTK_TRANSCODING == 1
     DJEncoderRegistration::registerCodecs();
 # endif
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/JobsEngine/IJob.h
--- a/OrthancFramework/Sources/JobsEngine/IJob.h	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/JobsEngine/IJob.h	Fri Jun 16 09:26:33 2023 +0200
@@ -62,5 +62,9 @@
                            MimeType& mime,
                            std::string& filename,
                            const std::string& key) = 0;
+
+    // This function can only be called if the job has reached its
+    // "success" state
+    virtual bool DeleteOutput(const std::string& key) = 0;
   };
 }
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/JobsEngine/JobsRegistry.cpp
--- a/OrthancFramework/Sources/JobsEngine/JobsRegistry.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/JobsEngine/JobsRegistry.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -679,6 +679,33 @@
     }
   }
 
+  bool JobsRegistry::DeleteJobOutput(const std::string& job,
+                                     const std::string& key)
+  {
+    boost::mutex::scoped_lock lock(mutex_);
+    CheckInvariants();
+
+    JobsIndex::const_iterator found = jobsIndex_.find(job);
+
+    if (found == jobsIndex_.end())
+    {
+      return false;
+    }
+    else
+    {
+      const JobHandler& handler = *found->second;
+
+      if (handler.GetState() == JobState_Success)
+      {
+        return handler.GetJob().DeleteOutput(key);
+      }
+      else
+      {
+        return false;
+      }
+    }
+  }
+
 
   void JobsRegistry::SubmitInternal(std::string& id,
                                     JobHandler* handler)
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/JobsEngine/JobsRegistry.h
--- a/OrthancFramework/Sources/JobsEngine/JobsRegistry.h	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/JobsEngine/JobsRegistry.h	Fri Jun 16 09:26:33 2023 +0200
@@ -153,6 +153,9 @@
                       const std::string& job,
                       const std::string& key);
 
+    bool DeleteJobOutput(const std::string& job,
+                         const std::string& key);
+
     void Serialize(Json::Value& target);
 
     void Submit(std::string& id,
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.cpp
--- a/OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -447,13 +447,6 @@
     return true;
   }
 
-  bool SequenceOfOperationsJob::GetOutput(std::string& output,
-                                          MimeType& mime,
-                                          std::string& filename,
-                                          const std::string& key)
-  {
-    return false;
-  }
 
   void SequenceOfOperationsJob::AwakeTrailingSleep()
   {
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.h
--- a/OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.h	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/JobsEngine/Operations/SequenceOfOperationsJob.h	Fri Jun 16 09:26:33 2023 +0200
@@ -127,7 +127,15 @@
     virtual bool GetOutput(std::string& output,
                            MimeType& mime,
                            std::string& filename,
-                           const std::string& key) ORTHANC_OVERRIDE;
+                           const std::string& key) ORTHANC_OVERRIDE
+    {
+      return false;
+    }
+
+    virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE
+    {
+      return false;
+    }
 
     void AwakeTrailingSleep();
   };
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.cpp
--- a/OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -269,14 +269,6 @@
     return true;
   }
 
-  bool SetOfCommandsJob::GetOutput(std::string &output,
-                                   MimeType &mime,
-                                   std::string& filename,
-                                   const std::string &key)
-  {
-    return false;
-  }
-
 
   SetOfCommandsJob::SetOfCommandsJob(ICommandUnserializer* unserializer,
                                      const Json::Value& source) :
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.h
--- a/OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.h	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/Sources/JobsEngine/SetOfCommandsJob.h	Fri Jun 16 09:26:33 2023 +0200
@@ -106,6 +106,14 @@
     virtual bool GetOutput(std::string& output,
                            MimeType& mime,
                            std::string& filename,
-                           const std::string& key) ORTHANC_OVERRIDE;
+                           const std::string& key) ORTHANC_OVERRIDE
+    {
+      return false;
+    }
+
+    virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE
+    {
+      return false;
+    }
   };
 }
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancFramework/UnitTestsSources/JobsTests.cpp
--- a/OrthancFramework/UnitTestsSources/JobsTests.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancFramework/UnitTestsSources/JobsTests.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -131,6 +131,11 @@
     {
       return false;
     }
+
+    virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE
+    {
+      return false;
+    }
   };
 
 
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancServer/Plugins/Engine/PluginsJob.h
--- a/OrthancServer/Plugins/Engine/PluginsJob.h	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancServer/Plugins/Engine/PluginsJob.h	Fri Jun 16 09:26:33 2023 +0200
@@ -76,6 +76,12 @@
       // TODO
       return false;
     }
+
+    virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE
+    {
+      // TODO
+      return false;
+    }
   };
 }
 
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestSystem.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -770,6 +770,36 @@
   }
 
 
+  static void DeleteJobOutput(RestApiDeleteCall& call)
+  {
+    if (call.IsDocumentation())
+    {
+      call.GetDocumentation()
+        .SetTag("Jobs")
+        .SetSummary("Delete a job output")
+        .SetDescription("Delete the output produced by a job. As of Orthanc 1.12.1, only the jobs that generate a "
+                        "DICOMDIR media or a ZIP archive provide such an output (with `key` equals to `archive`).")
+        .SetUriArgument("id", "Identifier of the job of interest")
+        .SetUriArgument("key", "Name of the output of interest");
+      return;
+    }
+
+    std::string job = call.GetUriComponent("id", "");
+    std::string key = call.GetUriComponent("key", "");
+
+    if (OrthancRestApi::GetContext(call).GetJobsEngine().
+        GetRegistry().DeleteJobOutput(job, key))
+    {
+      call.GetOutput().AnswerBuffer("", MimeType_PlainText);
+    }
+    else
+    {
+      throw OrthancException(ErrorCode_InexistentItem,
+                             "Job has no such output: " + key);
+    }
+  }
+
+
   enum JobAction
   {
     JobAction_Cancel,
@@ -1113,6 +1143,7 @@
     Register("/jobs/{id}/resubmit", ApplyJobAction<JobAction_Resubmit>);
     Register("/jobs/{id}/resume", ApplyJobAction<JobAction_Resume>);
     Register("/jobs/{id}/{key}", GetJobOutput);
+    Register("/jobs/{id}/{key}", DeleteJobOutput);
 
     // New in Orthanc 1.9.0
     Register("/tools/accepted-transfer-syntaxes", GetAcceptedTransferSyntaxes);
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancServer/Sources/ServerJobs/ArchiveJob.cpp
--- a/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -176,6 +176,7 @@
   class ArchiveJob::ThreadedInstanceLoader : public ArchiveJob::InstanceLoader
   {
     Semaphore                           availableInstancesSemaphore_;
+    Semaphore                           bufferedInstancesSemaphore_;
     std::map<std::string, boost::shared_ptr<std::string> >  availableInstances_;
     boost::mutex                        availableInstancesMutex_;
     SharedMessageQueue                  instancesToPreload_;
@@ -185,7 +186,8 @@
   public:
     ThreadedInstanceLoader(ServerContext& context, size_t threadCount, bool transcode, DicomTransferSyntax transferSyntax)
     : InstanceLoader(context, transcode, transferSyntax),
-      availableInstancesSemaphore_(0)
+      availableInstancesSemaphore_(0),
+      bufferedInstancesSemaphore_(3*threadCount)
     {
       for (size_t i = 0; i < threadCount; i++)
       {
@@ -227,6 +229,9 @@
         {
           return;
         }
+        
+        // wait for the consumers (zip writer), no need to accumulate instances in memory if loaders are faster than writers
+        that->bufferedInstancesSemaphore_.Acquire();
 
         try
         {
@@ -270,6 +275,7 @@
       {
         // wait for an instance to be available but this might not be the one we are waiting for !
         availableInstancesSemaphore_.Acquire();
+        bufferedInstancesSemaphore_.Release(); // unlock the "flow" of loaders
 
         boost::shared_ptr<std::string> dicomContent;
         {
@@ -1453,4 +1459,27 @@
       return false;
     }
   }
+
+  bool ArchiveJob::DeleteOutput(const std::string& key)
+  {   
+    if (key == "archive" &&
+        !mediaArchiveId_.empty())
+    {
+      SharedArchive::Accessor accessor(context_.GetMediaArchive(), mediaArchiveId_);
+
+      if (accessor.IsValid())
+      {
+        context_.GetMediaArchive().Remove(mediaArchiveId_);
+        return true;
+      }
+      else
+      {
+        return false;
+      }
+    }    
+    else
+    {
+      return false;
+    }
+  }
 }
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancServer/Sources/ServerJobs/ArchiveJob.h
--- a/OrthancServer/Sources/ServerJobs/ArchiveJob.h	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancServer/Sources/ServerJobs/ArchiveJob.h	Fri Jun 16 09:26:33 2023 +0200
@@ -120,5 +120,7 @@
                            MimeType& mime,
                            std::string& filename,
                            const std::string& key) ORTHANC_OVERRIDE;
+
+    virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE;
   };
 }
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.cpp
--- a/OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -295,15 +295,6 @@
   }
 
 
-  bool ThreadedSetOfInstancesJob::GetOutput(std::string &output,
-                                            MimeType &mime,
-                                            std::string& filename,
-                                            const std::string &key)
-  {
-    return false;
-  }
-
-
   size_t ThreadedSetOfInstancesJob::GetInstancesCount() const
   {
     boost::recursive_mutex::scoped_lock lock(mutex_);
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.h
--- a/OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.h	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancServer/Sources/ServerJobs/ThreadedSetOfInstancesJob.h	Fri Jun 16 09:26:33 2023 +0200
@@ -152,7 +152,15 @@
     virtual bool GetOutput(std::string& output,
                            MimeType& mime,
                            std::string& filename,
-                           const std::string& key) ORTHANC_OVERRIDE;
+                           const std::string& key) ORTHANC_OVERRIDE
+    {
+      return false;
+    }
+
+    virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE
+    {
+      return false;
+    }
 
     bool IsFailedInstance(const std::string& instance) const;
 
diff -r 9504de20d43d -r 79fa77e9fa0d OrthancServer/UnitTestsSources/ServerJobsTests.cpp
--- a/OrthancServer/UnitTestsSources/ServerJobsTests.cpp	Thu May 25 17:19:52 2023 +0200
+++ b/OrthancServer/UnitTestsSources/ServerJobsTests.cpp	Fri Jun 16 09:26:33 2023 +0200
@@ -134,6 +134,11 @@
     {
       return false;
     }
+
+    virtual bool DeleteOutput(const std::string& key) ORTHANC_OVERRIDE
+    {
+      return false;
+    }
   };
 
 
diff -r 9504de20d43d -r 79fa77e9fa0d TODO
--- a/TODO	Thu May 25 17:19:52 2023 +0200
+++ b/TODO	Fri Jun 16 09:26:33 2023 +0200
@@ -99,6 +99,8 @@
   https://groups.google.com/g/orthanc-users/c/y3-xa_GcdLM/m/m0Kr5G5UPAAJ
 * (1) Specify the transfer syntax in /tools/create-dicom
   https://groups.google.com/g/orthanc-users/c/o15Dekecgds/m/xmPE2y3bAwAJ
+* Support Palette PNG in /tools/create-dicom:
+  https://discourse.orthanc-server.org/t/404-on-tools-create-dicom-endpoint-with-specific-png/3562
 * (1) In the /studies/{id}/anonymize route, add an option to remove
   secondary captures.  They usually contains Patient info in the
   image. The SOPClassUID might be used to identify such secondary
@@ -186,6 +188,8 @@
     - ModalitiesInStudy
     - all computed counters at series/study/patient level
     - RequestAttributesSequence (sequence that must be included in all DicomWeb QIDO-RS for series)
+* Investigate increase of CPU consumption while idle after anonymize/delete
+  (https://discourse.orthanc-server.org/t/onchange-callbacks-and-cpu-loads/3534)
 
 * Long-shot & not sure it is even feasible at all: try to reduce memory usage by implementing streaming
   when receiving DICOM instances from the Rest API or from DICOM and store files directly to disk as they
@@ -322,6 +326,8 @@
 * Add more complex testing scenarios like data-migration, change of 
   configuration files, multiple orthanc interacting togethers with various 
   config.  This should probably look like the python toolbox tests ...
+  - add a test to validate Modalities and Peers stored in DB are not lost 
+    while upgrading from one version to the other (Sylvain)
   
 
 ---------------------