changeset 5302:ad3cd5ec2074

Reduced the memory usage when downloading archives when "ZipLoaderThreads" > 0
author Alain Mazy <am@osimis.io>
date Wed, 24 May 2023 11:44:21 +0200
parents 76dc541c5dda
children 02305a80e864
files NEWS OrthancServer/Sources/ServerJobs/ArchiveJob.cpp
diffstat 2 files changed, 8 insertions(+), 1 deletions(-) [+]
line wrap: on
line diff
--- a/NEWS	Mon May 22 17:58:53 2023 +0200
+++ b/NEWS	Wed May 24 11:44:21 2023 +0200
@@ -13,6 +13,7 @@
 * Fix orphan files remaining in storage when working with MaximumStorageSize
   (https://discourse.orthanc-server.org/t/issue-with-deleting-incoming-dicoms-when-maximumstoragesize-is-reached/3510)
 * When deleting a resource, its parents LastUpdate metadata are now updated.
+* Reduced the memory usage when downloading archives when "ZipLoaderThreads" > 0.
 
 
 Version 1.12.0 (2023-04-14)
--- a/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp	Mon May 22 17:58:53 2023 +0200
+++ b/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp	Wed May 24 11:44:21 2023 +0200
@@ -176,6 +176,7 @@
   class ArchiveJob::ThreadedInstanceLoader : public ArchiveJob::InstanceLoader
   {
     Semaphore                           availableInstancesSemaphore_;
+    Semaphore                           bufferedInstancesSemaphore_;
     std::map<std::string, boost::shared_ptr<std::string> >  availableInstances_;
     boost::mutex                        availableInstancesMutex_;
     SharedMessageQueue                  instancesToPreload_;
@@ -185,7 +186,8 @@
   public:
     ThreadedInstanceLoader(ServerContext& context, size_t threadCount, bool transcode, DicomTransferSyntax transferSyntax)
     : InstanceLoader(context, transcode, transferSyntax),
-      availableInstancesSemaphore_(0)
+      availableInstancesSemaphore_(0),
+      bufferedInstancesSemaphore_(3*threadCount)
     {
       for (size_t i = 0; i < threadCount; i++)
       {
@@ -227,6 +229,9 @@
         {
           return;
         }
+        
+        // wait for the consumers (zip writer), no need to accumulate instances in memory if loaders are faster than writers
+        that->bufferedInstancesSemaphore_.Acquire();
 
         try
         {
@@ -270,6 +275,7 @@
       {
         // wait for an instance to be available but this might not be the one we are waiting for !
         availableInstancesSemaphore_.Acquire();
+        bufferedInstancesSemaphore_.Release(); // unlock the "flow" of loaders
 
         boost::shared_ptr<std::string> dicomContent;
         {