changeset 6942:768842647d82 streaming

new configurations for storage data source
author Alain Mazy <am@orthanc.team>
date Wed, 10 Jun 2026 12:33:54 +0200
parents 461afaee9791
children ccfc89106db3
files NEWS OrthancServer/Resources/Configuration.json OrthancServer/Sources/OrthancConfiguration.cpp OrthancServer/Sources/OrthancConfiguration.h OrthancServer/Sources/ServerContext.cpp
diffstat 5 files changed, 72 insertions(+), 9 deletions(-) [+]
line wrap: on
line diff
--- a/NEWS	Wed Jun 10 11:12:26 2026 +0200
+++ b/NEWS	Wed Jun 10 12:33:54 2026 +0200
@@ -5,7 +5,12 @@
 -------
 
 * Full rewrite of access to storage area and of DICOM parsing/transcoding
-  to improve caching and to limit memory usage.
+  to improve caching and to limit memory usage.  
+  This change brings updated configurations (TODO: review):
+  - "StorageLoaderThreads"
+  - "StorageMemoryCapacity"
+  And new metrics:
+  - TODO list
 * New values for "OverwriteInstances" configuration. In previous versions, this configuration
   was a Boolean and it is now a string whose allowed values are "Never", "Always",
   and "IfChanged". For backward compatibility, you may still provide a Boolean value
--- a/OrthancServer/Resources/Configuration.json	Wed Jun 10 11:12:26 2026 +0200
+++ b/OrthancServer/Resources/Configuration.json	Wed Jun 10 12:33:54 2026 +0200
@@ -64,6 +64,16 @@
   // is disabled.  (new in Orthanc 1.10.0)
   "MaximumStorageCacheSize" : 128,
 
+  // Amount of RAM that are allocated to the storage loader threads.
+  // Note that this limit can be overpassed when a single file 
+  // exceeds this size.
+  // (new in Orthanc 1.12.11+)
+  "StorageMemoryCapacity": 256,
+
+  // Number of threads that are used to read files from the storage.
+  // (new in Orthanc 1.12.11+)
+  "StorageLoaderThreads": 4,
+
   // List of paths to the custom Lua scripts that are to be loaded
   // into this instance of Orthanc
   "LuaScripts" : [
--- a/OrthancServer/Sources/OrthancConfiguration.cpp	Wed Jun 10 11:12:26 2026 +0200
+++ b/OrthancServer/Sources/OrthancConfiguration.cpp	Wed Jun 10 12:33:54 2026 +0200
@@ -49,6 +49,8 @@
 static const char* const DICOM_LOSSY_TRANSCODING_QUALITY = "DicomLossyTranscodingQuality";
 static const char* const CONFIG_LOADER_THREADS = "LoaderThreads";
 static const char* const CONFIG_ZIP_LOADER_THREADS = "ZipLoaderThreads"; // for backward compatibility only
+static const char* const CONFIG_CONCURRENT_JOBS = "ConcurrentJobs";
+
 
 namespace Orthanc
 {
@@ -1364,4 +1366,9 @@
       return loaderThreads;
     }
   }
+
+  unsigned int OrthancConfiguration::GetConcurrentJobs() const
+  {
+    return GetUnsignedIntegerParameter(CONFIG_CONCURRENT_JOBS, 2);
+  }
 }
--- a/OrthancServer/Sources/OrthancConfiguration.h	Wed Jun 10 11:12:26 2026 +0200
+++ b/OrthancServer/Sources/OrthancConfiguration.h	Wed Jun 10 12:33:54 2026 +0200
@@ -269,6 +269,8 @@
 
     unsigned int GetLoaderThreads() const;
 
+    unsigned int GetConcurrentJobs() const;
+
     void Format(std::string& result) const;
     
     void SetDefaultEncoding(Encoding encoding);
--- a/OrthancServer/Sources/ServerContext.cpp	Wed Jun 10 11:12:26 2026 +0200
+++ b/OrthancServer/Sources/ServerContext.cpp	Wed Jun 10 12:33:54 2026 +0200
@@ -63,6 +63,7 @@
 #include <dcmtk/dcmdata/dcuid.h>        /* for variable dcmAllStorageSOPClassUIDs */
 
 #include <boost/regex.hpp>
+#include <math.h>
 
 #if HAVE_MALLOC_TRIM == 1
 #  include <malloc.h>
@@ -76,8 +77,6 @@
 static const char* const METRICS_STORAGE_AREA_REMOVE_DURATION = "orthanc_storage_remove_duration_ms";
 static const char* const METRICS_STORAGE_AREA_WRITTEN_BYTES = "orthanc_storage_written_bytes";
 
-
-
 static const char* const METRICS_STORAGE_AREA_CACHE_COUNT = "orthanc_storage_cache_count";
 static const char* const METRICS_STORAGE_AREA_CACHE_HIT_COUNT = "orthanc_storage_cache_hit_count";
 static const char* const METRICS_STORAGE_AREA_CACHE_MISS_COUNT = "orthanc_storage_cache_miss_count";
@@ -105,6 +104,8 @@
 static const char* const METRICS_TRANSCODER_MEMORY_MAX_USAGE_MB = "orthanc_transcoder_memory_max_usage_mb";
 static const char* const METRICS_TRANSCODER_MEMORY_COUNT = "orthanc_transcoder_memory_count";
 
+static const char* const CONFIG_STORAGE_LOADER_THREADS = "StorageLoaderThreads";
+static const char* const CONFIG_STORAGE_MEMORY_CAPACITY = "StorageMemoryCapacity";
 
 
 /**
@@ -538,7 +539,7 @@
         mediaArchive_.reset(
           new SharedArchive(lock.GetConfiguration().GetUnsignedIntegerParameter("MediaArchiveSize", 1)));
         defaultLocalAet_ = lock.GetConfiguration().GetOrthancAET();
-        jobsEngine_.SetWorkersCount(lock.GetConfiguration().GetUnsignedIntegerParameter("ConcurrentJobs", 2));
+        jobsEngine_.SetWorkersCount(lock.GetConfiguration().GetConcurrentJobs());
 
         saveJobs_ = lock.GetConfiguration().GetBooleanParameter("SaveJobs", true);
         if (readOnly_ && saveJobs_)
@@ -672,10 +673,50 @@
       LOG(INFO) << "Your platform does not support malloc_trim(), not starting the memory trimming thread";
 #endif
 
+      unsigned int storageLoaderThreads;
+      size_t storageCacheSizeMb;
+      unsigned int storageMemoryCapacityMb;
+
+      {
+        OrthancConfiguration::ReaderLock lock;
+        unsigned int loaderThreads = lock.GetConfiguration().GetLoaderThreads();
+        unsigned int concurrentJobs = lock.GetConfiguration().GetConcurrentJobs();
+        storageCacheSizeMb = lock.GetConfiguration().GetMaximumStorageCacheSize();
+
+        if (!lock.GetConfiguration().LookupUnsignedIntegerParameter(storageLoaderThreads, CONFIG_STORAGE_LOADER_THREADS))
+        {
+          storageLoaderThreads = loaderThreads * concurrentJobs;
+          if (storageLoaderThreads < 4)
+          {
+            storageLoaderThreads = 4;
+            LOG(WARNING) << "'" << CONFIG_STORAGE_LOADER_THREADS << "' is not defined in your configuration, setting it to " << storageLoaderThreads;
+          }
+          else
+          {
+            storageLoaderThreads = std::min(50u, storageLoaderThreads);
+            LOG(WARNING) << "'" << CONFIG_STORAGE_LOADER_THREADS << "' is not defined in your configuration, setting it to " << storageLoaderThreads << ", based on 'ConcurrentJobs' and the old 'LoaderThreads' configurations.";
+          }
+        }
+        else
+        {
+          LOG(WARNING) << "'" << CONFIG_STORAGE_LOADER_THREADS << "' is set to " << storageLoaderThreads;
+        }
+
+        if (!lock.GetConfiguration().LookupUnsignedIntegerParameter(storageMemoryCapacityMb, CONFIG_STORAGE_MEMORY_CAPACITY))
+        {
+          storageMemoryCapacityMb = 256;
+          LOG(WARNING) << "'" << CONFIG_STORAGE_MEMORY_CAPACITY << "' is not defined in your configuration, setting it to " << storageMemoryCapacityMb << ".  Depending on the available memory on the system, you might want to adapt this value.";
+        }
+        else
+        {
+          LOG(WARNING) << "'" << CONFIG_STORAGE_MEMORY_CAPACITY << "' is set to " << storageMemoryCapacityMb;
+        }
+      }
+
       // For streaming
       {
         boost::shared_ptr<ThreadPool> pool(new ThreadPool);
-        pool->SetCountThreads(4);  // TODO-Streaming - Parameter
+        pool->SetCountThreads(storageLoaderThreads);  // TODO-Streaming - Parameter
         pool->SetLoggingThreadName("STORAGE");
         pool->SetDequeueTimeout(100);
         pool->Start();
@@ -696,11 +737,9 @@
                                                  METRICS_STORAGE_AREA_MEMORY_COUNT,
                                                  METRICS_STORAGE_AREA_MEMORY_MAX_USAGE_MB));
 
-        //storageAreaReader_->SetCapacity(1);  // To test on highest pressure
-        storageAreaReader_->SetCapacity(256 * MEGABYTE); // TODO-Streaming - Parameter
+        storageAreaReader_->SetCapacity(storageMemoryCapacityMb * MEGABYTE);
 
-        //storageAreaReader_->CreateCache(256 * MEGABYTE); // TODO-Streaming - Parameter
-        storageAreaReader_->CreateCache(1 * GIGABYTE); // TODO-Streaming - Parameter
+        storageAreaReader_->CreateCache(storageCacheSizeMb * GIGABYTE);
       }
 
       {