# HG changeset patch
# User Alain Mazy <am@osimis.io>
# Date 1637655731 -3600
# Node ID 7afbb54bd028dd9e408ea2f8a61db3f4b5e8803b
# Parent  2ca4213fb50acc3b53b58b59eadf4fee35b6fed4# Parent  96ab170294fd633deeefa279512faba54c6c76fd
merge storage-cache

diff -r 2ca4213fb50a -r 7afbb54bd028 NEWS
--- a/NEWS	Tue Nov 23 09:20:59 2021 +0100
+++ b/NEWS	Tue Nov 23 09:22:11 2021 +0100
@@ -1,10 +1,26 @@
 Pending changes in the mainline
 ===============================
 
+General
+-------
+
+* Added a storage cache in RAM to avoid reading the same files multiple times from 
+  the storage.  This greatly improves, among other things, the performance of WADO-RS
+  retrieval of individual frames of multiframe instances.
+* New configuration option "MaximumStorageCacheSize" to configure the size of
+  the new storage cache.
+* New configuration option "ZipLoaderThreads" to configure the number of threads used
+  to read instances from storage when createing a Zip archive/media.
+
+
+Maintenance
+-----------
+
 * Fix handling of option "DeidentifyLogs", notably for tags (0010,0010) and (0010,0020)
-
 * New configuration options:
   - "DicomThreadsCount" to set the number of threads in the embedded DICOM server
+* Fix instances accumulating in DB while their attachments were not stored because of 
+  MaximumStorageSize limit reached with a single patient in DB.
 
 REST API
 --------
@@ -15,21 +31,12 @@
   it raises a 415 error code.
 * Archive jobs response now contains a header Content-Disposition:filename='archive.zip'
   
-
-
-Maintenance
------------
-
-* Fix instances accumulating in DB while their attachments were not stored because of 
-  MaximumStorageSize limit reached with a single patient in DB.
-
 Lua
 ---
 
 * New "ReceivedCStoreInstanceFilter" Lua callback to filter instances received
   through C-Store and return a specific C-Store status code.
 
-
 Plugins
 -------
 
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Resources/CMake/OrthancFrameworkConfiguration.cmake
--- a/OrthancFramework/Resources/CMake/OrthancFrameworkConfiguration.cmake	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Resources/CMake/OrthancFrameworkConfiguration.cmake	Tue Nov 23 09:22:11 2021 +0100
@@ -385,6 +385,7 @@
       ${CMAKE_CURRENT_LIST_DIR}/../../Sources/Compression/HierarchicalZipWriter.cpp
       ${CMAKE_CURRENT_LIST_DIR}/../../Sources/Compression/ZipWriter.cpp
       ${CMAKE_CURRENT_LIST_DIR}/../../Sources/FileStorage/StorageAccessor.cpp
+      ${CMAKE_CURRENT_LIST_DIR}/../../Sources/FileStorage/StorageCache.cpp
       )
   endif()
 endif()
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/Cache/MemoryObjectCache.h
--- a/OrthancFramework/Sources/Cache/MemoryObjectCache.h	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Sources/Cache/MemoryObjectCache.h	Tue Nov 23 09:22:11 2021 +0100
@@ -37,6 +37,9 @@
 
 namespace Orthanc
 {
+  /**
+   *  Note: this class is thread safe
+   **/
   class ORTHANC_PUBLIC MemoryObjectCache : public boost::noncopyable
   {
   private:
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/Cache/MemoryStringCache.cpp
--- a/OrthancFramework/Sources/Cache/MemoryStringCache.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Sources/Cache/MemoryStringCache.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -35,7 +35,12 @@
       content_(content)
     {
     }
-      
+
+    explicit StringValue(const char* buffer, size_t size) :
+      content_(buffer, size)
+    {
+    }
+
     const std::string& GetContent() const
     {
       return content_;
@@ -63,6 +68,13 @@
     cache_.Acquire(key, new StringValue(value));
   }
 
+  void MemoryStringCache::Add(const std::string& key,
+                              const void* buffer,
+                              size_t size)
+  {
+    cache_.Acquire(key, new StringValue(reinterpret_cast<const char*>(buffer), size));
+  }
+
   void MemoryStringCache::Invalidate(const std::string &key)
   {
     cache_.Invalidate(key);
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/Cache/MemoryStringCache.h
--- a/OrthancFramework/Sources/Cache/MemoryStringCache.h	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Sources/Cache/MemoryStringCache.h	Tue Nov 23 09:22:11 2021 +0100
@@ -29,6 +29,8 @@
   /**
    * Facade object around "MemoryObjectCache" that caches a dictionary
    * of strings, using the "fetch/add" paradigm of memcached.
+   * 
+   * Note: this class is thread safe
    **/
   class ORTHANC_PUBLIC MemoryStringCache : public boost::noncopyable
   {
@@ -44,7 +46,11 @@
 
     void Add(const std::string& key,
              const std::string& value);
-    
+
+    void Add(const std::string& key,
+             const void* buffer,
+             size_t size);
+
     void Invalidate(const std::string& key);
 
     bool Fetch(std::string& value,
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/FileStorage/StorageAccessor.cpp
--- a/OrthancFramework/Sources/FileStorage/StorageAccessor.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Sources/FileStorage/StorageAccessor.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -22,7 +22,10 @@
 
 #include "../PrecompiledHeaders.h"
 #include "StorageAccessor.h"
+#include "StorageCache.h"
 
+#include "../Logging.h"
+#include "../StringMemoryBuffer.h"
 #include "../Compatibility.h"
 #include "../Compression/ZlibCompressor.h"
 #include "../MetricsRegistry.h"
@@ -58,14 +61,18 @@
   };
 
 
-  StorageAccessor::StorageAccessor(IStorageArea &area) :
+  StorageAccessor::StorageAccessor(IStorageArea &area, StorageCache& cache) :
     area_(area),
+    cache_(cache),
     metrics_(NULL)
   {
   }
 
-  StorageAccessor::StorageAccessor(IStorageArea &area, MetricsRegistry &metrics) :
+  StorageAccessor::StorageAccessor(IStorageArea &area, 
+                                   StorageCache& cache,
+                                   MetricsRegistry &metrics) :
     area_(area),
+    cache_(cache),
     metrics_(&metrics)
   {
   }
@@ -93,6 +100,8 @@
         MetricsTimer timer(*this, METRICS_CREATE);
 
         area_.Create(uuid, data, size, type);
+        cache_.Add(uuid, type, data, size);
+
         return FileInfo(uuid, type, size, md5);
       }
 
@@ -123,6 +132,7 @@
           }
         }
 
+        cache_.Add(uuid, type, data, size);
         return FileInfo(uuid, type, size, md5,
                         CompressionType_ZlibWithSize, compressed.size(), compressedMD5);
       }
@@ -145,6 +155,13 @@
   void StorageAccessor::Read(std::string& content,
                              const FileInfo& info)
   {
+    if (cache_.Fetch(content, info.GetUuid(), info.GetContentType()))
+    {
+      LOG(INFO) << "Read attachment \"" << info.GetUuid() << "\" "
+                << "content type from cache";
+      return;
+    }
+
     switch (info.GetCompressionType())
     {
       case CompressionType_None:
@@ -152,7 +169,9 @@
         MetricsTimer timer(*this, METRICS_READ);
 
         std::unique_ptr<IMemoryBuffer> buffer(area_.Read(info.GetUuid(), info.GetContentType()));
-        buffer->MoveToString(content);        
+        buffer->MoveToString(content);
+
+        cache_.Add(info.GetUuid(), info.GetContentType(), content);
         break;
       }
 
@@ -168,6 +187,8 @@
         }
 
         zlib.Uncompress(content, compressed->GetData(), compressed->GetSize());
+
+        cache_.Add(info.GetUuid(), info.GetContentType(), content);
         break;
       }
 
@@ -196,6 +217,14 @@
   {
     MetricsTimer timer(*this, METRICS_REMOVE);
     area_.Remove(fileUuid, type);
+
+    cache_.Invalidate(fileUuid, type);
+    
+    // in ReadStartRange, we might have cached only the start of the file -> try to remove it
+    if (type == FileContentType_Dicom)
+    {
+      cache_.Invalidate(fileUuid, FileContentType_DicomUntilPixelData);
+    }
   }
 
   void StorageAccessor::Remove(const FileInfo &info)
@@ -203,15 +232,56 @@
     Remove(info.GetUuid(), info.GetContentType());
   }
 
+  IMemoryBuffer* StorageAccessor::ReadStartRange(const std::string& fileUuid,
+                                                 FileContentType contentType,
+                                                 uint64_t end /* exclusive */,
+                                                 FileContentType startFileContentType)
+  {
+    std::string content;
+    if (cache_.Fetch(content, fileUuid, contentType))
+    {
+      LOG(INFO) << "Read attachment \"" << fileUuid << "\" "
+                << "(range from " << 0 << " to " << end << ") from cache";
+
+      return StringMemoryBuffer::CreateFromCopy(content, 0, end);
+    }
+
+    if (cache_.Fetch(content, fileUuid, startFileContentType))
+    {
+      LOG(INFO) << "Read attachment \"" << fileUuid << "\" "
+                << "(range from " << 0 << " to " << end << ") from cache";
+
+      assert(content.size() == end);
+      return StringMemoryBuffer::CreateFromCopy(content);
+    }
+
+    std::unique_ptr<IMemoryBuffer> buffer(area_.ReadRange(fileUuid, contentType, 0, end));
+
+    // we've read only the first part of the file -> add an entry in the cache
+    // note the uuid is still the uuid of the full file but the type is the type of the start of the file !
+    assert(buffer->GetSize() == end);
+    cache_.Add(fileUuid, startFileContentType, buffer->GetData(), buffer->GetSize());
+    return buffer.release();
+  }
+
+
 #if ORTHANC_ENABLE_CIVETWEB == 1 || ORTHANC_ENABLE_MONGOOSE == 1
   void StorageAccessor::SetupSender(BufferHttpSender& sender,
                                     const FileInfo& info,
                                     const std::string& mime)
   {
+    if (cache_.Fetch(sender.GetBuffer(), info.GetUuid(), info.GetContentType()))
+    {
+      LOG(INFO) << "Read attachment \"" << info.GetUuid() << "\" "
+                << "content type from cache";
+    }
+    else
     {
       MetricsTimer timer(*this, METRICS_READ);
       std::unique_ptr<IMemoryBuffer> buffer(area_.Read(info.GetUuid(), info.GetContentType()));
       buffer->MoveToString(sender.GetBuffer());
+
+      cache_.Add(info.GetUuid(), info.GetContentType(), sender.GetBuffer());
     }
 
     sender.SetContentType(mime);
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/FileStorage/StorageAccessor.h
--- a/OrthancFramework/Sources/FileStorage/StorageAccessor.h	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Sources/FileStorage/StorageAccessor.h	Tue Nov 23 09:22:11 2021 +0100
@@ -54,6 +54,7 @@
 namespace Orthanc
 {
   class MetricsRegistry;
+  class StorageCache;
 
   /**
    * This class handles the compression/decompression of the raw files
@@ -66,6 +67,7 @@
     class MetricsTimer;
 
     IStorageArea&     area_;
+    StorageCache&     cache_;
     MetricsRegistry*  metrics_;
 
 #if ORTHANC_ENABLE_CIVETWEB == 1 || ORTHANC_ENABLE_MONGOOSE == 1
@@ -75,9 +77,11 @@
 #endif
 
   public:
-    explicit StorageAccessor(IStorageArea& area);
+    explicit StorageAccessor(IStorageArea& area,
+                             StorageCache& cache);
 
     StorageAccessor(IStorageArea& area,
+                    StorageCache& cache,
                     MetricsRegistry& metrics);
 
     FileInfo Write(const void* data,
@@ -97,6 +101,11 @@
     void ReadRaw(std::string& content,
                  const FileInfo& info);
 
+    IMemoryBuffer* ReadStartRange(const std::string& fileUuid,
+                                  FileContentType fullFileContentType,
+                                  uint64_t end /* exclusive */,
+                                  FileContentType startFileContentType);
+
     void Remove(const std::string& fileUuid,
                 FileContentType type);
 
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/FileStorage/StorageCache.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/OrthancFramework/Sources/FileStorage/StorageCache.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -0,0 +1,118 @@
+/**
+ * Orthanc - A Lightweight, RESTful DICOM Store
+ * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics
+ * Department, University Hospital of Liege, Belgium
+ * Copyright (C) 2017-2021 Osimis S.A., Belgium
+ *
+ * This program is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program. If not, see
+ * <http://www.gnu.org/licenses/>.
+ **/
+
+
+#include "../PrecompiledHeaders.h"
+#include "StorageCache.h"
+
+#include "../Compatibility.h"
+#include "../OrthancException.h"
+
+
+
+namespace Orthanc
+{
+  bool IsAcceptedContentType(FileContentType contentType)
+  {
+    return contentType == FileContentType_Dicom ||
+      contentType == FileContentType_DicomUntilPixelData ||
+      contentType == FileContentType_DicomAsJson;
+  }
+
+  const char* ToString(FileContentType contentType)
+  {
+    switch (contentType)
+    {
+      case FileContentType_Dicom:
+        return "dicom";
+      case FileContentType_DicomUntilPixelData:
+        return "dicom-header";
+      case FileContentType_DicomAsJson:
+        return "dicom-json";
+      default:
+        throw OrthancException(ErrorCode_InternalError,
+                               "ContentType not supported in StorageCache");
+    }
+  }
+
+  void GetCacheKey(std::string& key, const std::string& uuid, FileContentType contentType)
+  {
+    key = uuid + ":" + std::string(ToString(contentType));
+  }
+  
+  void StorageCache::SetMaximumSize(size_t size)
+  {
+    cache_.SetMaximumSize(size);
+  }
+
+  void StorageCache::Add(const std::string& uuid, 
+                         FileContentType contentType,
+                         const std::string& value)
+  {
+    if (!IsAcceptedContentType(contentType))
+    {
+      return;
+    }
+
+    std::string key;
+    GetCacheKey(key, uuid, contentType);
+    cache_.Add(key, value);
+  }
+
+  void StorageCache::Add(const std::string& uuid, 
+                         FileContentType contentType,
+                         const void* buffer,
+                         size_t size)
+  {
+    if (!IsAcceptedContentType(contentType))
+    {
+      return;
+    }
+    
+    std::string key;
+    GetCacheKey(key, uuid, contentType);
+    cache_.Add(key, buffer, size);
+  }
+
+  void StorageCache::Invalidate(const std::string& uuid, FileContentType contentType)
+  {
+    std::string key;
+    GetCacheKey(key, uuid, contentType);
+    cache_.Invalidate(key);
+  }
+
+  bool StorageCache::Fetch(std::string& value, 
+                           const std::string& uuid,
+                           FileContentType contentType)
+  {
+    if (!IsAcceptedContentType(contentType))
+    {
+      return false;
+    }
+
+    std::string key;
+    GetCacheKey(key, uuid, contentType);
+
+    return cache_.Fetch(value, key);
+  }
+
+
+}
\ No newline at end of file
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/FileStorage/StorageCache.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/OrthancFramework/Sources/FileStorage/StorageCache.h	Tue Nov 23 09:22:11 2021 +0100
@@ -0,0 +1,59 @@
+/**
+ * Orthanc - A Lightweight, RESTful DICOM Store
+ * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics
+ * Department, University Hospital of Liege, Belgium
+ * Copyright (C) 2017-2021 Osimis S.A., Belgium
+ *
+ * This program is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program. If not, see
+ * <http://www.gnu.org/licenses/>.
+ **/
+
+
+#pragma once
+
+#include "../Cache/MemoryStringCache.h"
+
+#include "../Compatibility.h"  // For ORTHANC_OVERRIDE
+
+#include <boost/thread/mutex.hpp>
+#include <map>
+
+namespace Orthanc
+{
+   /**
+   *  Note: this class is thread safe
+   **/
+   class ORTHANC_PUBLIC StorageCache : public boost::noncopyable
+    {
+      MemoryStringCache   cache_;
+    public:
+      void SetMaximumSize(size_t size);
+
+      void Add(const std::string& uuid, 
+               FileContentType contentType,
+               const std::string& value);
+
+      void Add(const std::string& uuid, 
+               FileContentType contentType,
+               const void* buffer,
+               size_t size);
+
+      void Invalidate(const std::string& uuid, FileContentType contentType);
+
+      bool Fetch(std::string& value, 
+                 const std::string& uuid,
+                 FileContentType contentType);
+
+    };
+}
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/MultiThreading/Semaphore.cpp
--- a/OrthancFramework/Sources/MultiThreading/Semaphore.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Sources/MultiThreading/Semaphore.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -31,10 +31,6 @@
   Semaphore::Semaphore(unsigned int availableResources) :
     availableResources_(availableResources)
   {
-    if (availableResources_ == 0)
-    {
-      throw OrthancException(ErrorCode_ParameterOutOfRange);
-    }
   }
 
   unsigned int Semaphore::GetAvailableResourcesCount() const
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/MultiThreading/Semaphore.h
--- a/OrthancFramework/Sources/MultiThreading/Semaphore.h	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Sources/MultiThreading/Semaphore.h	Tue Nov 23 09:22:11 2021 +0100
@@ -36,16 +36,16 @@
     boost::mutex mutex_;
     boost::condition_variable condition_;
 
+  public:
+    explicit Semaphore(unsigned int availableResources);
+
+    unsigned int GetAvailableResourcesCount() const;
+
     void Release(unsigned int resourceCount = 1);
 
     void Acquire(unsigned int resourceCount = 1);
 
     bool TryAcquire(unsigned int resourceCount = 1);
-  public:
-    explicit Semaphore(unsigned int availableResources);
-
-    unsigned int GetAvailableResourcesCount() const;
-
 
     class Locker : public boost::noncopyable
     {
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/StringMemoryBuffer.cpp
--- a/OrthancFramework/Sources/StringMemoryBuffer.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Sources/StringMemoryBuffer.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -47,4 +47,14 @@
     result->Copy(buffer);
     return result.release();
   }
+
+
+  IMemoryBuffer* StringMemoryBuffer::CreateFromCopy(const std::string& buffer, 
+                                                    size_t start /* inclusive */, 
+                                                    size_t end /* exclusive */)
+  {
+    std::unique_ptr<StringMemoryBuffer> result(new StringMemoryBuffer);
+    result->Copy(buffer, start, end);
+    return result.release();
+  }
 }
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/Sources/StringMemoryBuffer.h
--- a/OrthancFramework/Sources/StringMemoryBuffer.h	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/Sources/StringMemoryBuffer.h	Tue Nov 23 09:22:11 2021 +0100
@@ -38,6 +38,11 @@
       buffer_ = buffer;
     }
 
+    void Copy(const std::string& buffer, size_t start /* inclusive */, size_t end /* exclusive */)
+    {
+      buffer_.assign(buffer, start, end - start);
+    }
+
     void Swap(std::string& buffer)
     {
       buffer_.swap(buffer);
@@ -58,5 +63,7 @@
     static IMemoryBuffer* CreateFromSwap(std::string& buffer);
 
     static IMemoryBuffer* CreateFromCopy(const std::string& buffer);
+
+    static IMemoryBuffer* CreateFromCopy(const std::string& buffer, size_t start /* inclusive */, size_t end /* exclusive */);
   };
 }
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancFramework/UnitTestsSources/FileStorageTests.cpp
--- a/OrthancFramework/UnitTestsSources/FileStorageTests.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancFramework/UnitTestsSources/FileStorageTests.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -29,6 +29,7 @@
 
 #include "../Sources/FileStorage/FilesystemStorage.h"
 #include "../Sources/FileStorage/StorageAccessor.h"
+#include "../Sources/FileStorage/StorageCache.h"
 #include "../Sources/HttpServer/BufferHttpSender.h"
 #include "../Sources/HttpServer/FilesystemHttpSender.h"
 #include "../Sources/Logging.h"
@@ -124,7 +125,8 @@
 TEST(StorageAccessor, NoCompression)
 {
   FilesystemStorage s("UnitTestsStorage");
-  StorageAccessor accessor(s);
+  StorageCache cache;
+  StorageAccessor accessor(s, cache);
 
   std::string data = "Hello world";
   FileInfo info = accessor.Write(data, FileContentType_Dicom, CompressionType_None, true);
@@ -145,7 +147,8 @@
 TEST(StorageAccessor, Compression)
 {
   FilesystemStorage s("UnitTestsStorage");
-  StorageAccessor accessor(s);
+  StorageCache cache;
+  StorageAccessor accessor(s, cache);
 
   std::string data = "Hello world";
   FileInfo info = accessor.Write(data, FileContentType_Dicom, CompressionType_ZlibWithSize, true);
@@ -165,7 +168,8 @@
 TEST(StorageAccessor, Mix)
 {
   FilesystemStorage s("UnitTestsStorage");
-  StorageAccessor accessor(s);
+  StorageCache cache;
+  StorageAccessor accessor(s, cache);
 
   std::string r;
   std::string compressedData = "Hello";
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancServer/Resources/Configuration.json
--- a/OrthancServer/Resources/Configuration.json	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancServer/Resources/Configuration.json	Tue Nov 23 09:22:11 2021 +0100
@@ -40,7 +40,13 @@
   // in the storage (a value of "0" indicates no limit on the number
   // of patients)
   "MaximumPatientCount" : 0,
-  
+
+  // Maximum size of the storage cache in MB.  The storage cache
+  // is stored in RAM and contains a copy of recently accessed
+  // files (written or read).  A value of "0" indicates the cache
+  // is disabled.  (new in Orthanc 1.9.8)
+  "MaximumStorageCacheSize" : 128,
+
   // List of paths to the custom Lua scripts that are to be loaded
   // into this instance of Orthanc
   "LuaScripts" : [
@@ -837,5 +843,12 @@
   // disk space and might lead to HTTP timeouts on large archives). If
   // set to "true", the chunks of the ZIP file are progressively sent
   // as soon as one DICOM file gets compressed (new in Orthanc 1.9.4)
-  "SynchronousZipStream" : true
+  "SynchronousZipStream" : true,
+
+  // Default number of loader threads when generating Zip archive/media.
+  // A value of 0 means reading and writing are performed in sequence
+  // (default behaviour).  A value > 1 is meaningful only if the storage
+  // is a distributed network storage (e.g object storage plugin).
+  // (new in Orthanc 1.9.8)
+  "ZipLoaderThreads": 0
 }
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancServer/Sources/OrthancRestApi/OrthancRestArchive.cpp
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestArchive.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestArchive.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -51,7 +51,9 @@
   static const char* const KEY_RESOURCES = "Resources";
   static const char* const KEY_EXTENDED = "Extended";
   static const char* const KEY_TRANSCODE = "Transcode";
-  
+
+  static const char* const CONFIG_LOADER_THREADS = "ZipLoaderThreads";
+
   static void AddResourcesOfInterestFromArray(ArchiveJob& job,
                                               const Json::Value& resources)
   {
@@ -123,6 +125,7 @@
                                bool& transcode,              /* out */
                                DicomTransferSyntax& syntax,  /* out */
                                int& priority,                /* out */
+                               unsigned int& loaderThreads,  /* out */
                                const Json::Value& body,      /* in */
                                const bool defaultExtended    /* in */)
   {
@@ -151,6 +154,12 @@
     {
       transcode = false;
     }
+
+    {
+      OrthancConfiguration::ReaderLock lock;
+      loaderThreads = lock.GetConfiguration().GetUnsignedIntegerParameter(CONFIG_LOADER_THREADS, 0);  // New in Orthanc 1.9.8
+    }
+   
   }
 
 
@@ -554,8 +563,9 @@
       bool synchronous, extended, transcode;
       DicomTransferSyntax transferSyntax;
       int priority;
+      unsigned int loaderThreads;
       GetJobParameters(synchronous, extended, transcode, transferSyntax,
-                       priority, body, DEFAULT_IS_EXTENDED);
+                       priority, loaderThreads, body, DEFAULT_IS_EXTENDED);
       
       std::unique_ptr<ArchiveJob> job(new ArchiveJob(context, IS_MEDIA, extended));
       AddResourcesOfInterest(*job, body);
@@ -565,6 +575,8 @@
         job->SetTranscode(transferSyntax);
       }
       
+      job->SetLoaderThreads(loaderThreads);
+
       SubmitJob(call.GetOutput(), context, job, priority, synchronous, "Archive.zip");
     }
     else
@@ -578,6 +590,8 @@
   template <bool IS_MEDIA>
   static void CreateSingleGet(RestApiGetCall& call)
   {
+    static const char* const TRANSCODE = "transcode";
+
     if (call.IsDocumentation())
     {
       ResourceType t = StringToResourceType(call.GetFullUri()[0].c_str());
@@ -591,7 +605,7 @@
                         "which might *not* be desirable to archive large amount of data, as it might "
                         "lead to network timeouts. Prefer the asynchronous version using `POST` method.")
         .SetUriArgument("id", "Orthanc identifier of the " + r + " of interest")
-        .SetHttpGetArgument("transcode", RestApiCallDocumentation::Type_String,
+        .SetHttpGetArgument(TRANSCODE, RestApiCallDocumentation::Type_String,
                             "If present, the DICOM files in the archive will be transcoded to the provided "
                             "transfer syntax: https://book.orthanc-server.com/faq/transcoding.html", false)
         .AddAnswerType(MimeType_Zip, "ZIP file containing the archive");
@@ -621,12 +635,17 @@
     std::unique_ptr<ArchiveJob> job(new ArchiveJob(context, IS_MEDIA, extended));
     job->AddResource(id);
 
-    static const char* const TRANSCODE = "transcode";
     if (call.HasArgument(TRANSCODE))
     {
       job->SetTranscode(GetTransferSyntax(call.GetArgument(TRANSCODE, "")));
     }
 
+    {
+      OrthancConfiguration::ReaderLock lock;
+      unsigned int loaderThreads = lock.GetConfiguration().GetUnsignedIntegerParameter(CONFIG_LOADER_THREADS, 0);  // New in Orthanc 1.9.8
+      job->SetLoaderThreads(loaderThreads);
+    }
+
     SubmitJob(call.GetOutput(), context, job, 0 /* priority */,
               true /* synchronous */, id + ".zip");
   }
@@ -660,8 +679,9 @@
       bool synchronous, extended, transcode;
       DicomTransferSyntax transferSyntax;
       int priority;
+      unsigned int loaderThreads;
       GetJobParameters(synchronous, extended, transcode, transferSyntax,
-                       priority, body, false /* by default, not extented */);
+                       priority, loaderThreads, body, false /* by default, not extented */);
       
       std::unique_ptr<ArchiveJob> job(new ArchiveJob(context, IS_MEDIA, extended));
       job->AddResource(id);
@@ -671,6 +691,8 @@
         job->SetTranscode(transferSyntax);
       }
 
+      job->SetLoaderThreads(loaderThreads);
+
       SubmitJob(call.GetOutput(), context, job, priority, synchronous, id + ".zip");
     }
     else
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -2977,7 +2977,7 @@
     std::string publicId = call.GetUriComponent("id", "");
 
     std::string dicomContent;
-    context.ReadDicom(dicomContent, publicId);
+    context.ReadDicomForHeader(dicomContent, publicId);
 
     // TODO Consider using "DicomMap::ParseDicomMetaInformation()" to
     // speed up things here
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancServer/Sources/ServerContext.cpp
--- a/OrthancServer/Sources/ServerContext.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancServer/Sources/ServerContext.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -492,7 +492,7 @@
   void ServerContext::RemoveFile(const std::string& fileUuid,
                                  FileContentType type)
   {
-    StorageAccessor accessor(area_, GetMetricsRegistry());
+    StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
     accessor.Remove(fileUuid, type);
   }
 
@@ -534,7 +534,7 @@
     try
     {
       MetricsRegistry::Timer timer(GetMetricsRegistry(), "orthanc_store_dicom_duration_ms");
-      StorageAccessor accessor(area_, GetMetricsRegistry());
+      StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
 
       DicomInstanceHasher hasher(summary);
       resultPublicId = hasher.HashInstance();
@@ -781,7 +781,7 @@
     }
     else
     {
-      StorageAccessor accessor(area_, GetMetricsRegistry());
+      StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
       accessor.AnswerFile(output, attachment, GetFileContentMime(content));
     }
   }
@@ -811,7 +811,7 @@
 
     std::string content;
 
-    StorageAccessor accessor(area_, GetMetricsRegistry());
+    StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
     accessor.Read(content, attachment);
 
     FileInfo modified = accessor.Write(content.empty() ? NULL : content.c_str(),
@@ -867,7 +867,7 @@
       std::string dicom;
 
       {
-        StorageAccessor accessor(area_, GetMetricsRegistry());
+        StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
         accessor.Read(dicom, attachment);
       }
 
@@ -932,8 +932,8 @@
       
         std::unique_ptr<IMemoryBuffer> dicom;
         {
-          MetricsRegistry::Timer timer(GetMetricsRegistry(), "orthanc_storage_read_range_duration_ms");
-          dicom.reset(area_.ReadRange(attachment.GetUuid(), FileContentType_Dicom, 0, pixelDataOffset));
+          StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
+          dicom.reset(accessor.ReadStartRange(attachment.GetUuid(), FileContentType_Dicom, pixelDataOffset, FileContentType_DicomUntilPixelData));
         }
 
         if (dicom.get() == NULL)
@@ -962,7 +962,7 @@
         std::string dicomAsJson;
 
         {
-          StorageAccessor accessor(area_, GetMetricsRegistry());
+          StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
           accessor.Read(dicomAsJson, attachment);
         }
 
@@ -1031,7 +1031,15 @@
     int64_t revision;
     ReadAttachment(dicom, revision, instancePublicId, FileContentType_Dicom, true /* uncompress */);
   }
-    
+
+  void ServerContext::ReadDicomForHeader(std::string& dicom,
+                                         const std::string& instancePublicId)
+  {
+    if (!ReadDicomUntilPixelData(dicom, instancePublicId))
+    {
+      ReadDicom(dicom, instancePublicId);
+    }
+  }
 
   bool ServerContext::ReadDicomUntilPixelData(std::string& dicom,
                                               const std::string& instancePublicId)
@@ -1060,8 +1068,10 @@
       {
         uint64_t pixelDataOffset = boost::lexical_cast<uint64_t>(s);
 
+        StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
+
         std::unique_ptr<IMemoryBuffer> buffer(
-          area_.ReadRange(attachment.GetUuid(), attachment.GetContentType(), 0, pixelDataOffset));
+          accessor.ReadStartRange(attachment.GetUuid(), attachment.GetContentType(), pixelDataOffset, FileContentType_DicomUntilPixelData));
         buffer->MoveToString(dicom);
         return true;   // Success
       }
@@ -1092,7 +1102,7 @@
     assert(attachment.GetContentType() == content);
 
     {
-      StorageAccessor accessor(area_, GetMetricsRegistry());
+      StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
 
       if (uncompressIfNeeded)
       {
@@ -1192,7 +1202,7 @@
     // TODO Should we use "gzip" instead?
     CompressionType compression = (compressionEnabled_ ? CompressionType_ZlibWithSize : CompressionType_None);
 
-    StorageAccessor accessor(area_, GetMetricsRegistry());
+    StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry());
     FileInfo attachment = accessor.Write(data, size, attachmentType, compression, storeMD5_);
 
     try
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancServer/Sources/ServerContext.h
--- a/OrthancServer/Sources/ServerContext.h	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancServer/Sources/ServerContext.h	Tue Nov 23 09:22:11 2021 +0100
@@ -43,6 +43,7 @@
 #include "../../OrthancFramework/Sources/DicomParsing/DicomModification.h"
 #include "../../OrthancFramework/Sources/DicomParsing/IDicomTranscoder.h"
 #include "../../OrthancFramework/Sources/DicomParsing/ParsedDicomCache.h"
+#include "../../OrthancFramework/Sources/FileStorage/StorageCache.h"
 #include "../../OrthancFramework/Sources/MultiThreading/Semaphore.h"
 
 
@@ -205,6 +206,7 @@
 
     ServerIndex index_;
     IStorageArea& area_;
+    StorageCache storageCache_;
 
     bool compressionEnabled_;
     bool storeMD5_;
@@ -324,6 +326,11 @@
       return index_;
     }
 
+    void SetMaximumStorageCacheSize(size_t size)
+    {
+      return storageCache_.SetMaximumSize(size);
+    }
+
     void SetCompressionEnabled(bool enabled);
 
     bool IsCompressionEnabled() const
@@ -361,7 +368,10 @@
 
     void ReadDicom(std::string& dicom,
                    const std::string& instancePublicId);
-    
+
+    void ReadDicomForHeader(std::string& dicom,
+                            const std::string& instancePublicId);
+
     bool ReadDicomUntilPixelData(std::string& dicom,
                                  const std::string& instancePublicId);
 
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancServer/Sources/ServerJobs/ArchiveJob.cpp
--- a/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -40,6 +40,7 @@
 #include "../../../OrthancFramework/Sources/DicomParsing/FromDcmtkBridge.h"
 #include "../../../OrthancFramework/Sources/Logging.h"
 #include "../../../OrthancFramework/Sources/OrthancException.h"
+#include "../../../OrthancFramework/Sources/MultiThreading/Semaphore.h"
 #include "../OrthancConfiguration.h"
 #include "../ServerContext.h"
 
@@ -88,6 +89,181 @@
   }
 
 
+  class ArchiveJob::InstanceLoader : public boost::noncopyable
+  {
+  protected:
+    ServerContext&                        context_;
+  public:
+    InstanceLoader(ServerContext& context)
+    : context_(context)
+    {
+    }
+
+    virtual ~InstanceLoader()
+    {
+    }
+
+    virtual void PrepareDicom(const std::string& instanceId)
+    {
+
+    }
+
+    virtual void GetDicom(std::string& dicom, const std::string& instanceId) = 0;
+
+    virtual void Clear()
+    {
+    }
+  };
+
+  class ArchiveJob::SynchronousInstanceLoader : public ArchiveJob::InstanceLoader
+  {
+  public:
+    SynchronousInstanceLoader(ServerContext& context)
+    : InstanceLoader(context)
+    {
+    }
+
+    virtual void GetDicom(std::string& dicom, const std::string& instanceId) ORTHANC_OVERRIDE
+    {
+      context_.ReadDicom(dicom, instanceId);
+    }
+  };
+
+  class InstanceId : public Orthanc::IDynamicObject
+  {
+  private:
+    std::string id_;
+
+  public:
+    InstanceId(const std::string& id) : id_(id)
+    {
+    }
+
+    virtual ~InstanceId() ORTHANC_OVERRIDE
+    {
+    }
+
+    std::string GetId() const {return id_;};
+  };
+
+  class ArchiveJob::ThreadedInstanceLoader : public ArchiveJob::InstanceLoader
+  {
+    Semaphore                           availableInstancesSemaphore_;
+    std::map<std::string, boost::shared_ptr<std::string>>  availableInstances_;
+    boost::mutex                        availableInstancesMutex_;
+    SharedMessageQueue                  instancesToPreload_;
+    std::vector<boost::thread*>         threads_;
+
+
+  public:
+    ThreadedInstanceLoader(ServerContext& context, size_t threadCount)
+    : InstanceLoader(context),
+      availableInstancesSemaphore_(0)
+    {
+      for (size_t i = 0; i < threadCount; i++)
+      {
+        threads_.push_back(new boost::thread(PreloaderWorkerThread, this));
+      }
+    }
+
+    virtual ~ThreadedInstanceLoader()
+    {
+      Clear();
+    }
+
+    virtual void Clear() ORTHANC_OVERRIDE
+    {
+      for (size_t i = 0; i < threads_.size(); i++)
+      {
+        instancesToPreload_.Enqueue(NULL);
+      }
+
+      for (size_t i = 0; i < threads_.size(); i++)
+      {
+        if (threads_[i]->joinable())
+        {
+          threads_[i]->join();
+        }
+        delete threads_[i];
+      }
+
+      threads_.clear();
+      availableInstances_.clear();
+    }
+
+    static void PreloaderWorkerThread(ThreadedInstanceLoader* that)
+    {
+      while (true)
+      {
+        std::unique_ptr<InstanceId> instanceId(dynamic_cast<InstanceId*>(that->instancesToPreload_.Dequeue(0)));
+        if (instanceId.get() == NULL)  // that's the signal to exit the thread
+        {
+          return;
+        }
+
+        try
+        {
+          boost::shared_ptr<std::string> dicomContent(new std::string());
+          that->context_.ReadDicom(*dicomContent, instanceId->GetId());
+          {
+            boost::mutex::scoped_lock lock(that->availableInstancesMutex_);
+            that->availableInstances_[instanceId->GetId()] = dicomContent;
+          }
+
+          that->availableInstancesSemaphore_.Release();
+        }
+        catch (OrthancException& e)
+        {
+          boost::mutex::scoped_lock lock(that->availableInstancesMutex_);
+          // store a NULL result to notify that we could not read the instance
+          that->availableInstances_[instanceId->GetId()] = boost::shared_ptr<std::string>(); 
+          that->availableInstancesSemaphore_.Release();
+        }
+      }
+    }
+
+    virtual void PrepareDicom(const std::string& instanceId) ORTHANC_OVERRIDE
+    {
+      instancesToPreload_.Enqueue(new InstanceId(instanceId));
+    }
+
+    virtual void GetDicom(std::string& dicom, const std::string& instanceId) ORTHANC_OVERRIDE
+    {
+      while (true)
+      {
+        // wait for an instance to be available but this might not be the one we are waiting for !
+        availableInstancesSemaphore_.Acquire();
+
+        boost::shared_ptr<std::string> dicomContent;
+        {
+          if (availableInstances_.find(instanceId) != availableInstances_.end())
+          {
+            // this is the instance we were waiting for
+            dicomContent = availableInstances_[instanceId];
+            availableInstances_.erase(instanceId);
+
+            if (dicomContent.get() == NULL)  // there has been an error while reading the file
+            {
+              throw OrthancException(ErrorCode_InexistentItem);
+            }
+            dicom.swap(*dicomContent);
+
+            if (availableInstances_.size() > 0)
+            {
+              // we have just read the instance we were waiting for but there are still other instances available ->
+              // make sure the next GetDicom call does not wait !
+              availableInstancesSemaphore_.Release();
+            }
+            return;
+          }
+          // we have not found the expected instance, simply wait for the next loader thread to signal the semaphore when
+          // a new instance is available
+        }
+      }
+    }
+  };
+
+
   class ArchiveJob::ResourceIdentifiers : public boost::noncopyable
   {
   private:
@@ -402,6 +578,7 @@
         
       void Apply(HierarchicalZipWriter& writer,
                  ServerContext& context,
+                 InstanceLoader& instanceLoader,
                  DicomDirWriter* dicomDir,
                  const std::string& dicomDirFolder,
                  bool transcode,
@@ -423,7 +600,7 @@
 
             try
             {
-              context.ReadDicom(content, instanceId_);
+              instanceLoader.GetDicom(content, instanceId_);
             }
             catch (OrthancException& e)
             {
@@ -494,10 +671,12 @@
     std::deque<Command*>  commands_;
     uint64_t              uncompressedSize_;
     unsigned int          instancesCount_;
+    InstanceLoader&       instanceLoader_;
 
       
     void ApplyInternal(HierarchicalZipWriter& writer,
                        ServerContext& context,
+                       InstanceLoader& instanceLoader,
                        size_t index,
                        DicomDirWriter* dicomDir,
                        const std::string& dicomDirFolder,
@@ -509,13 +688,14 @@
         throw OrthancException(ErrorCode_ParameterOutOfRange);
       }
 
-      commands_[index]->Apply(writer, context, dicomDir, dicomDirFolder, transcode, transferSyntax);
+      commands_[index]->Apply(writer, context, instanceLoader, dicomDir, dicomDirFolder, transcode, transferSyntax);
     }
       
   public:
-    ZipCommands() :
+    ZipCommands(InstanceLoader& instanceLoader) :
       uncompressedSize_(0),
-      instancesCount_(0)
+      instancesCount_(0),
+      instanceLoader_(instanceLoader)
     {
     }
       
@@ -547,23 +727,25 @@
     // "media" flavor (with DICOMDIR)
     void Apply(HierarchicalZipWriter& writer,
                ServerContext& context,
+               InstanceLoader& instanceLoader,
                size_t index,
                DicomDirWriter& dicomDir,
                const std::string& dicomDirFolder,
                bool transcode,
                DicomTransferSyntax transferSyntax) const
     {
-      ApplyInternal(writer, context, index, &dicomDir, dicomDirFolder, transcode, transferSyntax);
+      ApplyInternal(writer, context, instanceLoader, index, &dicomDir, dicomDirFolder, transcode, transferSyntax);
     }
 
     // "archive" flavor (without DICOMDIR)
     void Apply(HierarchicalZipWriter& writer,
                ServerContext& context,
+               InstanceLoader& instanceLoader,
                size_t index,
                bool transcode,
                DicomTransferSyntax transferSyntax) const
     {
-      ApplyInternal(writer, context, index, NULL, "", transcode, transferSyntax);
+      ApplyInternal(writer, context, instanceLoader, index, NULL, "", transcode, transferSyntax);
     }
       
     void AddOpenDirectory(const std::string& filename)
@@ -580,6 +762,7 @@
                           const std::string& instanceId,
                           uint64_t uncompressedSize)
     {
+      instanceLoader_.PrepareDicom(instanceId);
       commands_.push_back(new Command(Type_WriteInstance, filename, instanceId));
       instancesCount_ ++;
       uncompressedSize_ += uncompressedSize;
@@ -747,6 +930,7 @@
   {
   private:
     ServerContext&                          context_;
+    InstanceLoader&                         instanceLoader_;
     ZipCommands                             commands_;
     std::unique_ptr<HierarchicalZipWriter>  zip_;
     std::unique_ptr<DicomDirWriter>         dicomDir_;
@@ -755,10 +939,13 @@
 
   public:
     ZipWriterIterator(ServerContext& context,
+                      InstanceLoader& instanceLoader,
                       ArchiveIndex& archive,
                       bool isMedia,
                       bool enableExtendedSopClass) :
       context_(context),
+      instanceLoader_(instanceLoader),
+      commands_(instanceLoader),
       isMedia_(isMedia),
       isStream_(false)
     {
@@ -882,13 +1069,13 @@
         if (isMedia_)
         {
           assert(dicomDir_.get() != NULL);
-          commands_.Apply(*zip_, context_, index, *dicomDir_,
+          commands_.Apply(*zip_, context_, instanceLoader_, index, *dicomDir_,
                           MEDIA_IMAGES_FOLDER, transcode, transferSyntax);
         }
         else
         {
           assert(dicomDir_.get() == NULL);
-          commands_.Apply(*zip_, context_, index, transcode, transferSyntax);
+          commands_.Apply(*zip_, context_, instanceLoader_, index, transcode, transferSyntax);
         }
       }
     }
@@ -917,7 +1104,8 @@
     uncompressedSize_(0),
     archiveSize_(0),
     transcode_(false),
-    transferSyntax_(DicomTransferSyntax_LittleEndianImplicit)
+    transferSyntax_(DicomTransferSyntax_LittleEndianImplicit),
+    loaderThreads_(0)
   {
   }
 
@@ -993,6 +1181,19 @@
   }
 
   
+  void ArchiveJob::SetLoaderThreads(unsigned int loaderThreads)
+  {
+    if (writer_.get() != NULL)   // Already started
+    {
+      throw OrthancException(ErrorCode_BadSequenceOfCalls);
+    }
+    else
+    {
+      loaderThreads_ = loaderThreads;
+    }
+  }
+
+
   void ArchiveJob::Reset()
   {
     throw OrthancException(ErrorCode_BadSequenceOfCalls,
@@ -1002,6 +1203,16 @@
   
   void ArchiveJob::Start()
   {
+    if (loaderThreads_ == 0)
+    {
+      // default behaviour before loaderThreads was introducted in 1.9.8
+      instanceLoader_.reset(new SynchronousInstanceLoader(context_));
+    }
+    else
+    {
+      instanceLoader_.reset(new ThreadedInstanceLoader(context_, loaderThreads_));
+    }
+
     if (writer_.get() != NULL)
     {
       throw OrthancException(ErrorCode_BadSequenceOfCalls);
@@ -1023,7 +1234,7 @@
           assert(asynchronousTarget_.get() != NULL);
           asynchronousTarget_->Touch();  // Make sure we can write to the temporary file
           
-          writer_.reset(new ZipWriterIterator(context_, *archive_, isMedia_, enableExtendedSopClass_));
+          writer_.reset(new ZipWriterIterator(context_, *instanceLoader_, *archive_, isMedia_, enableExtendedSopClass_));
           writer_->SetOutputFile(asynchronousTarget_->GetPath());
         }
       }
@@ -1031,7 +1242,7 @@
       {
         assert(synchronousTarget_.get() != NULL);
     
-        writer_.reset(new ZipWriterIterator(context_, *archive_, isMedia_, enableExtendedSopClass_));
+        writer_.reset(new ZipWriterIterator(context_, *instanceLoader_, *archive_, isMedia_, enableExtendedSopClass_));
         writer_->AcquireOutputStream(synchronousTarget_.release());
       }
 
@@ -1076,6 +1287,11 @@
       writer_.reset();
     }
 
+    if (instanceLoader_.get() != NULL)
+    {
+      instanceLoader_->Clear();
+    }
+
     if (asynchronousTarget_.get() != NULL)
     {
       // Asynchronous behavior: Move the resulting file into the media archive
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancServer/Sources/ServerJobs/ArchiveJob.h
--- a/OrthancServer/Sources/ServerJobs/ArchiveJob.h	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancServer/Sources/ServerJobs/ArchiveJob.h	Tue Nov 23 09:22:11 2021 +0100
@@ -55,10 +55,14 @@
     class ResourceIdentifiers;
     class ZipCommands;
     class ZipWriterIterator;
-    
+    class InstanceLoader;
+    class SynchronousInstanceLoader;
+    class ThreadedInstanceLoader;
+
     std::unique_ptr<ZipWriter::IOutputStream>  synchronousTarget_;  // Only valid before "Start()"
     std::unique_ptr<TemporaryFile>        asynchronousTarget_;
     ServerContext&                        context_;
+    std::unique_ptr<InstanceLoader>       instanceLoader_;
     boost::shared_ptr<ArchiveIndex>       archive_;
     bool                                  isMedia_;
     bool                                  enableExtendedSopClass_;
@@ -75,6 +79,9 @@
     bool                 transcode_;
     DicomTransferSyntax  transferSyntax_;
 
+    // New in Orthanc 1.9.8
+    unsigned int         loaderThreads_;
+
     void FinalizeTarget();
     
   public:
@@ -97,6 +104,8 @@
 
     void SetTranscode(DicomTransferSyntax transferSyntax);
 
+    void SetLoaderThreads(unsigned int loaderThreads);
+
     virtual void Reset() ORTHANC_OVERRIDE;
 
     virtual void Start() ORTHANC_OVERRIDE;
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancServer/Sources/ServerToolbox.cpp
--- a/OrthancServer/Sources/ServerToolbox.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancServer/Sources/ServerToolbox.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -36,6 +36,7 @@
 
 #include "../../OrthancFramework/Sources/DicomParsing/ParsedDicomFile.h"
 #include "../../OrthancFramework/Sources/FileStorage/StorageAccessor.h"
+#include "../../OrthancFramework/Sources/FileStorage/StorageCache.h"
 #include "../../OrthancFramework/Sources/Logging.h"
 #include "../../OrthancFramework/Sources/OrthancException.h"
 #include "Database/IDatabaseWrapper.h"
@@ -176,7 +177,8 @@
         try
         {
           // Read and parse the content of the DICOM file
-          StorageAccessor accessor(storageArea);
+          StorageCache cache; // we create a temporary cache for this operation (required by the StorageAccessor)
+          StorageAccessor accessor(storageArea, cache);
 
           std::string content;
           accessor.Read(content, attachment);
diff -r 2ca4213fb50a -r 7afbb54bd028 OrthancServer/Sources/main.cpp
--- a/OrthancServer/Sources/main.cpp	Tue Nov 23 09:20:59 2021 +0100
+++ b/OrthancServer/Sources/main.cpp	Tue Nov 23 09:22:11 2021 +0100
@@ -1525,6 +1525,16 @@
     {
       context.GetIndex().SetMaximumStorageSize(0);
     }
+
+    try
+    {
+      uint64_t size = lock.GetConfiguration().GetUnsignedIntegerParameter("MaximumStorageCacheSize", 128);
+      context.SetMaximumStorageCacheSize(size * 1024 * 1024);
+    }
+    catch (...)
+    {
+      context.SetMaximumStorageCacheSize(128);
+    }
   }
 
   {