Mercurial > hg > orthanc
changeset 4810:7afbb54bd028
merge storage-cache
author | Alain Mazy <am@osimis.io> |
---|---|
date | Tue, 23 Nov 2021 09:22:11 +0100 |
parents | 2ca4213fb50a (current diff) 96ab170294fd (diff) |
children | 064d86287630 cd6dc99e0470 |
files | NEWS OrthancServer/Resources/Configuration.json OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp OrthancServer/Sources/ServerContext.cpp OrthancServer/Sources/ServerContext.h OrthancServer/Sources/ServerJobs/ArchiveJob.cpp OrthancServer/Sources/ServerJobs/ArchiveJob.h OrthancServer/Sources/main.cpp |
diffstat | 23 files changed, 656 insertions(+), 62 deletions(-) [+] |
line wrap: on
line diff
--- a/NEWS Tue Nov 23 09:20:59 2021 +0100 +++ b/NEWS Tue Nov 23 09:22:11 2021 +0100 @@ -1,10 +1,26 @@ Pending changes in the mainline =============================== +General +------- + +* Added a storage cache in RAM to avoid reading the same files multiple times from + the storage. This greatly improves, among other things, the performance of WADO-RS + retrieval of individual frames of multiframe instances. +* New configuration option "MaximumStorageCacheSize" to configure the size of + the new storage cache. +* New configuration option "ZipLoaderThreads" to configure the number of threads used + to read instances from storage when createing a Zip archive/media. + + +Maintenance +----------- + * Fix handling of option "DeidentifyLogs", notably for tags (0010,0010) and (0010,0020) - * New configuration options: - "DicomThreadsCount" to set the number of threads in the embedded DICOM server +* Fix instances accumulating in DB while their attachments were not stored because of + MaximumStorageSize limit reached with a single patient in DB. REST API -------- @@ -15,21 +31,12 @@ it raises a 415 error code. * Archive jobs response now contains a header Content-Disposition:filename='archive.zip' - - -Maintenance ------------ - -* Fix instances accumulating in DB while their attachments were not stored because of - MaximumStorageSize limit reached with a single patient in DB. - Lua --- * New "ReceivedCStoreInstanceFilter" Lua callback to filter instances received through C-Store and return a specific C-Store status code. - Plugins -------
--- a/OrthancFramework/Resources/CMake/OrthancFrameworkConfiguration.cmake Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Resources/CMake/OrthancFrameworkConfiguration.cmake Tue Nov 23 09:22:11 2021 +0100 @@ -385,6 +385,7 @@ ${CMAKE_CURRENT_LIST_DIR}/../../Sources/Compression/HierarchicalZipWriter.cpp ${CMAKE_CURRENT_LIST_DIR}/../../Sources/Compression/ZipWriter.cpp ${CMAKE_CURRENT_LIST_DIR}/../../Sources/FileStorage/StorageAccessor.cpp + ${CMAKE_CURRENT_LIST_DIR}/../../Sources/FileStorage/StorageCache.cpp ) endif() endif()
--- a/OrthancFramework/Sources/Cache/MemoryObjectCache.h Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Sources/Cache/MemoryObjectCache.h Tue Nov 23 09:22:11 2021 +0100 @@ -37,6 +37,9 @@ namespace Orthanc { + /** + * Note: this class is thread safe + **/ class ORTHANC_PUBLIC MemoryObjectCache : public boost::noncopyable { private:
--- a/OrthancFramework/Sources/Cache/MemoryStringCache.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Sources/Cache/MemoryStringCache.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -35,7 +35,12 @@ content_(content) { } - + + explicit StringValue(const char* buffer, size_t size) : + content_(buffer, size) + { + } + const std::string& GetContent() const { return content_; @@ -63,6 +68,13 @@ cache_.Acquire(key, new StringValue(value)); } + void MemoryStringCache::Add(const std::string& key, + const void* buffer, + size_t size) + { + cache_.Acquire(key, new StringValue(reinterpret_cast<const char*>(buffer), size)); + } + void MemoryStringCache::Invalidate(const std::string &key) { cache_.Invalidate(key);
--- a/OrthancFramework/Sources/Cache/MemoryStringCache.h Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Sources/Cache/MemoryStringCache.h Tue Nov 23 09:22:11 2021 +0100 @@ -29,6 +29,8 @@ /** * Facade object around "MemoryObjectCache" that caches a dictionary * of strings, using the "fetch/add" paradigm of memcached. + * + * Note: this class is thread safe **/ class ORTHANC_PUBLIC MemoryStringCache : public boost::noncopyable { @@ -44,7 +46,11 @@ void Add(const std::string& key, const std::string& value); - + + void Add(const std::string& key, + const void* buffer, + size_t size); + void Invalidate(const std::string& key); bool Fetch(std::string& value,
--- a/OrthancFramework/Sources/FileStorage/StorageAccessor.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Sources/FileStorage/StorageAccessor.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -22,7 +22,10 @@ #include "../PrecompiledHeaders.h" #include "StorageAccessor.h" +#include "StorageCache.h" +#include "../Logging.h" +#include "../StringMemoryBuffer.h" #include "../Compatibility.h" #include "../Compression/ZlibCompressor.h" #include "../MetricsRegistry.h" @@ -58,14 +61,18 @@ }; - StorageAccessor::StorageAccessor(IStorageArea &area) : + StorageAccessor::StorageAccessor(IStorageArea &area, StorageCache& cache) : area_(area), + cache_(cache), metrics_(NULL) { } - StorageAccessor::StorageAccessor(IStorageArea &area, MetricsRegistry &metrics) : + StorageAccessor::StorageAccessor(IStorageArea &area, + StorageCache& cache, + MetricsRegistry &metrics) : area_(area), + cache_(cache), metrics_(&metrics) { } @@ -93,6 +100,8 @@ MetricsTimer timer(*this, METRICS_CREATE); area_.Create(uuid, data, size, type); + cache_.Add(uuid, type, data, size); + return FileInfo(uuid, type, size, md5); } @@ -123,6 +132,7 @@ } } + cache_.Add(uuid, type, data, size); return FileInfo(uuid, type, size, md5, CompressionType_ZlibWithSize, compressed.size(), compressedMD5); } @@ -145,6 +155,13 @@ void StorageAccessor::Read(std::string& content, const FileInfo& info) { + if (cache_.Fetch(content, info.GetUuid(), info.GetContentType())) + { + LOG(INFO) << "Read attachment \"" << info.GetUuid() << "\" " + << "content type from cache"; + return; + } + switch (info.GetCompressionType()) { case CompressionType_None: @@ -152,7 +169,9 @@ MetricsTimer timer(*this, METRICS_READ); std::unique_ptr<IMemoryBuffer> buffer(area_.Read(info.GetUuid(), info.GetContentType())); - buffer->MoveToString(content); + buffer->MoveToString(content); + + cache_.Add(info.GetUuid(), info.GetContentType(), content); break; } @@ -168,6 +187,8 @@ } zlib.Uncompress(content, compressed->GetData(), compressed->GetSize()); + + cache_.Add(info.GetUuid(), info.GetContentType(), content); break; } @@ -196,6 +217,14 @@ { MetricsTimer timer(*this, METRICS_REMOVE); area_.Remove(fileUuid, type); + + cache_.Invalidate(fileUuid, type); + + // in ReadStartRange, we might have cached only the start of the file -> try to remove it + if (type == FileContentType_Dicom) + { + cache_.Invalidate(fileUuid, FileContentType_DicomUntilPixelData); + } } void StorageAccessor::Remove(const FileInfo &info) @@ -203,15 +232,56 @@ Remove(info.GetUuid(), info.GetContentType()); } + IMemoryBuffer* StorageAccessor::ReadStartRange(const std::string& fileUuid, + FileContentType contentType, + uint64_t end /* exclusive */, + FileContentType startFileContentType) + { + std::string content; + if (cache_.Fetch(content, fileUuid, contentType)) + { + LOG(INFO) << "Read attachment \"" << fileUuid << "\" " + << "(range from " << 0 << " to " << end << ") from cache"; + + return StringMemoryBuffer::CreateFromCopy(content, 0, end); + } + + if (cache_.Fetch(content, fileUuid, startFileContentType)) + { + LOG(INFO) << "Read attachment \"" << fileUuid << "\" " + << "(range from " << 0 << " to " << end << ") from cache"; + + assert(content.size() == end); + return StringMemoryBuffer::CreateFromCopy(content); + } + + std::unique_ptr<IMemoryBuffer> buffer(area_.ReadRange(fileUuid, contentType, 0, end)); + + // we've read only the first part of the file -> add an entry in the cache + // note the uuid is still the uuid of the full file but the type is the type of the start of the file ! + assert(buffer->GetSize() == end); + cache_.Add(fileUuid, startFileContentType, buffer->GetData(), buffer->GetSize()); + return buffer.release(); + } + + #if ORTHANC_ENABLE_CIVETWEB == 1 || ORTHANC_ENABLE_MONGOOSE == 1 void StorageAccessor::SetupSender(BufferHttpSender& sender, const FileInfo& info, const std::string& mime) { + if (cache_.Fetch(sender.GetBuffer(), info.GetUuid(), info.GetContentType())) + { + LOG(INFO) << "Read attachment \"" << info.GetUuid() << "\" " + << "content type from cache"; + } + else { MetricsTimer timer(*this, METRICS_READ); std::unique_ptr<IMemoryBuffer> buffer(area_.Read(info.GetUuid(), info.GetContentType())); buffer->MoveToString(sender.GetBuffer()); + + cache_.Add(info.GetUuid(), info.GetContentType(), sender.GetBuffer()); } sender.SetContentType(mime);
--- a/OrthancFramework/Sources/FileStorage/StorageAccessor.h Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Sources/FileStorage/StorageAccessor.h Tue Nov 23 09:22:11 2021 +0100 @@ -54,6 +54,7 @@ namespace Orthanc { class MetricsRegistry; + class StorageCache; /** * This class handles the compression/decompression of the raw files @@ -66,6 +67,7 @@ class MetricsTimer; IStorageArea& area_; + StorageCache& cache_; MetricsRegistry* metrics_; #if ORTHANC_ENABLE_CIVETWEB == 1 || ORTHANC_ENABLE_MONGOOSE == 1 @@ -75,9 +77,11 @@ #endif public: - explicit StorageAccessor(IStorageArea& area); + explicit StorageAccessor(IStorageArea& area, + StorageCache& cache); StorageAccessor(IStorageArea& area, + StorageCache& cache, MetricsRegistry& metrics); FileInfo Write(const void* data, @@ -97,6 +101,11 @@ void ReadRaw(std::string& content, const FileInfo& info); + IMemoryBuffer* ReadStartRange(const std::string& fileUuid, + FileContentType fullFileContentType, + uint64_t end /* exclusive */, + FileContentType startFileContentType); + void Remove(const std::string& fileUuid, FileContentType type);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancFramework/Sources/FileStorage/StorageCache.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -0,0 +1,118 @@ +/** + * Orthanc - A Lightweight, RESTful DICOM Store + * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics + * Department, University Hospital of Liege, Belgium + * Copyright (C) 2017-2021 Osimis S.A., Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/>. + **/ + + +#include "../PrecompiledHeaders.h" +#include "StorageCache.h" + +#include "../Compatibility.h" +#include "../OrthancException.h" + + + +namespace Orthanc +{ + bool IsAcceptedContentType(FileContentType contentType) + { + return contentType == FileContentType_Dicom || + contentType == FileContentType_DicomUntilPixelData || + contentType == FileContentType_DicomAsJson; + } + + const char* ToString(FileContentType contentType) + { + switch (contentType) + { + case FileContentType_Dicom: + return "dicom"; + case FileContentType_DicomUntilPixelData: + return "dicom-header"; + case FileContentType_DicomAsJson: + return "dicom-json"; + default: + throw OrthancException(ErrorCode_InternalError, + "ContentType not supported in StorageCache"); + } + } + + void GetCacheKey(std::string& key, const std::string& uuid, FileContentType contentType) + { + key = uuid + ":" + std::string(ToString(contentType)); + } + + void StorageCache::SetMaximumSize(size_t size) + { + cache_.SetMaximumSize(size); + } + + void StorageCache::Add(const std::string& uuid, + FileContentType contentType, + const std::string& value) + { + if (!IsAcceptedContentType(contentType)) + { + return; + } + + std::string key; + GetCacheKey(key, uuid, contentType); + cache_.Add(key, value); + } + + void StorageCache::Add(const std::string& uuid, + FileContentType contentType, + const void* buffer, + size_t size) + { + if (!IsAcceptedContentType(contentType)) + { + return; + } + + std::string key; + GetCacheKey(key, uuid, contentType); + cache_.Add(key, buffer, size); + } + + void StorageCache::Invalidate(const std::string& uuid, FileContentType contentType) + { + std::string key; + GetCacheKey(key, uuid, contentType); + cache_.Invalidate(key); + } + + bool StorageCache::Fetch(std::string& value, + const std::string& uuid, + FileContentType contentType) + { + if (!IsAcceptedContentType(contentType)) + { + return false; + } + + std::string key; + GetCacheKey(key, uuid, contentType); + + return cache_.Fetch(value, key); + } + + +} \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancFramework/Sources/FileStorage/StorageCache.h Tue Nov 23 09:22:11 2021 +0100 @@ -0,0 +1,59 @@ +/** + * Orthanc - A Lightweight, RESTful DICOM Store + * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics + * Department, University Hospital of Liege, Belgium + * Copyright (C) 2017-2021 Osimis S.A., Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/>. + **/ + + +#pragma once + +#include "../Cache/MemoryStringCache.h" + +#include "../Compatibility.h" // For ORTHANC_OVERRIDE + +#include <boost/thread/mutex.hpp> +#include <map> + +namespace Orthanc +{ + /** + * Note: this class is thread safe + **/ + class ORTHANC_PUBLIC StorageCache : public boost::noncopyable + { + MemoryStringCache cache_; + public: + void SetMaximumSize(size_t size); + + void Add(const std::string& uuid, + FileContentType contentType, + const std::string& value); + + void Add(const std::string& uuid, + FileContentType contentType, + const void* buffer, + size_t size); + + void Invalidate(const std::string& uuid, FileContentType contentType); + + bool Fetch(std::string& value, + const std::string& uuid, + FileContentType contentType); + + }; +}
--- a/OrthancFramework/Sources/MultiThreading/Semaphore.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Sources/MultiThreading/Semaphore.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -31,10 +31,6 @@ Semaphore::Semaphore(unsigned int availableResources) : availableResources_(availableResources) { - if (availableResources_ == 0) - { - throw OrthancException(ErrorCode_ParameterOutOfRange); - } } unsigned int Semaphore::GetAvailableResourcesCount() const
--- a/OrthancFramework/Sources/MultiThreading/Semaphore.h Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Sources/MultiThreading/Semaphore.h Tue Nov 23 09:22:11 2021 +0100 @@ -36,16 +36,16 @@ boost::mutex mutex_; boost::condition_variable condition_; + public: + explicit Semaphore(unsigned int availableResources); + + unsigned int GetAvailableResourcesCount() const; + void Release(unsigned int resourceCount = 1); void Acquire(unsigned int resourceCount = 1); bool TryAcquire(unsigned int resourceCount = 1); - public: - explicit Semaphore(unsigned int availableResources); - - unsigned int GetAvailableResourcesCount() const; - class Locker : public boost::noncopyable {
--- a/OrthancFramework/Sources/StringMemoryBuffer.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Sources/StringMemoryBuffer.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -47,4 +47,14 @@ result->Copy(buffer); return result.release(); } + + + IMemoryBuffer* StringMemoryBuffer::CreateFromCopy(const std::string& buffer, + size_t start /* inclusive */, + size_t end /* exclusive */) + { + std::unique_ptr<StringMemoryBuffer> result(new StringMemoryBuffer); + result->Copy(buffer, start, end); + return result.release(); + } }
--- a/OrthancFramework/Sources/StringMemoryBuffer.h Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/Sources/StringMemoryBuffer.h Tue Nov 23 09:22:11 2021 +0100 @@ -38,6 +38,11 @@ buffer_ = buffer; } + void Copy(const std::string& buffer, size_t start /* inclusive */, size_t end /* exclusive */) + { + buffer_.assign(buffer, start, end - start); + } + void Swap(std::string& buffer) { buffer_.swap(buffer); @@ -58,5 +63,7 @@ static IMemoryBuffer* CreateFromSwap(std::string& buffer); static IMemoryBuffer* CreateFromCopy(const std::string& buffer); + + static IMemoryBuffer* CreateFromCopy(const std::string& buffer, size_t start /* inclusive */, size_t end /* exclusive */); }; }
--- a/OrthancFramework/UnitTestsSources/FileStorageTests.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancFramework/UnitTestsSources/FileStorageTests.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -29,6 +29,7 @@ #include "../Sources/FileStorage/FilesystemStorage.h" #include "../Sources/FileStorage/StorageAccessor.h" +#include "../Sources/FileStorage/StorageCache.h" #include "../Sources/HttpServer/BufferHttpSender.h" #include "../Sources/HttpServer/FilesystemHttpSender.h" #include "../Sources/Logging.h" @@ -124,7 +125,8 @@ TEST(StorageAccessor, NoCompression) { FilesystemStorage s("UnitTestsStorage"); - StorageAccessor accessor(s); + StorageCache cache; + StorageAccessor accessor(s, cache); std::string data = "Hello world"; FileInfo info = accessor.Write(data, FileContentType_Dicom, CompressionType_None, true); @@ -145,7 +147,8 @@ TEST(StorageAccessor, Compression) { FilesystemStorage s("UnitTestsStorage"); - StorageAccessor accessor(s); + StorageCache cache; + StorageAccessor accessor(s, cache); std::string data = "Hello world"; FileInfo info = accessor.Write(data, FileContentType_Dicom, CompressionType_ZlibWithSize, true); @@ -165,7 +168,8 @@ TEST(StorageAccessor, Mix) { FilesystemStorage s("UnitTestsStorage"); - StorageAccessor accessor(s); + StorageCache cache; + StorageAccessor accessor(s, cache); std::string r; std::string compressedData = "Hello";
--- a/OrthancServer/Resources/Configuration.json Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancServer/Resources/Configuration.json Tue Nov 23 09:22:11 2021 +0100 @@ -40,7 +40,13 @@ // in the storage (a value of "0" indicates no limit on the number // of patients) "MaximumPatientCount" : 0, - + + // Maximum size of the storage cache in MB. The storage cache + // is stored in RAM and contains a copy of recently accessed + // files (written or read). A value of "0" indicates the cache + // is disabled. (new in Orthanc 1.9.8) + "MaximumStorageCacheSize" : 128, + // List of paths to the custom Lua scripts that are to be loaded // into this instance of Orthanc "LuaScripts" : [ @@ -837,5 +843,12 @@ // disk space and might lead to HTTP timeouts on large archives). If // set to "true", the chunks of the ZIP file are progressively sent // as soon as one DICOM file gets compressed (new in Orthanc 1.9.4) - "SynchronousZipStream" : true + "SynchronousZipStream" : true, + + // Default number of loader threads when generating Zip archive/media. + // A value of 0 means reading and writing are performed in sequence + // (default behaviour). A value > 1 is meaningful only if the storage + // is a distributed network storage (e.g object storage plugin). + // (new in Orthanc 1.9.8) + "ZipLoaderThreads": 0 }
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestArchive.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestArchive.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -51,7 +51,9 @@ static const char* const KEY_RESOURCES = "Resources"; static const char* const KEY_EXTENDED = "Extended"; static const char* const KEY_TRANSCODE = "Transcode"; - + + static const char* const CONFIG_LOADER_THREADS = "ZipLoaderThreads"; + static void AddResourcesOfInterestFromArray(ArchiveJob& job, const Json::Value& resources) { @@ -123,6 +125,7 @@ bool& transcode, /* out */ DicomTransferSyntax& syntax, /* out */ int& priority, /* out */ + unsigned int& loaderThreads, /* out */ const Json::Value& body, /* in */ const bool defaultExtended /* in */) { @@ -151,6 +154,12 @@ { transcode = false; } + + { + OrthancConfiguration::ReaderLock lock; + loaderThreads = lock.GetConfiguration().GetUnsignedIntegerParameter(CONFIG_LOADER_THREADS, 0); // New in Orthanc 1.9.8 + } + } @@ -554,8 +563,9 @@ bool synchronous, extended, transcode; DicomTransferSyntax transferSyntax; int priority; + unsigned int loaderThreads; GetJobParameters(synchronous, extended, transcode, transferSyntax, - priority, body, DEFAULT_IS_EXTENDED); + priority, loaderThreads, body, DEFAULT_IS_EXTENDED); std::unique_ptr<ArchiveJob> job(new ArchiveJob(context, IS_MEDIA, extended)); AddResourcesOfInterest(*job, body); @@ -565,6 +575,8 @@ job->SetTranscode(transferSyntax); } + job->SetLoaderThreads(loaderThreads); + SubmitJob(call.GetOutput(), context, job, priority, synchronous, "Archive.zip"); } else @@ -578,6 +590,8 @@ template <bool IS_MEDIA> static void CreateSingleGet(RestApiGetCall& call) { + static const char* const TRANSCODE = "transcode"; + if (call.IsDocumentation()) { ResourceType t = StringToResourceType(call.GetFullUri()[0].c_str()); @@ -591,7 +605,7 @@ "which might *not* be desirable to archive large amount of data, as it might " "lead to network timeouts. Prefer the asynchronous version using `POST` method.") .SetUriArgument("id", "Orthanc identifier of the " + r + " of interest") - .SetHttpGetArgument("transcode", RestApiCallDocumentation::Type_String, + .SetHttpGetArgument(TRANSCODE, RestApiCallDocumentation::Type_String, "If present, the DICOM files in the archive will be transcoded to the provided " "transfer syntax: https://book.orthanc-server.com/faq/transcoding.html", false) .AddAnswerType(MimeType_Zip, "ZIP file containing the archive"); @@ -621,12 +635,17 @@ std::unique_ptr<ArchiveJob> job(new ArchiveJob(context, IS_MEDIA, extended)); job->AddResource(id); - static const char* const TRANSCODE = "transcode"; if (call.HasArgument(TRANSCODE)) { job->SetTranscode(GetTransferSyntax(call.GetArgument(TRANSCODE, ""))); } + { + OrthancConfiguration::ReaderLock lock; + unsigned int loaderThreads = lock.GetConfiguration().GetUnsignedIntegerParameter(CONFIG_LOADER_THREADS, 0); // New in Orthanc 1.9.8 + job->SetLoaderThreads(loaderThreads); + } + SubmitJob(call.GetOutput(), context, job, 0 /* priority */, true /* synchronous */, id + ".zip"); } @@ -660,8 +679,9 @@ bool synchronous, extended, transcode; DicomTransferSyntax transferSyntax; int priority; + unsigned int loaderThreads; GetJobParameters(synchronous, extended, transcode, transferSyntax, - priority, body, false /* by default, not extented */); + priority, loaderThreads, body, false /* by default, not extented */); std::unique_ptr<ArchiveJob> job(new ArchiveJob(context, IS_MEDIA, extended)); job->AddResource(id); @@ -671,6 +691,8 @@ job->SetTranscode(transferSyntax); } + job->SetLoaderThreads(loaderThreads); + SubmitJob(call.GetOutput(), context, job, priority, synchronous, id + ".zip"); } else
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -2977,7 +2977,7 @@ std::string publicId = call.GetUriComponent("id", ""); std::string dicomContent; - context.ReadDicom(dicomContent, publicId); + context.ReadDicomForHeader(dicomContent, publicId); // TODO Consider using "DicomMap::ParseDicomMetaInformation()" to // speed up things here
--- a/OrthancServer/Sources/ServerContext.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancServer/Sources/ServerContext.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -492,7 +492,7 @@ void ServerContext::RemoveFile(const std::string& fileUuid, FileContentType type) { - StorageAccessor accessor(area_, GetMetricsRegistry()); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); accessor.Remove(fileUuid, type); } @@ -534,7 +534,7 @@ try { MetricsRegistry::Timer timer(GetMetricsRegistry(), "orthanc_store_dicom_duration_ms"); - StorageAccessor accessor(area_, GetMetricsRegistry()); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); DicomInstanceHasher hasher(summary); resultPublicId = hasher.HashInstance(); @@ -781,7 +781,7 @@ } else { - StorageAccessor accessor(area_, GetMetricsRegistry()); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); accessor.AnswerFile(output, attachment, GetFileContentMime(content)); } } @@ -811,7 +811,7 @@ std::string content; - StorageAccessor accessor(area_, GetMetricsRegistry()); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); accessor.Read(content, attachment); FileInfo modified = accessor.Write(content.empty() ? NULL : content.c_str(), @@ -867,7 +867,7 @@ std::string dicom; { - StorageAccessor accessor(area_, GetMetricsRegistry()); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); accessor.Read(dicom, attachment); } @@ -932,8 +932,8 @@ std::unique_ptr<IMemoryBuffer> dicom; { - MetricsRegistry::Timer timer(GetMetricsRegistry(), "orthanc_storage_read_range_duration_ms"); - dicom.reset(area_.ReadRange(attachment.GetUuid(), FileContentType_Dicom, 0, pixelDataOffset)); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); + dicom.reset(accessor.ReadStartRange(attachment.GetUuid(), FileContentType_Dicom, pixelDataOffset, FileContentType_DicomUntilPixelData)); } if (dicom.get() == NULL) @@ -962,7 +962,7 @@ std::string dicomAsJson; { - StorageAccessor accessor(area_, GetMetricsRegistry()); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); accessor.Read(dicomAsJson, attachment); } @@ -1031,7 +1031,15 @@ int64_t revision; ReadAttachment(dicom, revision, instancePublicId, FileContentType_Dicom, true /* uncompress */); } - + + void ServerContext::ReadDicomForHeader(std::string& dicom, + const std::string& instancePublicId) + { + if (!ReadDicomUntilPixelData(dicom, instancePublicId)) + { + ReadDicom(dicom, instancePublicId); + } + } bool ServerContext::ReadDicomUntilPixelData(std::string& dicom, const std::string& instancePublicId) @@ -1060,8 +1068,10 @@ { uint64_t pixelDataOffset = boost::lexical_cast<uint64_t>(s); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); + std::unique_ptr<IMemoryBuffer> buffer( - area_.ReadRange(attachment.GetUuid(), attachment.GetContentType(), 0, pixelDataOffset)); + accessor.ReadStartRange(attachment.GetUuid(), attachment.GetContentType(), pixelDataOffset, FileContentType_DicomUntilPixelData)); buffer->MoveToString(dicom); return true; // Success } @@ -1092,7 +1102,7 @@ assert(attachment.GetContentType() == content); { - StorageAccessor accessor(area_, GetMetricsRegistry()); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); if (uncompressIfNeeded) { @@ -1192,7 +1202,7 @@ // TODO Should we use "gzip" instead? CompressionType compression = (compressionEnabled_ ? CompressionType_ZlibWithSize : CompressionType_None); - StorageAccessor accessor(area_, GetMetricsRegistry()); + StorageAccessor accessor(area_, storageCache_, GetMetricsRegistry()); FileInfo attachment = accessor.Write(data, size, attachmentType, compression, storeMD5_); try
--- a/OrthancServer/Sources/ServerContext.h Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancServer/Sources/ServerContext.h Tue Nov 23 09:22:11 2021 +0100 @@ -43,6 +43,7 @@ #include "../../OrthancFramework/Sources/DicomParsing/DicomModification.h" #include "../../OrthancFramework/Sources/DicomParsing/IDicomTranscoder.h" #include "../../OrthancFramework/Sources/DicomParsing/ParsedDicomCache.h" +#include "../../OrthancFramework/Sources/FileStorage/StorageCache.h" #include "../../OrthancFramework/Sources/MultiThreading/Semaphore.h" @@ -205,6 +206,7 @@ ServerIndex index_; IStorageArea& area_; + StorageCache storageCache_; bool compressionEnabled_; bool storeMD5_; @@ -324,6 +326,11 @@ return index_; } + void SetMaximumStorageCacheSize(size_t size) + { + return storageCache_.SetMaximumSize(size); + } + void SetCompressionEnabled(bool enabled); bool IsCompressionEnabled() const @@ -361,7 +368,10 @@ void ReadDicom(std::string& dicom, const std::string& instancePublicId); - + + void ReadDicomForHeader(std::string& dicom, + const std::string& instancePublicId); + bool ReadDicomUntilPixelData(std::string& dicom, const std::string& instancePublicId);
--- a/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancServer/Sources/ServerJobs/ArchiveJob.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -40,6 +40,7 @@ #include "../../../OrthancFramework/Sources/DicomParsing/FromDcmtkBridge.h" #include "../../../OrthancFramework/Sources/Logging.h" #include "../../../OrthancFramework/Sources/OrthancException.h" +#include "../../../OrthancFramework/Sources/MultiThreading/Semaphore.h" #include "../OrthancConfiguration.h" #include "../ServerContext.h" @@ -88,6 +89,181 @@ } + class ArchiveJob::InstanceLoader : public boost::noncopyable + { + protected: + ServerContext& context_; + public: + InstanceLoader(ServerContext& context) + : context_(context) + { + } + + virtual ~InstanceLoader() + { + } + + virtual void PrepareDicom(const std::string& instanceId) + { + + } + + virtual void GetDicom(std::string& dicom, const std::string& instanceId) = 0; + + virtual void Clear() + { + } + }; + + class ArchiveJob::SynchronousInstanceLoader : public ArchiveJob::InstanceLoader + { + public: + SynchronousInstanceLoader(ServerContext& context) + : InstanceLoader(context) + { + } + + virtual void GetDicom(std::string& dicom, const std::string& instanceId) ORTHANC_OVERRIDE + { + context_.ReadDicom(dicom, instanceId); + } + }; + + class InstanceId : public Orthanc::IDynamicObject + { + private: + std::string id_; + + public: + InstanceId(const std::string& id) : id_(id) + { + } + + virtual ~InstanceId() ORTHANC_OVERRIDE + { + } + + std::string GetId() const {return id_;}; + }; + + class ArchiveJob::ThreadedInstanceLoader : public ArchiveJob::InstanceLoader + { + Semaphore availableInstancesSemaphore_; + std::map<std::string, boost::shared_ptr<std::string>> availableInstances_; + boost::mutex availableInstancesMutex_; + SharedMessageQueue instancesToPreload_; + std::vector<boost::thread*> threads_; + + + public: + ThreadedInstanceLoader(ServerContext& context, size_t threadCount) + : InstanceLoader(context), + availableInstancesSemaphore_(0) + { + for (size_t i = 0; i < threadCount; i++) + { + threads_.push_back(new boost::thread(PreloaderWorkerThread, this)); + } + } + + virtual ~ThreadedInstanceLoader() + { + Clear(); + } + + virtual void Clear() ORTHANC_OVERRIDE + { + for (size_t i = 0; i < threads_.size(); i++) + { + instancesToPreload_.Enqueue(NULL); + } + + for (size_t i = 0; i < threads_.size(); i++) + { + if (threads_[i]->joinable()) + { + threads_[i]->join(); + } + delete threads_[i]; + } + + threads_.clear(); + availableInstances_.clear(); + } + + static void PreloaderWorkerThread(ThreadedInstanceLoader* that) + { + while (true) + { + std::unique_ptr<InstanceId> instanceId(dynamic_cast<InstanceId*>(that->instancesToPreload_.Dequeue(0))); + if (instanceId.get() == NULL) // that's the signal to exit the thread + { + return; + } + + try + { + boost::shared_ptr<std::string> dicomContent(new std::string()); + that->context_.ReadDicom(*dicomContent, instanceId->GetId()); + { + boost::mutex::scoped_lock lock(that->availableInstancesMutex_); + that->availableInstances_[instanceId->GetId()] = dicomContent; + } + + that->availableInstancesSemaphore_.Release(); + } + catch (OrthancException& e) + { + boost::mutex::scoped_lock lock(that->availableInstancesMutex_); + // store a NULL result to notify that we could not read the instance + that->availableInstances_[instanceId->GetId()] = boost::shared_ptr<std::string>(); + that->availableInstancesSemaphore_.Release(); + } + } + } + + virtual void PrepareDicom(const std::string& instanceId) ORTHANC_OVERRIDE + { + instancesToPreload_.Enqueue(new InstanceId(instanceId)); + } + + virtual void GetDicom(std::string& dicom, const std::string& instanceId) ORTHANC_OVERRIDE + { + while (true) + { + // wait for an instance to be available but this might not be the one we are waiting for ! + availableInstancesSemaphore_.Acquire(); + + boost::shared_ptr<std::string> dicomContent; + { + if (availableInstances_.find(instanceId) != availableInstances_.end()) + { + // this is the instance we were waiting for + dicomContent = availableInstances_[instanceId]; + availableInstances_.erase(instanceId); + + if (dicomContent.get() == NULL) // there has been an error while reading the file + { + throw OrthancException(ErrorCode_InexistentItem); + } + dicom.swap(*dicomContent); + + if (availableInstances_.size() > 0) + { + // we have just read the instance we were waiting for but there are still other instances available -> + // make sure the next GetDicom call does not wait ! + availableInstancesSemaphore_.Release(); + } + return; + } + // we have not found the expected instance, simply wait for the next loader thread to signal the semaphore when + // a new instance is available + } + } + } + }; + + class ArchiveJob::ResourceIdentifiers : public boost::noncopyable { private: @@ -402,6 +578,7 @@ void Apply(HierarchicalZipWriter& writer, ServerContext& context, + InstanceLoader& instanceLoader, DicomDirWriter* dicomDir, const std::string& dicomDirFolder, bool transcode, @@ -423,7 +600,7 @@ try { - context.ReadDicom(content, instanceId_); + instanceLoader.GetDicom(content, instanceId_); } catch (OrthancException& e) { @@ -494,10 +671,12 @@ std::deque<Command*> commands_; uint64_t uncompressedSize_; unsigned int instancesCount_; + InstanceLoader& instanceLoader_; void ApplyInternal(HierarchicalZipWriter& writer, ServerContext& context, + InstanceLoader& instanceLoader, size_t index, DicomDirWriter* dicomDir, const std::string& dicomDirFolder, @@ -509,13 +688,14 @@ throw OrthancException(ErrorCode_ParameterOutOfRange); } - commands_[index]->Apply(writer, context, dicomDir, dicomDirFolder, transcode, transferSyntax); + commands_[index]->Apply(writer, context, instanceLoader, dicomDir, dicomDirFolder, transcode, transferSyntax); } public: - ZipCommands() : + ZipCommands(InstanceLoader& instanceLoader) : uncompressedSize_(0), - instancesCount_(0) + instancesCount_(0), + instanceLoader_(instanceLoader) { } @@ -547,23 +727,25 @@ // "media" flavor (with DICOMDIR) void Apply(HierarchicalZipWriter& writer, ServerContext& context, + InstanceLoader& instanceLoader, size_t index, DicomDirWriter& dicomDir, const std::string& dicomDirFolder, bool transcode, DicomTransferSyntax transferSyntax) const { - ApplyInternal(writer, context, index, &dicomDir, dicomDirFolder, transcode, transferSyntax); + ApplyInternal(writer, context, instanceLoader, index, &dicomDir, dicomDirFolder, transcode, transferSyntax); } // "archive" flavor (without DICOMDIR) void Apply(HierarchicalZipWriter& writer, ServerContext& context, + InstanceLoader& instanceLoader, size_t index, bool transcode, DicomTransferSyntax transferSyntax) const { - ApplyInternal(writer, context, index, NULL, "", transcode, transferSyntax); + ApplyInternal(writer, context, instanceLoader, index, NULL, "", transcode, transferSyntax); } void AddOpenDirectory(const std::string& filename) @@ -580,6 +762,7 @@ const std::string& instanceId, uint64_t uncompressedSize) { + instanceLoader_.PrepareDicom(instanceId); commands_.push_back(new Command(Type_WriteInstance, filename, instanceId)); instancesCount_ ++; uncompressedSize_ += uncompressedSize; @@ -747,6 +930,7 @@ { private: ServerContext& context_; + InstanceLoader& instanceLoader_; ZipCommands commands_; std::unique_ptr<HierarchicalZipWriter> zip_; std::unique_ptr<DicomDirWriter> dicomDir_; @@ -755,10 +939,13 @@ public: ZipWriterIterator(ServerContext& context, + InstanceLoader& instanceLoader, ArchiveIndex& archive, bool isMedia, bool enableExtendedSopClass) : context_(context), + instanceLoader_(instanceLoader), + commands_(instanceLoader), isMedia_(isMedia), isStream_(false) { @@ -882,13 +1069,13 @@ if (isMedia_) { assert(dicomDir_.get() != NULL); - commands_.Apply(*zip_, context_, index, *dicomDir_, + commands_.Apply(*zip_, context_, instanceLoader_, index, *dicomDir_, MEDIA_IMAGES_FOLDER, transcode, transferSyntax); } else { assert(dicomDir_.get() == NULL); - commands_.Apply(*zip_, context_, index, transcode, transferSyntax); + commands_.Apply(*zip_, context_, instanceLoader_, index, transcode, transferSyntax); } } } @@ -917,7 +1104,8 @@ uncompressedSize_(0), archiveSize_(0), transcode_(false), - transferSyntax_(DicomTransferSyntax_LittleEndianImplicit) + transferSyntax_(DicomTransferSyntax_LittleEndianImplicit), + loaderThreads_(0) { } @@ -993,6 +1181,19 @@ } + void ArchiveJob::SetLoaderThreads(unsigned int loaderThreads) + { + if (writer_.get() != NULL) // Already started + { + throw OrthancException(ErrorCode_BadSequenceOfCalls); + } + else + { + loaderThreads_ = loaderThreads; + } + } + + void ArchiveJob::Reset() { throw OrthancException(ErrorCode_BadSequenceOfCalls, @@ -1002,6 +1203,16 @@ void ArchiveJob::Start() { + if (loaderThreads_ == 0) + { + // default behaviour before loaderThreads was introducted in 1.9.8 + instanceLoader_.reset(new SynchronousInstanceLoader(context_)); + } + else + { + instanceLoader_.reset(new ThreadedInstanceLoader(context_, loaderThreads_)); + } + if (writer_.get() != NULL) { throw OrthancException(ErrorCode_BadSequenceOfCalls); @@ -1023,7 +1234,7 @@ assert(asynchronousTarget_.get() != NULL); asynchronousTarget_->Touch(); // Make sure we can write to the temporary file - writer_.reset(new ZipWriterIterator(context_, *archive_, isMedia_, enableExtendedSopClass_)); + writer_.reset(new ZipWriterIterator(context_, *instanceLoader_, *archive_, isMedia_, enableExtendedSopClass_)); writer_->SetOutputFile(asynchronousTarget_->GetPath()); } } @@ -1031,7 +1242,7 @@ { assert(synchronousTarget_.get() != NULL); - writer_.reset(new ZipWriterIterator(context_, *archive_, isMedia_, enableExtendedSopClass_)); + writer_.reset(new ZipWriterIterator(context_, *instanceLoader_, *archive_, isMedia_, enableExtendedSopClass_)); writer_->AcquireOutputStream(synchronousTarget_.release()); } @@ -1076,6 +1287,11 @@ writer_.reset(); } + if (instanceLoader_.get() != NULL) + { + instanceLoader_->Clear(); + } + if (asynchronousTarget_.get() != NULL) { // Asynchronous behavior: Move the resulting file into the media archive
--- a/OrthancServer/Sources/ServerJobs/ArchiveJob.h Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancServer/Sources/ServerJobs/ArchiveJob.h Tue Nov 23 09:22:11 2021 +0100 @@ -55,10 +55,14 @@ class ResourceIdentifiers; class ZipCommands; class ZipWriterIterator; - + class InstanceLoader; + class SynchronousInstanceLoader; + class ThreadedInstanceLoader; + std::unique_ptr<ZipWriter::IOutputStream> synchronousTarget_; // Only valid before "Start()" std::unique_ptr<TemporaryFile> asynchronousTarget_; ServerContext& context_; + std::unique_ptr<InstanceLoader> instanceLoader_; boost::shared_ptr<ArchiveIndex> archive_; bool isMedia_; bool enableExtendedSopClass_; @@ -75,6 +79,9 @@ bool transcode_; DicomTransferSyntax transferSyntax_; + // New in Orthanc 1.9.8 + unsigned int loaderThreads_; + void FinalizeTarget(); public: @@ -97,6 +104,8 @@ void SetTranscode(DicomTransferSyntax transferSyntax); + void SetLoaderThreads(unsigned int loaderThreads); + virtual void Reset() ORTHANC_OVERRIDE; virtual void Start() ORTHANC_OVERRIDE;
--- a/OrthancServer/Sources/ServerToolbox.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancServer/Sources/ServerToolbox.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -36,6 +36,7 @@ #include "../../OrthancFramework/Sources/DicomParsing/ParsedDicomFile.h" #include "../../OrthancFramework/Sources/FileStorage/StorageAccessor.h" +#include "../../OrthancFramework/Sources/FileStorage/StorageCache.h" #include "../../OrthancFramework/Sources/Logging.h" #include "../../OrthancFramework/Sources/OrthancException.h" #include "Database/IDatabaseWrapper.h" @@ -176,7 +177,8 @@ try { // Read and parse the content of the DICOM file - StorageAccessor accessor(storageArea); + StorageCache cache; // we create a temporary cache for this operation (required by the StorageAccessor) + StorageAccessor accessor(storageArea, cache); std::string content; accessor.Read(content, attachment);
--- a/OrthancServer/Sources/main.cpp Tue Nov 23 09:20:59 2021 +0100 +++ b/OrthancServer/Sources/main.cpp Tue Nov 23 09:22:11 2021 +0100 @@ -1525,6 +1525,16 @@ { context.GetIndex().SetMaximumStorageSize(0); } + + try + { + uint64_t size = lock.GetConfiguration().GetUnsignedIntegerParameter("MaximumStorageCacheSize", 128); + context.SetMaximumStorageCacheSize(size * 1024 * 1024); + } + catch (...) + { + context.SetMaximumStorageCacheSize(128); + } } {