# HG changeset patch # User Alain Mazy # Date 1675275898 -3600 # Node ID dce028ec0fe933578f3038b4a8f95160ae4e7a91 # Parent a72e00a23191440604719ef2967f7d44b005cbbc# Parent 217863b0945722cc3b2735dc24d4402939cee792 merge diff -r a72e00a23191 -r dce028ec0fe9 NEWS --- a/NEWS Tue Jan 31 16:21:44 2023 +0100 +++ b/NEWS Wed Feb 01 19:24:58 2023 +0100 @@ -12,6 +12,11 @@ * New configuration "KeepAliveTimeout" with a default value of 1 second. * ResourceModification jobs (/modify + /anonymize) can now use multiple threads to speed up processing - New configuration "JobsEngineThreadsCount.ResourceModification" to configure the number of threads. +* Introduced a new Housekeeper thread in Orthanc (different from the Housekeeper sample plugin). This thread + regularly try to give back memory that Orthanc no longer uses to the system. This reduces the overall memory + consumption. However, only the memory at the end of a memory arena is given back to the system. Fragmented memory + is not given back and therefore, the memory consumption may still stay high. More information in + OrthancServer/Resources/ImplementationNotes/memory_consumption.txt. REST API -------- diff -r a72e00a23191 -r dce028ec0fe9 OrthancServer/Resources/ImplementationNotes/memory_consumption.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OrthancServer/Resources/ImplementationNotes/memory_consumption.txt Wed Feb 01 19:24:58 2023 +0100 @@ -0,0 +1,78 @@ +In Orthanc 1.11.3, we have introduced a Housekeeper thread that +tries to give back unused memory back to the system. This is implemented +by calling malloc_trim every 100ms. + +Here is how we validated the effect of this new feature: +------------------------------------------------------- + +We compared the behaviour of 2 osimis/orthanc Docker images from the mainline +on Feb 1st 2023. One image without the call to malloc_trim and the other with +this call. + +1st test: unconstrained Docker containers +......................................... + +5 large studies are uploaded to each instance of Orthanc (around 1GB in total). +A script triggers anonymization of these studies as quick as possible. +We compare the memory used by the containers after 2 minutes of execution +(using `docker stats`): +- without malloc_trim: 1500 MB +- with malloc_trim: 410 MB + +2nd test: memory constrained Docker containers +.............................................. + +Each Orthanc container is limited to 400MB (through the docker-compose configuration +`mem_limit: 400m`) +5 large studies are uploaded to each instance of Orthanc (around 1GB in total). +Each study is anonymized manually, one by one and then, we repeat the operation. +We compare the memory used by the containers after 2 minutes of execution +(using `docker stats`): + +# study without malloc_trim with_malloc_trim +0 ~ 50 MB ~ 50 MB +1 ~ 140 MB ~ 140 MB +2 ~ 390 MB ~ 340 MB +3 ~ 398 MB ~ 345 MB +4 out-of-memory crash ~ 345 MB +5..20 ~ 380 MB (stable) + +Note: the use of malloc_trim does not guarantee that Orthanc will never reach a +out-of-memory error, especially on very constrained systems. +Depending on the allocation pattern, the Orthanc memory can get +very fragmented and increase since malloc_trim only releases memory +at the end of each of malloc arena. However, note that, even long before the +introduction of malloc_trim, we have observed Orthanc instances running for years +without ever reaching out-of-memory errors and Orthanc is usually considered as +very stable. + + + + + + +malloc_trim documentation +------------------------- + +from (https://stackoverflow.com/questions/40513716/malloc-trim0-releases-fastbins-of-thread-arenas) + + If possible, gives memory back to the system (via negative + arguments to sbrk) if there is unused memory at the `high' end of + the malloc pool. You can call this after freeing large blocks of + memory to potentially reduce the system-level memory requirements + of a program. However, it cannot guarantee to reduce memory. Under + some allocation patterns, some large free blocks of memory will be + locked between two used chunks, so they cannot be given back to + the system. + + The `pad' argument to malloc_trim represents the amount of free + trailing space to leave untrimmed. If this argument is zero, + only the minimum amount of memory to maintain internal data + structures will be left (one page or less). Non-zero arguments + can be supplied to maintain enough trailing space to service + future expected allocations without having to re-obtain memory + from the system. + + Malloc_trim returns 1 if it actually released any memory, else 0. + On systems that do not support "negative sbrks", it will always + return 0. diff -r a72e00a23191 -r dce028ec0fe9 OrthancServer/Sources/ServerContext.cpp --- a/OrthancServer/Sources/ServerContext.cpp Tue Jan 31 16:21:44 2023 +0100 +++ b/OrthancServer/Sources/ServerContext.cpp Wed Feb 01 19:24:58 2023 +0100 @@ -48,7 +48,7 @@ #include #include - +#include static size_t DICOM_CACHE_SIZE = 128 * 1024 * 1024; // 128 MB @@ -104,6 +104,19 @@ { } + void ServerContext::HousekeeperThread(ServerContext* that, + unsigned int sleepDelay) + { + while (!that->done_) + { + boost::this_thread::sleep(boost::posix_time::milliseconds(sleepDelay)); + + // If possible, gives memory back to the system + // (see OrthancServer/Resources/ImplementationNotes/memory_consumption.txt) + + malloc_trim(256*1024); + } + } void ServerContext::ChangeThread(ServerContext* that, unsigned int sleepDelay) @@ -417,7 +430,8 @@ listeners_.push_back(ServerListener(luaListener_, "Lua")); changeThread_ = boost::thread(ChangeThread, this, (unitTesting ? 20 : 100)); - + housekeeperThread_ = boost::thread(HousekeeperThread, this, 100); + dynamic_cast(*dcmtkTranscoder_).SetLossyQuality(lossyQuality); } catch (OrthancException&) diff -r a72e00a23191 -r dce028ec0fe9 OrthancServer/Sources/ServerContext.h --- a/OrthancServer/Sources/ServerContext.h Tue Jan 31 16:21:44 2023 +0100 +++ b/OrthancServer/Sources/ServerContext.h Wed Feb 01 19:24:58 2023 +0100 @@ -187,6 +187,9 @@ static void SaveJobsThread(ServerContext* that, unsigned int sleepDelay); + static void HousekeeperThread(ServerContext* that, + unsigned int sleepDelay); + void SaveJobsEngine(); virtual void SignalJobSubmitted(const std::string& jobId) ORTHANC_OVERRIDE; @@ -230,6 +233,7 @@ SharedMessageQueue pendingChanges_; boost::thread changeThread_; boost::thread saveJobsThread_; + boost::thread housekeeperThread_; std::unique_ptr queryRetrieveArchive_; std::string defaultLocalAet_;