Mercurial > hg > orthanc
changeset 5881:1c549458ea5d find-refactoring tip
UpdateStatistics Thread -> DbHousekeeping
| author | Alain Mazy <am@orthanc.team> |
|---|---|
| date | Wed, 27 Nov 2024 16:02:42 +0100 |
| parents | 00f650dc07bf |
| children | |
| files | NEWS OrthancServer/Plugins/Engine/OrthancPluginDatabaseV4.cpp OrthancServer/Plugins/Include/orthanc/OrthancDatabasePlugin.proto OrthancServer/Resources/ImplementationNotes/DatabasesClassHierarchy.txt OrthancServer/Sources/Database/BaseDatabaseWrapper.cpp OrthancServer/Sources/Database/BaseDatabaseWrapper.h OrthancServer/Sources/Database/IDatabaseWrapper.h OrthancServer/Sources/Database/StatelessDatabaseOperations.cpp OrthancServer/Sources/Database/StatelessDatabaseOperations.h OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp OrthancServer/Sources/ServerIndex.cpp OrthancServer/Sources/ServerIndex.h |
| diffstat | 12 files changed, 91 insertions(+), 37 deletions(-) [+] |
line wrap: on
line diff
--- a/NEWS Wed Nov 27 15:00:58 2024 +0100 +++ b/NEWS Wed Nov 27 16:02:42 2024 +0100 @@ -52,10 +52,9 @@ * DICOM TLS: "DicomTlsTrustedCertificates" is not required anymore when issuing an outgoing SCU connexion when "DicomTlsRemoteCertificateRequired" is set to false. -* Introduced a new thread to update the statistics at regular interval for the - DB plugins that are implementing the UpdateAndGetStatistics function (currently only - PostgreSQL). This avoids very long update times in case you don't call /statistics - for a long period. +* Introduced a new thread to perform DB Housekeeping at regular interval (1s) for the + DB plugins requiring it (currently only PostgreSQL). E.g: This avoids very long update + times in case you don't call /statistics for a long period. * Fix C-Find queries not returning computed tags like ModalitiesInStudy, NumberOfStudyRelatedSeries, ... in very specific use-cases. * Fix extremely rare error when 2 threads are trying to create the same folder in the File Storage
--- a/OrthancServer/Plugins/Engine/OrthancPluginDatabaseV4.cpp Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Plugins/Engine/OrthancPluginDatabaseV4.cpp Wed Nov 27 16:02:42 2024 +0100 @@ -1063,6 +1063,12 @@ uncompressedSize = response.update_and_get_statistics().total_uncompressed_size(); } + virtual void PerformDbHousekeeping() ORTHANC_OVERRIDE + { + DatabasePluginMessages::TransactionResponse response; + ExecuteTransaction(response, DatabasePluginMessages::OPERATION_PERFORM_DB_HOUSEKEEPING); + } + virtual bool LookupMetadata(std::string& target, int64_t& revision, int64_t id, @@ -1881,10 +1887,11 @@ dbCapabilities_.SetRevisionsSupport(systemInfo.supports_revisions()); dbCapabilities_.SetLabelsSupport(systemInfo.supports_labels()); dbCapabilities_.SetAtomicIncrementGlobalProperty(systemInfo.supports_increment_global_property()); - dbCapabilities_.SetUpdateAndGetStatistics(systemInfo.has_update_and_get_statistics()); + dbCapabilities_.SetHasUpdateAndGetStatistics(systemInfo.has_update_and_get_statistics()); dbCapabilities_.SetMeasureLatency(systemInfo.has_measure_latency()); dbCapabilities_.SetHasExtendedChanges(systemInfo.has_extended_changes()); dbCapabilities_.SetHasFindSupport(systemInfo.supports_find()); + dbCapabilities_.SetHasDbHousekeeping(systemInfo.has_db_housekeeping()); } open_ = true;
--- a/OrthancServer/Plugins/Include/orthanc/OrthancDatabasePlugin.proto Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Plugins/Include/orthanc/OrthancDatabasePlugin.proto Wed Nov 27 16:02:42 2024 +0100 @@ -160,6 +160,7 @@ bool has_measure_latency = 7; bool supports_find = 8; // New in Orthanc 1.12.5 bool has_extended_changes = 9; // New in Orthanc 1.12.5 + bool has_db_housekeeping = 10; // New in Orthanc 1.12.5 } } @@ -315,6 +316,7 @@ OPERATION_FIND = 50; // New in Orthanc 1.12.5 OPERATION_GET_CHANGES_EXTENDED = 51; // New in Orthanc 1.12.5 OPERATION_COUNT_RESOURCES = 52; // New in Orthanc 1.12.5 + OPERATION_PERFORM_DB_HOUSEKEEPING = 53; // New in Orthanc 1.12.5 } message Rollback { @@ -705,6 +707,13 @@ } } +message PerformDbHousekeeping { + message Request { + } + message Response { + } +} + message ClearMainDicomTags { message Request { int64 id = 1; @@ -1033,6 +1042,7 @@ Find.Request find = 150; GetChangesExtended.Request get_changes_extended = 151; Find.Request count_resources = 152; + PerformDbHousekeeping.Request perform_db_housekeeping = 153; } message TransactionResponse { @@ -1089,6 +1099,7 @@ repeated Find.Response find = 150; // One message per found resources GetChangesExtended.Response get_changes_extended = 151; CountResources.Response count_resources = 152; + PerformDbHousekeeping.Response perform_db_housekeeping = 153; } enum RequestType {
--- a/OrthancServer/Resources/ImplementationNotes/DatabasesClassHierarchy.txt Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Resources/ImplementationNotes/DatabasesClassHierarchy.txt Wed Nov 27 16:02:42 2024 +0100 @@ -21,7 +21,7 @@ - define it in OrthancPluginDatabaseV4 - define a NotImplemented default implementation in BaseDatabaseWrapper - optionally define it in SQLiteDatabaseWrapper if it can be implemented in SQLite -- very likely define it as a DbCapabilities in IDatabaseWrapper::DbCapabilities (e.g: Has/SetUpdateAndGetStatistics()) such that the Orthanc +- very likely define it as a DbCapabilities in IDatabaseWrapper::DbCapabilities (e.g: Has/SetHasUpdateAndGetStatistics()) such that the Orthanc core knows if it can use it or not. Then, in the orthanc-databases repo, you should:
--- a/OrthancServer/Sources/Database/BaseDatabaseWrapper.cpp Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Sources/Database/BaseDatabaseWrapper.cpp Wed Nov 27 16:02:42 2024 +0100 @@ -46,6 +46,11 @@ throw OrthancException(ErrorCode_NotImplemented); // Not supported } + void BaseDatabaseWrapper::BaseTransaction::PerformDbHousekeeping() + { + throw OrthancException(ErrorCode_NotImplemented); // Not supported + } + void BaseDatabaseWrapper::BaseTransaction::GetChangesExtended(std::list<ServerIndexChange>& target /*out*/, bool& done /*out*/, int64_t since,
--- a/OrthancServer/Sources/Database/BaseDatabaseWrapper.h Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Sources/Database/BaseDatabaseWrapper.h Wed Nov 27 16:02:42 2024 +0100 @@ -50,6 +50,8 @@ int64_t& compressedSize, int64_t& uncompressedSize) ORTHANC_OVERRIDE; + virtual void PerformDbHousekeeping() ORTHANC_OVERRIDE; + virtual void ExecuteCount(uint64_t& count, const FindRequest& request, const Capabilities& capabilities) ORTHANC_OVERRIDE;
--- a/OrthancServer/Sources/Database/IDatabaseWrapper.h Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Sources/Database/IDatabaseWrapper.h Wed Nov 27 16:02:42 2024 +0100 @@ -56,6 +56,7 @@ bool hasMeasureLatency_; bool hasFindSupport_; bool hasExtendedChanges_; + bool hasDbHousekeeping_; public: Capabilities() : @@ -66,7 +67,8 @@ hasUpdateAndGetStatistics_(false), hasMeasureLatency_(false), hasFindSupport_(false), - hasExtendedChanges_(false) + hasExtendedChanges_(false), + hasDbHousekeeping_(false) { } @@ -120,7 +122,7 @@ return hasAtomicIncrementGlobalProperty_; } - void SetUpdateAndGetStatistics(bool value) + void SetHasUpdateAndGetStatistics(bool value) { hasUpdateAndGetStatistics_ = value; } @@ -130,6 +132,16 @@ return hasUpdateAndGetStatistics_; } + void SetHasDbHousekeeping(bool value) + { + hasDbHousekeeping_ = value; + } + + bool HasDbHousekeeping() const + { + return hasDbHousekeeping_; + } + void SetMeasureLatency(bool value) { hasMeasureLatency_ = value; @@ -401,6 +413,9 @@ int64_t to, uint32_t limit, const std::set<ChangeType>& filterType) = 0; + + // New in Orthanc 1.12.5 + virtual void PerformDbHousekeeping() = 0; };
--- a/OrthancServer/Sources/Database/StatelessDatabaseOperations.cpp Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Sources/Database/StatelessDatabaseOperations.cpp Wed Nov 27 16:02:42 2024 +0100 @@ -3476,4 +3476,24 @@ } } } + + void StatelessDatabaseOperations::PerformDbHousekeeping() + { + class Operations : public IReadWriteOperations + { + public: + Operations() + { + } + + virtual void Apply(ReadWriteTransaction& transaction) ORTHANC_OVERRIDE + { + transaction.PerformDbHousekeeping(); + } + }; + + Operations operations; + Apply(operations); + } + }
--- a/OrthancServer/Sources/Database/StatelessDatabaseOperations.h Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Sources/Database/StatelessDatabaseOperations.h Wed Nov 27 16:02:42 2024 +0100 @@ -406,6 +406,11 @@ return transaction_.UpdateAndGetStatistics(patientsCount, studiesCount, seriesCount, instancesCount, compressedSize, uncompressedSize); } + void PerformDbHousekeeping() + { + return transaction_.PerformDbHousekeeping(); + } + void SetMetadata(int64_t id, MetadataType type, const std::string& value, @@ -735,5 +740,7 @@ void ExecuteCount(uint64_t& count, const FindRequest& request); + void PerformDbHousekeeping(); + }; }
--- a/OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Sources/OrthancRestApi/OrthancRestResources.cpp Wed Nov 27 16:02:42 2024 +0100 @@ -3101,18 +3101,8 @@ "Limit the number of reported resources", false) .SetRequestField(KEY_SINCE, RestApiCallDocumentation::Type_Number, "Show only the resources since the provided index (in conjunction with `Limit`)", false) - // .SetRequestField(KEY_REQUESTED_TAGS, RestApiCallDocumentation::Type_JsonListOfStrings, - // "A list of DICOM tags to include in the response (applicable only if \"Expand\" is set to true). " - // "The tags requested tags are returned in the 'RequestedTags' field in the response. " - // "Note that, if you are requesting tags that are not listed in the Main Dicom Tags stored in DB, building the response " - // "might be slow since Orthanc will need to access the DICOM files. If not specified, Orthanc will return " - // "all Main Dicom Tags to keep backward compatibility with Orthanc prior to 1.11.0.", false) .SetRequestField(KEY_ORDER_BY, RestApiCallDocumentation::Type_JsonListOfObjects, "Array of associative arrays containing the requested ordering (new in Orthanc 1.12.5)", true) - // .SetRequestField(KEY_RESPONSE_CONTENT, RestApiCallDocumentation::Type_JsonListOfStrings, - // "Defines the content of response for each returned resource. Allowed values are `MainDicomTags`, " - // "`Metadata`, `Children`, `Parent`, `Labels`, `Status`, `IsStable`, `Attachments`. " - // "(new in Orthanc 1.12.5)", true) .AddAnswerType(MimeType_Json, "JSON array containing either the Orthanc identifiers, or detailed information " "about the reported resources (if `Expand` argument is `true`)");
--- a/OrthancServer/Sources/ServerIndex.cpp Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Sources/ServerIndex.cpp Wed Nov 27 16:02:42 2024 +0100 @@ -266,19 +266,19 @@ } }; - void ServerIndex::UpdateStatisticsThread(ServerIndex* that, - unsigned int threadSleepGranularityMilliseconds) + void ServerIndex::PerformDbHouskeeping(ServerIndex* that, + unsigned int threadSleepGranularityMilliseconds) { - Logging::SetCurrentThreadName("DB-STATS"); + Logging::SetCurrentThreadName("DB-HK"); - static const unsigned int SLEEP_SECONDS = 10; + static const unsigned int SLEEP_SECONDS = 1; if (threadSleepGranularityMilliseconds > 1000) { throw OrthancException(ErrorCode_ParameterOutOfRange); } - LOG(INFO) << "Starting the update statistics thread (sleep = " << SLEEP_SECONDS << " seconds)"; + LOG(INFO) << "Starting the DB Housekeeping thread (sleep = " << SLEEP_SECONDS << " seconds)"; unsigned int count = 0; unsigned int countThreshold = (1000 * SLEEP_SECONDS) / threadSleepGranularityMilliseconds; @@ -290,14 +290,13 @@ if (count >= countThreshold) { - uint64_t diskSize, uncompressedSize, countPatients, countStudies, countSeries, countInstances; - that->GetGlobalStatistics(diskSize, uncompressedSize, countPatients, countStudies, countSeries, countInstances); + that->PerformDbHousekeeping(); count = 0; } } - LOG(INFO) << "Stopping the update statistics thread"; + LOG(INFO) << "Stopping the DB Housekeeping thread"; } void ServerIndex::FlushThread(ServerIndex* that, @@ -377,18 +376,17 @@ } } - // For some DB plugins that implements the UpdateAndGetStatistics function, updating - // the statistics can take quite some time if you have not done it for a long time - // -> make sure they are updated at regular interval - if (GetDatabaseCapabilities().HasUpdateAndGetStatistics()) + // For some DB plugins, some housekeeping might be required at regular interval + // like computing the statistics or update some tables after an upgrade + if (GetDatabaseCapabilities().HasDbHousekeeping()) { if (readOnly) { - LOG(WARNING) << "READ-ONLY SYSTEM: not starting the UpdateStatisticsThread"; + LOG(WARNING) << "READ-ONLY SYSTEM: not starting the DB Housekeeping thread"; } else { - updateStatisticsThread_ = boost::thread(UpdateStatisticsThread, this, threadSleepGranularityMilliseconds); + dbHousekeepingThread_ = boost::thread(PerformDbHouskeeping, this, threadSleepGranularityMilliseconds); } } @@ -425,9 +423,9 @@ flushThread_.join(); } - if (updateStatisticsThread_.joinable()) + if (dbHousekeepingThread_.joinable()) { - updateStatisticsThread_.join(); + dbHousekeepingThread_.join(); } if (unstableResourcesMonitorThread_.joinable())
--- a/OrthancServer/Sources/ServerIndex.h Wed Nov 27 15:00:58 2024 +0100 +++ b/OrthancServer/Sources/ServerIndex.h Wed Nov 27 16:02:42 2024 +0100 @@ -42,7 +42,7 @@ bool done_; boost::mutex monitoringMutex_; boost::thread flushThread_; - boost::thread updateStatisticsThread_; + boost::thread dbHousekeepingThread_; boost::thread unstableResourcesMonitorThread_; LeastRecentlyUsedIndex<std::pair<ResourceType, int64_t>, UnstableResourcePayload> unstableResources_; @@ -55,8 +55,8 @@ static void FlushThread(ServerIndex* that, unsigned int threadSleep); - static void UpdateStatisticsThread(ServerIndex* that, - unsigned int threadSleep); + static void PerformDbHouskeeping(ServerIndex* that, + unsigned int threadSleep); static void UnstableResourcesMonitorThread(ServerIndex* that, unsigned int threadSleep);