Mercurial > hg > orthanc-databases
changeset 501:594859656a06 large-queries
Added support for ExtendedApiV1: /changes
author | Alain Mazy <am@orthanc.team> |
---|---|
date | Thu, 11 Apr 2024 18:52:42 +0200 |
parents | c27071770c04 |
children | 2ab3d45c0b3c |
files | Framework/Common/DatabaseManager.cpp Framework/Common/DatabaseManager.h Framework/Common/StatementId.cpp Framework/Common/StatementId.h Framework/Common/StatementLocation.cpp Framework/Common/StatementLocation.h Framework/Plugins/DatabaseBackendAdapterV4.cpp Framework/Plugins/IDatabaseBackend.h Framework/Plugins/IndexBackend.cpp Framework/Plugins/IndexBackend.h MySQL/NEWS Odbc/NEWS PostgreSQL/NEWS Resources/CMake/DatabasesFrameworkConfiguration.cmake |
diffstat | 14 files changed, 324 insertions(+), 149 deletions(-) [+] |
line wrap: on
line diff
--- a/Framework/Common/DatabaseManager.cpp Tue Apr 09 15:47:30 2024 +0200 +++ b/Framework/Common/DatabaseManager.cpp Thu Apr 11 18:52:42 2024 +0200 @@ -78,9 +78,9 @@ } - IPrecompiledStatement* DatabaseManager::LookupCachedStatement(const StatementLocation& location) const + IPrecompiledStatement* DatabaseManager::LookupCachedStatement(const StatementId& statementId) const { - CachedStatements::const_iterator found = cachedStatements_.find(location); + CachedStatements::const_iterator found = cachedStatements_.find(statementId); if (found == cachedStatements_.end()) { @@ -94,10 +94,10 @@ } - IPrecompiledStatement& DatabaseManager::CacheStatement(const StatementLocation& location, + IPrecompiledStatement& DatabaseManager::CacheStatement(const StatementId& statementId, const Query& query) { - LOG(TRACE) << "Caching statement from " << location.GetFile() << ":" << location.GetLine(); + LOG(TRACE) << "Caching statement from " << statementId.GetFile() << ":" << statementId.GetLine() << "" << statementId.GetDynamicStatement(); std::unique_ptr<IPrecompiledStatement> statement(GetDatabase().Compile(query)); @@ -107,8 +107,8 @@ throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); } - assert(cachedStatements_.find(location) == cachedStatements_.end()); - cachedStatements_[location] = statement.release(); + assert(cachedStatements_.find(statementId) == cachedStatements_.end()); + cachedStatements_[statementId] = statement.release(); return *tmp; } @@ -550,13 +550,13 @@ } - DatabaseManager::CachedStatement::CachedStatement(const StatementLocation& location, + DatabaseManager::CachedStatement::CachedStatement(const StatementId& statementId, DatabaseManager& manager, const std::string& sql) : StatementBase(manager), - location_(location) + statementId_(statementId) { - statement_ = GetManager().LookupCachedStatement(location_); + statement_ = GetManager().LookupCachedStatement(statementId_); if (statement_ == NULL) { @@ -565,7 +565,7 @@ else { LOG(TRACE) << "Reusing cached statement from " - << location_.GetFile() << ":" << location_.GetLine(); + << statementId_.GetFile() << ":" << statementId_.GetLine() << " " << statementId_.GetDynamicStatement(); } } @@ -579,7 +579,7 @@ { // Register the newly-created statement assert(statement_ == NULL); - statement_ = &GetManager().CacheStatement(location_, *query); + statement_ = &GetManager().CacheStatement(statementId_, *query); } assert(statement_ != NULL);
--- a/Framework/Common/DatabaseManager.h Tue Apr 09 15:47:30 2024 +0200 +++ b/Framework/Common/DatabaseManager.h Thu Apr 11 18:52:42 2024 +0200 @@ -23,7 +23,7 @@ #pragma once #include "IDatabaseFactory.h" -#include "StatementLocation.h" +#include "StatementId.h" #include <Compatibility.h> // For std::unique_ptr<> #include <Enumerations.h> @@ -48,7 +48,7 @@ class DatabaseManager : public boost::noncopyable { private: - typedef std::map<StatementLocation, IPrecompiledStatement*> CachedStatements; + typedef std::map<StatementId, IPrecompiledStatement*> CachedStatements; std::unique_ptr<IDatabaseFactory> factory_; std::unique_ptr<IDatabase> database_; @@ -58,9 +58,9 @@ void CloseIfUnavailable(Orthanc::ErrorCode e); - IPrecompiledStatement* LookupCachedStatement(const StatementLocation& location) const; + IPrecompiledStatement* LookupCachedStatement(const StatementId& statementId) const; - IPrecompiledStatement& CacheStatement(const StatementLocation& location, + IPrecompiledStatement& CacheStatement(const StatementId& statementId, const Query& query); ITransaction& GetTransaction(); @@ -206,11 +206,11 @@ class CachedStatement : public StatementBase { private: - StatementLocation location_; + StatementId statementId_; IPrecompiledStatement* statement_; public: - CachedStatement(const StatementLocation& location, + CachedStatement(const StatementId& statementId, DatabaseManager& manager, const std::string& sql);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Framework/Common/StatementId.cpp Thu Apr 11 18:52:42 2024 +0200 @@ -0,0 +1,43 @@ +/** + * Orthanc - A Lightweight, RESTful DICOM Store + * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics + * Department, University Hospital of Liege, Belgium + * Copyright (C) 2017-2024 Osimis S.A., Belgium + * Copyright (C) 2021-2024 Sebastien Jodogne, ICTEAM UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +#include "StatementId.h" + +#include <string.h> + +namespace OrthancDatabases +{ + bool StatementId::operator< (const StatementId& other) const + { + if (line_ != other.line_) + { + return line_ < other.line_; + } + + if (strcmp(file_, other.file_) < 0) + { + return true; + } + + return statement_ < other.statement_; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Framework/Common/StatementId.h Thu Apr 11 18:52:42 2024 +0200 @@ -0,0 +1,76 @@ +/** + * Orthanc - A Lightweight, RESTful DICOM Store + * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics + * Department, University Hospital of Liege, Belgium + * Copyright (C) 2017-2024 Osimis S.A., Belgium + * Copyright (C) 2021-2024 Sebastien Jodogne, ICTEAM UCLouvain, Belgium + * + * This program is free software: you can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + **/ + + +#pragma once + +#include <string> + +#define STATEMENT_FROM_HERE ::OrthancDatabases::StatementId(__FILE__, __LINE__) +#define STATEMENT_FROM_HERE_DYNAMIC(sql) ::OrthancDatabases::StatementId(__FILE__, __LINE__, sql) + + +namespace OrthancDatabases +{ + class StatementId + { + private: + const char* file_; + int line_; + std::string statement_; + + StatementId(); // Forbidden + + public: + StatementId(const char* file, + int line) : + file_(file), + line_(line) + { + } + + StatementId(const char* file, + int line, + const std::string& statement) : + file_(file), + line_(line), + statement_(statement) + { + } + + const char* GetFile() const + { + return file_; + } + + int GetLine() const + { + return line_; + } + + const std::string& GetDynamicStatement() const + { + return statement_; + } + + bool operator< (const StatementId& other) const; + }; +}
--- a/Framework/Common/StatementLocation.cpp Tue Apr 09 15:47:30 2024 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,40 +0,0 @@ -/** - * Orthanc - A Lightweight, RESTful DICOM Store - * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics - * Department, University Hospital of Liege, Belgium - * Copyright (C) 2017-2024 Osimis S.A., Belgium - * Copyright (C) 2021-2024 Sebastien Jodogne, ICTEAM UCLouvain, Belgium - * - * This program is free software: you can redistribute it and/or - * modify it under the terms of the GNU Affero General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - **/ - - -#include "StatementLocation.h" - -#include <string.h> - -namespace OrthancDatabases -{ - bool StatementLocation::operator< (const StatementLocation& other) const - { - if (line_ != other.line_) - { - return line_ < other.line_; - } - else - { - return strcmp(file_, other.file_) < 0; - } - } -}
--- a/Framework/Common/StatementLocation.h Tue Apr 09 15:47:30 2024 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,58 +0,0 @@ -/** - * Orthanc - A Lightweight, RESTful DICOM Store - * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics - * Department, University Hospital of Liege, Belgium - * Copyright (C) 2017-2024 Osimis S.A., Belgium - * Copyright (C) 2021-2024 Sebastien Jodogne, ICTEAM UCLouvain, Belgium - * - * This program is free software: you can redistribute it and/or - * modify it under the terms of the GNU Affero General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - **/ - - -#pragma once - -#define STATEMENT_FROM_HERE ::OrthancDatabases::StatementLocation(__FILE__, __LINE__) - - -namespace OrthancDatabases -{ - class StatementLocation - { - private: - const char* file_; - int line_; - - StatementLocation(); // Forbidden - - public: - StatementLocation(const char* file, - int line) : - file_(file), - line_(line) - { - } - - const char* GetFile() const - { - return file_; - } - - int GetLine() const - { - return line_; - } - - bool operator< (const StatementLocation& other) const; - }; -}
--- a/Framework/Plugins/DatabaseBackendAdapterV4.cpp Tue Apr 09 15:47:30 2024 +0200 +++ b/Framework/Plugins/DatabaseBackendAdapterV4.cpp Thu Apr 11 18:52:42 2024 +0200 @@ -438,6 +438,9 @@ response.mutable_get_system_information()->set_has_measure_latency(accessor.GetBackend().HasMeasureLatency()); #endif +#if ORTHANC_PLUGINS_VERSION_IS_ABOVE(1, 13, 0) + response.mutable_get_system_information()->set_has_extended_api_v1(accessor.GetBackend().HasExtendedApiV1()); +#endif break; } @@ -782,7 +785,19 @@ response.mutable_get_changes()->set_done(done); break; } - +#if ORTHANC_PLUGINS_VERSION_IS_ABOVE(1, 13, 0) + case Orthanc::DatabasePluginMessages::OPERATION_GET_CHANGES_2: + { + Output output(*response.mutable_get_changes()); + + bool done; + backend.GetChanges2(output, done, manager, request.get_changes2().since(), request.get_changes2().to(), static_cast<OrthancPluginChangeType>(request.get_changes2().change_type()), request.get_changes2().limit()); + + response.mutable_get_changes()->set_done(done); + break; + } +#endif + case Orthanc::DatabasePluginMessages::OPERATION_GET_CHILDREN_INTERNAL_ID: { std::list<int64_t> values;
--- a/Framework/Plugins/IDatabaseBackend.h Tue Apr 09 15:47:30 2024 +0200 +++ b/Framework/Plugins/IDatabaseBackend.h Thu Apr 11 18:52:42 2024 +0200 @@ -109,6 +109,14 @@ int64_t since, uint32_t limit) = 0; + virtual void GetChanges2(IDatabaseBackendOutput& output, + bool& done /*out*/, + DatabaseManager& manager, + int64_t since, + int64_t to, + int32_t changeType, + uint32_t limit) = 0; + virtual void GetChildrenInternalId(std::list<int64_t>& target /*out*/, DatabaseManager& manager, int64_t id) = 0; @@ -376,6 +384,8 @@ // New in Orthanc 1.12.3 virtual uint64_t MeasureLatency(DatabaseManager& manager) = 0; + // New in Orthanc 1.13.0 + virtual bool HasExtendedApiV1() = 0; }; }
--- a/Framework/Plugins/IndexBackend.cpp Tue Apr 09 15:47:30 2024 +0200 +++ b/Framework/Plugins/IndexBackend.cpp Thu Apr 11 18:52:42 2024 +0200 @@ -116,28 +116,60 @@ DatabaseManager& manager, DatabaseManager::CachedStatement& statement, const Dictionary& args, - uint32_t limit) + uint32_t limit, + bool returnFirstResults) { + struct Change + { + int64_t seq_; + int32_t changeType_; + OrthancPluginResourceType resourceType_; + std::string publicId_; + std::string changeDate_; + + Change(int64_t seq, int32_t changeType, OrthancPluginResourceType resourceType, const std::string& publicId, const std::string& changeDate) + : seq_(seq), changeType_(changeType), resourceType_(resourceType), publicId_(publicId), changeDate_(changeDate) + { + } + }; + statement.Execute(args); - uint32_t count = 0; - - while (count < limit && - !statement.IsDone()) + std::list<Change> changes; + while (!statement.IsDone()) { - output.AnswerChange( + changes.push_back(Change( statement.ReadInteger64(0), statement.ReadInteger32(1), static_cast<OrthancPluginResourceType>(statement.ReadInteger32(2)), statement.ReadString(3), - statement.ReadString(4)); + statement.ReadString(4) + )); statement.Next(); - count++; } - - done = (count < limit || - statement.IsDone()); + + done = changes.size() <= limit; // 'done' means we have returned all requested changes + + // if we have retrieved more changes than requested -> cleanup + if (changes.size() > limit) + { + assert(changes.size() == limit+1); // the statement should only request 1 element more + + if (returnFirstResults) + { + changes.pop_back(); + } + else + { + changes.pop_front(); + } + } + + for (std::list<Change>::const_iterator it = changes.begin(); it != changes.end(); ++it) + { + output.AnswerChange(it->seq_, it->changeType_, it->resourceType_, it->publicId_, it->changeDate_); + } } @@ -553,39 +585,113 @@ ReadListOfStrings(target, statement, args); } - - /* Use GetOutput().AnswerChange() */ void IndexBackend::GetChanges(IDatabaseBackendOutput& output, bool& done /*out*/, DatabaseManager& manager, int64_t since, uint32_t limit) { - std::string suffix; +#if ORTHANC_PLUGINS_VERSION_IS_ABOVE(1, 13, 0) + GetChanges2(output, done, manager, since, -1, _OrthancPluginChangeType_All, limit); +#else + GetChanges2(output, done, manager, since, -1, 65535, limit); +#endif + } + + /* Use GetOutput().AnswerChange() */ + void IndexBackend::GetChanges2(IDatabaseBackendOutput& output, + bool& done /*out*/, + DatabaseManager& manager, + int64_t since, + int64_t to, + int32_t changeType, + uint32_t limit) + { + std::string limitSuffix; if (manager.GetDialect() == Dialect_MSSQL) { - suffix = "OFFSET 0 ROWS FETCH FIRST ${limit} ROWS ONLY"; + limitSuffix = "OFFSET 0 ROWS FETCH FIRST ${limit} ROWS ONLY"; } else { - suffix = "LIMIT ${limit}"; + limitSuffix = "LIMIT ${limit}"; } - DatabaseManager::CachedStatement statement( - STATEMENT_FROM_HERE, manager, - "SELECT Changes.seq, Changes.changeType, Changes.resourceType, Resources.publicId, " - "Changes.date FROM Changes INNER JOIN Resources " - "ON Changes.internalId = Resources.internalId WHERE seq>${since} ORDER BY seq " + suffix); - + std::vector<std::string> filters; + bool hasSince = false; + bool hasTo = false; + bool hasFilterType = false; + + if (since > 0) + { + hasSince = true; + filters.push_back("seq>${since}"); + } + if (to != -1) + { + hasTo = true; + filters.push_back("seq<=${to}"); + } + if (changeType != _OrthancPluginChangeType_All) + { + hasFilterType = true; + filters.push_back("changeType=${changeType}"); + } + + std::string filtersString; + if (filters.size() > 0) + { + Orthanc::Toolbox::JoinStrings(filtersString, filters, " AND "); + filtersString = "WHERE " + filtersString; + } + + std::string sql; + bool returnFirstResults; + if (hasTo && !hasSince) + { + // in this case, we want the largest values but we want them ordered in ascending order + sql = "SELECT * FROM (SELECT Changes.seq, Changes.changeType, Changes.resourceType, Resources.publicId, Changes.date " + "FROM Changes INNER JOIN Resources " + "ON Changes.internalId = Resources.internalId " + filtersString + " ORDER BY seq DESC " + limitSuffix + + ") AS FilteredChanges ORDER BY seq ASC"; + + returnFirstResults = false; + } + else + { + // default query: we want the smallest values ordered in ascending order + sql = "SELECT Changes.seq, Changes.changeType, Changes.resourceType, Resources.publicId, " + "Changes.date FROM Changes INNER JOIN Resources " + "ON Changes.internalId = Resources.internalId " + filtersString + " ORDER BY seq ASC " + limitSuffix; + returnFirstResults = true; + } + + DatabaseManager::CachedStatement statement(STATEMENT_FROM_HERE_DYNAMIC(sql), manager, sql); statement.SetReadOnly(true); + Dictionary args; + statement.SetParameterType("limit", ValueType_Integer64); - statement.SetParameterType("since", ValueType_Integer64); - - Dictionary args; - args.SetIntegerValue("limit", limit + 1); - args.SetIntegerValue("since", since); - - ReadChangesInternal(output, done, manager, statement, args, limit); + args.SetIntegerValue("limit", limit + 1); // we take limit+1 because we use the +1 to know if "Done" must be set to true + + if (hasSince) + { + statement.SetParameterType("since", ValueType_Integer64); + args.SetIntegerValue("since", since); + } + + if (hasTo) + { + statement.SetParameterType("to", ValueType_Integer64); + args.SetIntegerValue("to", to); + } + + if (hasFilterType) + { + statement.SetParameterType("changeType", ValueType_Integer64); + args.SetIntegerValue("changeType", changeType); + } + + ReadChangesInternal(output, done, manager, statement, args, limit, returnFirstResults); } @@ -685,7 +791,7 @@ Dictionary args; bool done; // Ignored - ReadChangesInternal(output, done, manager, statement, args, 1); + ReadChangesInternal(output, done, manager, statement, args, 1, true); }
--- a/Framework/Plugins/IndexBackend.h Tue Apr 09 15:47:30 2024 +0200 +++ b/Framework/Plugins/IndexBackend.h Thu Apr 11 18:52:42 2024 +0200 @@ -64,7 +64,8 @@ DatabaseManager& manager, DatabaseManager::CachedStatement& statement, const Dictionary& args, - uint32_t limit); + uint32_t limit, + bool returnFirstResults); void ReadExportedResourcesInternal(IDatabaseBackendOutput& output, bool& done, @@ -129,7 +130,15 @@ DatabaseManager& manager, int64_t since, uint32_t limit) ORTHANC_OVERRIDE; - + + virtual void GetChanges2(IDatabaseBackendOutput& output, + bool& done /*out*/, + DatabaseManager& manager, + int64_t since, + int64_t to, + int32_t changeType, + uint32_t limit) ORTHANC_OVERRIDE; + virtual void GetChildrenInternalId(std::list<int64_t>& target /*out*/, DatabaseManager& manager, int64_t id) ORTHANC_OVERRIDE; @@ -419,6 +428,13 @@ virtual uint64_t MeasureLatency(DatabaseManager& manager) ORTHANC_OVERRIDE; + // New primitive since Orthanc 1.13.0 + virtual bool HasExtendedApiV1() ORTHANC_OVERRIDE + { + return true; + } + + /** * "maxDatabaseRetries" is to handle * "OrthancPluginErrorCode_DatabaseCannotSerialize" if there is a
--- a/MySQL/NEWS Tue Apr 09 15:47:30 2024 +0200 +++ b/MySQL/NEWS Thu Apr 11 18:52:42 2024 +0200 @@ -7,6 +7,8 @@ Optimal Orthanc runtime: 1.12.0+ * Fix check of Orthanc runtime version +* Added support for ExtendedApiV1: + - changes?type=...&to=... Release 5.1 (2023-06-27)
--- a/Odbc/NEWS Tue Apr 09 15:47:30 2024 +0200 +++ b/Odbc/NEWS Thu Apr 11 18:52:42 2024 +0200 @@ -7,6 +7,8 @@ Optimal Orthanc runtime: 1.12.0+ * Fix check of Orthanc runtime version +* Added support for ExtendedApiV1: + - changes?type=...&to=... Release 1.2 (2024-03-06)
--- a/PostgreSQL/NEWS Tue Apr 09 15:47:30 2024 +0200 +++ b/PostgreSQL/NEWS Thu Apr 11 18:52:42 2024 +0200 @@ -6,6 +6,9 @@ Minimum Orthanc runtime: 1.12.3 * Fix updates from plugin version 3.3 to latest version +* Added support for ExtendedApiV1: + - changes?type=...&to=... + Release 6.2 (2024-03-25)
--- a/Resources/CMake/DatabasesFrameworkConfiguration.cmake Tue Apr 09 15:47:30 2024 +0200 +++ b/Resources/CMake/DatabasesFrameworkConfiguration.cmake Thu Apr 11 18:52:42 2024 +0200 @@ -119,7 +119,7 @@ ${ORTHANC_DATABASES_ROOT}/Framework/Common/ResultFileValue.cpp ${ORTHANC_DATABASES_ROOT}/Framework/Common/RetryDatabaseFactory.cpp ${ORTHANC_DATABASES_ROOT}/Framework/Common/RetryDatabaseFactory.cpp - ${ORTHANC_DATABASES_ROOT}/Framework/Common/StatementLocation.cpp + ${ORTHANC_DATABASES_ROOT}/Framework/Common/StatementId.cpp ${ORTHANC_DATABASES_ROOT}/Framework/Common/Utf8StringValue.cpp )