# HG changeset patch # User Sebastien Jodogne # Date 1548864735 -3600 # Node ID fca730c267d7716d8eeb5472052beb9206e83d04 # Parent 6d558598d7133b85e8baac28b74f17495cfe769d New primitives to set and refresh metrics diff -r 6d558598d713 -r fca730c267d7 Core/JobsEngine/JobsRegistry.cpp --- a/Core/JobsEngine/JobsRegistry.cpp Wed Jan 30 12:41:20 2019 +0100 +++ b/Core/JobsEngine/JobsRegistry.cpp Wed Jan 30 17:12:15 2019 +0100 @@ -1402,14 +1402,16 @@ void JobsRegistry::GetStatistics(unsigned int& pending, unsigned int& running, - unsigned int& completed) + unsigned int& success, + unsigned int& failed) { boost::mutex::scoped_lock lock(mutex_); CheckInvariants(); pending = 0; running = 0; - completed = 0; + success = 0; + failed = 0; for (JobsIndex::const_iterator it = jobsIndex_.begin(); it != jobsIndex_.end(); ++it) @@ -1419,18 +1421,21 @@ switch (job.GetState()) { case JobState_Retry: - case JobState_Paused: case JobState_Pending: pending ++; break; + case JobState_Paused: case JobState_Running: running ++; break; case JobState_Success: + success ++; + break; + case JobState_Failure: - completed ++; + failed ++; break; default: diff -r 6d558598d713 -r fca730c267d7 Core/JobsEngine/JobsRegistry.h --- a/Core/JobsEngine/JobsRegistry.h Wed Jan 30 12:41:20 2019 +0100 +++ b/Core/JobsEngine/JobsRegistry.h Wed Jan 30 17:12:15 2019 +0100 @@ -201,7 +201,8 @@ void GetStatistics(unsigned int& pending, unsigned int& running, - unsigned int& completed); + unsigned int& success, + unsigned int& errors); class RunningJob : public boost::noncopyable { diff -r 6d558598d713 -r fca730c267d7 LinuxCompilation.txt --- a/LinuxCompilation.txt Wed Jan 30 12:41:20 2019 +0100 +++ b/LinuxCompilation.txt Wed Jan 30 17:12:15 2019 +0100 @@ -130,6 +130,20 @@ -DDCMTK_LIBRARIES=dcmjpls \ -DCMAKE_BUILD_TYPE=Release \ ~/Orthanc +# make + + +NB: Instructions to use clang and ninja: + +# sudo apt-get install ninja-build +# CC=/usr/bin/clang CXX=/usr/bin/clang++ cmake -G Ninja \ + -DALLOW_DOWNLOADS=ON \ + -DUSE_GOOGLE_TEST_DEBIAN_PACKAGE=ON \ + -DUSE_SYSTEM_CIVETWEB=OFF \ + -DDCMTK_LIBRARIES=dcmjpls \ + -DCMAKE_BUILD_TYPE=Release \ + ~/Orthanc +# ninja diff -r 6d558598d713 -r fca730c267d7 NEWS --- a/NEWS Wed Jan 30 12:41:20 2019 +0100 +++ b/NEWS Wed Jan 30 17:12:15 2019 +0100 @@ -13,6 +13,11 @@ * New URI "/tools/metrics" to dynamically enable/disable the collection of metrics * New URI "/tools/metrics-prometheus" to retrieve metrics using Prometheus text format +Plugins +------- + +* New primitives to set and refresh metrics + Maintenance ----------- diff -r 6d558598d713 -r fca730c267d7 OrthancServer/OrthancRestApi/OrthancRestSystem.cpp --- a/OrthancServer/OrthancRestApi/OrthancRestSystem.cpp Wed Jan 30 12:41:20 2019 +0100 +++ b/OrthancServer/OrthancRestApi/OrthancRestSystem.cpp Wed Jan 30 17:12:15 2019 +0100 @@ -407,6 +407,10 @@ static void GetMetricsPrometheus(RestApiGetCall& call) { +#if ORTHANC_ENABLE_PLUGINS == 1 + OrthancRestApi::GetContext(call).GetPlugins().RefreshMetrics(); +#endif + static const float MEGA_BYTES = 1024 * 1024; ServerContext& context = OrthancRestApi::GetContext(call); @@ -415,8 +419,8 @@ context.GetIndex().GetGlobalStatistics(diskSize, uncompressedSize, countPatients, countStudies, countSeries, countInstances); - unsigned int jobsPending, jobsRunning, jobsCompleted; - context.GetJobsEngine().GetRegistry().GetStatistics(jobsPending, jobsRunning, jobsCompleted); + unsigned int jobsPending, jobsRunning, jobsSuccess, jobsFailed; + context.GetJobsEngine().GetRegistry().GetStatistics(jobsPending, jobsRunning, jobsSuccess, jobsFailed); MetricsRegistry& registry = context.GetMetricsRegistry(); registry.SetValue("orthanc_disk_size_mb", static_cast(diskSize) / MEGA_BYTES); @@ -427,7 +431,9 @@ registry.SetValue("orthanc_count_instances", static_cast(countInstances)); registry.SetValue("orthanc_jobs_pending", jobsPending); registry.SetValue("orthanc_jobs_running", jobsRunning); - registry.SetValue("orthanc_jobs_completed", jobsCompleted); + registry.SetValue("orthanc_jobs_completed", jobsSuccess + jobsFailed); + registry.SetValue("orthanc_jobs_success", jobsSuccess); + registry.SetValue("orthanc_jobs_failed", jobsFailed); std::string s; registry.ExportPrometheusText(s); diff -r 6d558598d713 -r fca730c267d7 Plugins/Engine/OrthancPlugins.cpp --- a/Plugins/Engine/OrthancPlugins.cpp Wed Jan 30 12:41:20 2019 +0100 +++ b/Plugins/Engine/OrthancPlugins.cpp Wed Jan 30 17:12:15 2019 +0100 @@ -44,29 +44,30 @@ #include "../../Core/ChunkedBuffer.h" +#include "../../Core/Compression/GzipCompressor.h" +#include "../../Core/Compression/ZlibCompressor.h" #include "../../Core/DicomFormat/DicomArray.h" +#include "../../Core/DicomParsing/FromDcmtkBridge.h" +#include "../../Core/DicomParsing/Internals/DicomImageDecoder.h" +#include "../../Core/DicomParsing/ToDcmtkBridge.h" #include "../../Core/HttpServer/HttpToolbox.h" +#include "../../Core/Images/Image.h" +#include "../../Core/Images/ImageProcessing.h" +#include "../../Core/Images/JpegReader.h" +#include "../../Core/Images/JpegWriter.h" +#include "../../Core/Images/PngReader.h" +#include "../../Core/Images/PngWriter.h" #include "../../Core/Logging.h" +#include "../../Core/MetricsRegistry.h" #include "../../Core/OrthancException.h" #include "../../Core/SerializationToolbox.h" #include "../../Core/Toolbox.h" -#include "../../Core/DicomParsing/FromDcmtkBridge.h" -#include "../../Core/DicomParsing/ToDcmtkBridge.h" +#include "../../OrthancServer/DefaultDicomImageDecoder.h" #include "../../OrthancServer/OrthancConfiguration.h" +#include "../../OrthancServer/OrthancFindRequestHandler.h" +#include "../../OrthancServer/Search/HierarchicalMatcher.h" #include "../../OrthancServer/ServerContext.h" #include "../../OrthancServer/ServerToolbox.h" -#include "../../OrthancServer/Search/HierarchicalMatcher.h" -#include "../../Core/DicomParsing/Internals/DicomImageDecoder.h" -#include "../../Core/Compression/ZlibCompressor.h" -#include "../../Core/Compression/GzipCompressor.h" -#include "../../Core/Images/Image.h" -#include "../../Core/Images/PngReader.h" -#include "../../Core/Images/PngWriter.h" -#include "../../Core/Images/JpegReader.h" -#include "../../Core/Images/JpegWriter.h" -#include "../../Core/Images/ImageProcessing.h" -#include "../../OrthancServer/DefaultDicomImageDecoder.h" -#include "../../OrthancServer/OrthancFindRequestHandler.h" #include "PluginsEnumerations.h" #include "PluginsJob.h" @@ -462,6 +463,7 @@ typedef std::list IncomingHttpRequestFilters2; typedef std::list DecodeImageCallbacks; typedef std::list JobsUnserializers; + typedef std::list RefreshMetricsCallbacks; typedef std::map Properties; PluginsManager manager_; @@ -476,6 +478,7 @@ _OrthancPluginMoveCallback moveCallbacks_; IncomingHttpRequestFilters incomingHttpRequestFilters_; IncomingHttpRequestFilters2 incomingHttpRequestFilters2_; + RefreshMetricsCallbacks refreshMetricsCallbacks_; std::auto_ptr storageArea_; boost::recursive_mutex restCallbackMutex_; @@ -485,6 +488,7 @@ boost::mutex worklistCallbackMutex_; boost::mutex decodeImageCallbackMutex_; boost::mutex jobsUnserializersMutex_; + boost::mutex refreshMetricsMutex_; boost::recursive_mutex invokeServiceMutex_; Properties properties_; @@ -1310,6 +1314,18 @@ } + void OrthancPlugins::RegisterRefreshMetricsCallback(const void* parameters) + { + const _OrthancPluginRegisterRefreshMetricsCallback& p = + *reinterpret_cast(parameters); + + boost::mutex::scoped_lock lock(pimpl_->refreshMetricsMutex_); + + LOG(INFO) << "Plugin has registered a callback to refresh its metrics"; + pimpl_->refreshMetricsCallbacks_.push_back(p.callback); + } + + void OrthancPlugins::AnswerBuffer(const void* parameters) { const _OrthancPluginAnswerBuffer& p = @@ -3096,6 +3112,34 @@ return true; } + case _OrthancPluginService_SetMetricsValue: + { + const _OrthancPluginSetMetricsValue& p = + *reinterpret_cast(parameters); + + MetricsType type; + switch (p.type) + { + case OrthancPluginMetricsType_Default: + type = MetricsType_Default; + break; + + case OrthancPluginMetricsType_Timer: + type = MetricsType_MaxOver10Seconds; + break; + + default: + throw OrthancException(ErrorCode_ParameterOutOfRange); + } + + { + PImpl::ServerContextLock lock(*pimpl_); + lock.GetContext().GetMetricsRegistry().SetValue(p.name, p.value, type); + } + + return true; + } + default: return false; } @@ -3157,6 +3201,10 @@ RegisterIncomingHttpRequestFilter2(parameters); return true; + case _OrthancPluginService_RegisterRefreshMetricsCallback: + RegisterRefreshMetricsCallback(parameters); + return true; + case _OrthancPluginService_RegisterStorageArea: { LOG(INFO) << "Plugin has registered a custom storage area"; @@ -3659,4 +3707,20 @@ return NULL; } + + + void OrthancPlugins::RefreshMetrics() + { + boost::mutex::scoped_lock lock(pimpl_->refreshMetricsMutex_); + + for (PImpl::RefreshMetricsCallbacks::iterator + it = pimpl_->refreshMetricsCallbacks_.begin(); + it != pimpl_->refreshMetricsCallbacks_.end(); ++it) + { + if (*it != NULL) + { + (*it) (); + } + } + } } diff -r 6d558598d713 -r fca730c267d7 Plugins/Engine/OrthancPlugins.h --- a/Plugins/Engine/OrthancPlugins.h Wed Jan 30 12:41:20 2019 +0100 +++ b/Plugins/Engine/OrthancPlugins.h Wed Jan 30 17:12:15 2019 +0100 @@ -111,6 +111,8 @@ void RegisterIncomingHttpRequestFilter2(const void* parameters); + void RegisterRefreshMetricsCallback(const void* parameters); + void AnswerBuffer(const void* parameters); void Redirect(const void* parameters); @@ -313,6 +315,8 @@ IJob* UnserializeJob(const std::string& type, const Json::Value& value); + + void RefreshMetrics(); }; } diff -r 6d558598d713 -r fca730c267d7 Plugins/Include/orthanc/OrthancCPlugin.h --- a/Plugins/Include/orthanc/OrthancCPlugin.h Wed Jan 30 12:41:20 2019 +0100 +++ b/Plugins/Include/orthanc/OrthancCPlugin.h Wed Jan 30 17:12:15 2019 +0100 @@ -24,6 +24,7 @@ * - Possibly register a custom decoder for DICOM images using OrthancPluginRegisterDecodeImageCallback(). * - Possibly register a callback to filter incoming HTTP requests using OrthancPluginRegisterIncomingHttpRequestFilter2(). * - Possibly register a callback to unserialize jobs using OrthancPluginRegisterJobsUnserializer(). + * - Possibly register a callback to refresh its metrics using OrthancPluginRegisterRefreshMetricsCallback(). * -# void OrthancPluginFinalize(): * This function is invoked by Orthanc during its shutdown. The plugin * must free all its memory. @@ -119,7 +120,7 @@ #define ORTHANC_PLUGINS_MINIMAL_MAJOR_NUMBER 1 #define ORTHANC_PLUGINS_MINIMAL_MINOR_NUMBER 5 -#define ORTHANC_PLUGINS_MINIMAL_REVISION_NUMBER 2 +#define ORTHANC_PLUGINS_MINIMAL_REVISION_NUMBER 4 #if !defined(ORTHANC_PLUGINS_VERSION_IS_ABOVE) @@ -424,6 +425,7 @@ _OrthancPluginService_GenerateUuid = 28, _OrthancPluginService_RegisterPrivateDictionaryTag = 29, _OrthancPluginService_AutodetectMimeType = 30, + _OrthancPluginService_SetMetricsValue = 31, /* Registration of callbacks */ _OrthancPluginService_RegisterRestCallback = 1000, @@ -437,6 +439,7 @@ _OrthancPluginService_RegisterFindCallback = 1008, _OrthancPluginService_RegisterMoveCallback = 1009, _OrthancPluginService_RegisterIncomingHttpRequestFilter2 = 1010, + _OrthancPluginService_RegisterRefreshMetricsCallback = 1011, /* Sending answers to REST calls */ _OrthancPluginService_AnswerBuffer = 2000, @@ -892,6 +895,23 @@ /** + * The available type of metrics. + **/ + typedef enum + { + OrthancPluginMetricsType_Default, /*!< Default metrics */ + + /** + * This metrics represents a time duration. Orthanc will keep the + * maximum value of the metrics over a sliding window of ten + * seconds, which is useful if the metrics is sampled frequently. + **/ + OrthancPluginMetricsType_Timer + } OrthancPluginMetricsType; + + + + /** * @brief A memory buffer allocated by the core system of Orthanc. * * A memory buffer allocated by the core system of Orthanc. When the @@ -1422,7 +1442,7 @@ /** - * @brief Callback executed to unserialized a custom job. + * @brief Callback executed to unserialize a custom job. * * Signature of a callback function that unserializes a job that was * saved in the Orthanc database. @@ -1440,6 +1460,23 @@ /** + * @brief Callback executed to update the metrics of the plugin. + * + * Signature of a callback function that is called by Orthanc + * whenever a monitoring tool (such as Prometheus) asks the current + * values of the metrics. This callback gives the plugin a chance to + * update its metrics, by calling OrthancPluginSetMetricsValue(). + * This is typically useful for metrics that are expensive to + * acquire. + * + * @see OrthancPluginRegisterRefreshMetrics() + * @ingroups Callbacks + **/ + typedef void (*OrthancPluginRefreshMetricsCallback) (); + + + + /** * @brief Data structure that contains information about the Orthanc core. **/ typedef struct _OrthancPluginContext_t @@ -1531,7 +1568,8 @@ sizeof(int32_t) != sizeof(OrthancPluginIdentifierConstraint) || sizeof(int32_t) != sizeof(OrthancPluginInstanceOrigin) || sizeof(int32_t) != sizeof(OrthancPluginJobStepStatus) || - sizeof(int32_t) != sizeof(OrthancPluginConstraintType)) + sizeof(int32_t) != sizeof(OrthancPluginConstraintType) || + sizeof(int32_t) != sizeof(OrthancPluginMetricsType)) { /* Mismatch in the size of the enumerations */ return 0; @@ -6526,6 +6564,67 @@ } + + typedef struct + { + const char* name; + float value; + OrthancPluginMetricsType type; + } _OrthancPluginSetMetricsValue; + + /** + * @brief Set the value of a metrics. + * + * This function sets the value of a metrics to monitor the behavior + * of the plugin through tools such as Prometheus. The values of all + * the metrics are stored within the Orthanc context. + * + * @param context The Orthanc plugin context, as received by OrthancPluginInitialize(). + * @param name The name of the metrics to be set. + * @param value The value of the metrics. + * @param type The type of the metrics. This parameter is only taken into consideration + * the first time this metrics is set. + * @ingroup Toolbox + **/ + ORTHANC_PLUGIN_INLINE void OrthancPluginSetMetricsValue( + OrthancPluginContext* context, + const char* name, + float value, + OrthancPluginMetricsType type) + { + _OrthancPluginSetMetricsValue params; + params.name = name; + params.value = value; + params.type = type; + context->InvokeService(context, _OrthancPluginService_SetMetricsValue, ¶ms); + } + + + + typedef struct + { + OrthancPluginRefreshMetricsCallback callback; + } _OrthancPluginRegisterRefreshMetricsCallback; + + /** + * @brief Register a callback to refresh the metrics. + * + * This function registers a callback to refresh the metrics. The + * callback must make calls to OrthancPluginSetMetricsValue(). + * + * @param context The Orthanc plugin context, as received by OrthancPluginInitialize(). + * @param callback The callback function to handle the refresh. + * @ingroup Callbacks + **/ + ORTHANC_PLUGIN_INLINE void OrthancPluginRegisterRefreshMetricsCallback( + OrthancPluginContext* context, + OrthancPluginRefreshMetricsCallback callback) + { + _OrthancPluginRegisterRefreshMetricsCallback params; + params.callback = callback; + context->InvokeService(context, _OrthancPluginService_RegisterRefreshMetricsCallback, ¶ms); + } + #ifdef __cplusplus } #endif diff -r 6d558598d713 -r fca730c267d7 Plugins/Samples/Basic/Plugin.c --- a/Plugins/Samples/Basic/Plugin.c Wed Jan 30 12:41:20 2019 +0100 +++ b/Plugins/Samples/Basic/Plugin.c Wed Jan 30 17:12:15 2019 +0100 @@ -335,6 +335,8 @@ case OrthancPluginChangeType_OrthancStarted: { + OrthancPluginSetMetricsValue(context, "sample_started", 1, OrthancPluginMetricsType_Default); + /* Make REST requests to the built-in Orthanc API */ OrthancPluginRestApiGet(context, &tmp, "/changes"); OrthancPluginFreeMemoryBuffer(context, &tmp); @@ -392,6 +394,13 @@ } +ORTHANC_PLUGINS_API void RefreshMetrics() +{ + static unsigned int count = 0; + OrthancPluginSetMetricsValue(context, "sample_counter", count++, OrthancPluginMetricsType_Default); +} + + ORTHANC_PLUGINS_API int32_t OrthancPluginInitialize(OrthancPluginContext* c) { char info[1024], *s; @@ -455,7 +464,9 @@ OrthancPluginRegisterOnStoredInstanceCallback(context, OnStoredCallback); OrthancPluginRegisterOnChangeCallback(context, OnChangeCallback); OrthancPluginRegisterIncomingHttpRequestFilter(context, FilterIncomingHttpRequest); + OrthancPluginRegisterRefreshMetricsCallback(context, RefreshMetrics); + /* Declare several properties of the plugin */ OrthancPluginSetRootUri(context, "/plugin/hello"); OrthancPluginSetDescription(context, "This is the description of the sample plugin that can be seen in Orthanc Explorer."); diff -r 6d558598d713 -r fca730c267d7 Plugins/Samples/Common/OrthancPluginCppWrapper.cpp --- a/Plugins/Samples/Common/OrthancPluginCppWrapper.cpp Wed Jan 30 12:41:20 2019 +0100 +++ b/Plugins/Samples/Common/OrthancPluginCppWrapper.cpp Wed Jan 30 17:12:15 2019 +0100 @@ -2033,4 +2033,21 @@ } } #endif + + +#if HAS_ORTHANC_PLUGIN_METRICS == 1 + MetricsTimer::MetricsTimer(const char* name) : + name_(name) + { + start_ = boost::posix_time::microsec_clock::universal_time(); + } + + MetricsTimer::~MetricsTimer() + { + const boost::posix_time::ptime stop = boost::posix_time::microsec_clock::universal_time(); + const boost::posix_time::time_duration diff = stop - start_; + OrthancPluginSetMetricsValue(GetGlobalContext(), name_.c_str(), diff.total_milliseconds(), + OrthancPluginMetricsType_Timer); + } +#endif } diff -r 6d558598d713 -r fca730c267d7 Plugins/Samples/Common/OrthancPluginCppWrapper.h --- a/Plugins/Samples/Common/OrthancPluginCppWrapper.h Wed Jan 30 12:41:20 2019 +0100 +++ b/Plugins/Samples/Common/OrthancPluginCppWrapper.h Wed Jan 30 17:12:15 2019 +0100 @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -49,10 +50,10 @@ #if !defined(ORTHANC_PLUGINS_VERSION_IS_ABOVE) #define ORTHANC_PLUGINS_VERSION_IS_ABOVE(major, minor, revision) \ (ORTHANC_PLUGINS_MINIMAL_MAJOR_NUMBER > major || \ - (ORTHANC_PLUGINS_MINIMAL_MAJOR_NUMBER == major && \ - (ORTHANC_PLUGINS_MINIMAL_MINOR_NUMBER > minor || \ - (ORTHANC_PLUGINS_MINIMAL_MINOR_NUMBER == minor && \ - ORTHANC_PLUGINS_MINIMAL_REVISION_NUMBER >= revision)))) + (ORTHANC_PLUGINS_MINIMAL_MAJOR_NUMBER == major && \ + (ORTHANC_PLUGINS_MINIMAL_MINOR_NUMBER > minor || \ + (ORTHANC_PLUGINS_MINIMAL_MINOR_NUMBER == minor && \ + ORTHANC_PLUGINS_MINIMAL_REVISION_NUMBER >= revision)))) #endif @@ -78,6 +79,12 @@ # define HAS_ORTHANC_PLUGIN_EXCEPTION_DETAILS 0 #endif +#if ORTHANC_PLUGINS_VERSION_IS_ABOVE(1, 5, 4) +# define HAS_ORTHANC_PLUGIN_METRICS 1 +#else +# define HAS_ORTHANC_PLUGIN_METRICS 0 +#endif + namespace OrthancPlugins @@ -740,4 +747,26 @@ int priority); }; #endif + + +#if HAS_ORTHANC_PLUGIN_METRICS == 1 + inline void SetMetricsValue(char* name, + float value) + { + OrthancPluginSetMetricsValue(GetGlobalContext(), name, + value, OrthancPluginMetricsType_Default); + } + + class MetricsTimer : public boost::noncopyable + { + private: + std::string name_; + boost::posix_time::ptime start_; + + public: + MetricsTimer(const char* name); + + ~MetricsTimer(); + }; +#endif } diff -r 6d558598d713 -r fca730c267d7 UnitTestsSources/ServerIndexTests.cpp --- a/UnitTestsSources/ServerIndexTests.cpp Wed Jan 30 12:41:20 2019 +0100 +++ b/UnitTestsSources/ServerIndexTests.cpp Wed Jan 30 17:12:15 2019 +0100 @@ -808,8 +808,8 @@ context.GetIndex().GetGlobalStatistics(diskSize, uncompressedSize, countPatients, countStudies, countSeries, countInstances); - ASSERT_EQ(0, countInstances); - ASSERT_EQ(0, diskSize); + ASSERT_EQ(0u, countInstances); + ASSERT_EQ(0u, diskSize); { DicomInstanceToStore toStore; @@ -863,7 +863,7 @@ context.GetIndex().GetGlobalStatistics(diskSize, uncompressedSize, countPatients, countStudies, countSeries, countInstances); - ASSERT_EQ(1, countInstances); + ASSERT_EQ(1u, countInstances); ASSERT_EQ(dicom2.GetCompressedSize() + json2.GetCompressedSize(), diskSize); ASSERT_EQ(dicom2.GetUncompressedSize() + json2.GetUncompressedSize(), uncompressedSize);