Mercurial > hg > orthanc-object-storage

changeset 120:12ea59c97c40

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
sync orthanc folder + show timings + EnableAwsSdkLogs
author Alain Mazy <am@osimis.io>
date Tue, 21 Nov 2023 09:46:51 +0100
parents ca68456d789a
children afea958425af
files Aws/AwsS3StoragePlugin.cpp Aws/CMakeLists.txt Azure/CMakeLists.txt Common/Resources/Orthanc/CMake/DownloadOrthancFramework.cmake Common/Resources/Orthanc/Plugins/OrthancPluginCppWrapper.cpp Common/Resources/Orthanc/Plugins/OrthancPluginCppWrapper.h Common/Resources/SyncOrthancFolder.py Common/StoragePlugin.cpp Google/CMakeLists.txt NEWS
diffstat 10 files changed, 140 insertions(+), 25 deletions(-) [+]
line wrap: on
line diff
--- a/Aws/AwsS3StoragePlugin.cpp	Tue Oct 17 13:10:29 2023 +0200
+++ b/Aws/AwsS3StoragePlugin.cpp	Tue Nov 21 09:46:51 2023 +0100
@@ -17,6 +17,7 @@
  **/
 
 #include "AwsS3StoragePlugin.h"
+#include <Logging.h>
 
 #include <aws/core/Aws.h>
 #include <aws/s3/S3Client.h>
@@ -26,6 +27,9 @@
 #include <aws/s3/model/DeleteObjectRequest.h>
 #include <aws/core/auth/AWSCredentialsProvider.h>
 #include <aws/core/utils/HashingUtils.h>
+#include <aws/core/utils/logging/DefaultLogSystem.h>
+#include <aws/core/utils/logging/DefaultCRTLogSystem.h>
+#include <aws/core/utils/logging/AWSLogging.h>
 #include <aws/core/utils/memory/stl/AWSStreamFwd.h>
 #include <aws/core/utils/memory/stl/AWSStringStream.h>
 #include <aws/core/utils/memory/AWSMemory.h>
@@ -33,6 +37,7 @@
 #include <aws/core/utils/StringUtils.h>
 #include <aws/transfer/TransferManager.h>
 #include <aws/crt/Api.h>
+#include <iostream>
 #include <fstream>
 
 #include <boost/lexical_cast.hpp>
@@ -378,6 +383,85 @@
 static std::unique_ptr<Aws::Crt::ApiHandle>  api_;
 static std::unique_ptr<Aws::SDKOptions>  sdkOptions_;
 
+#include <stdarg.h>
+
+class AwsOrthancLogger : public Aws::Utils::Logging::LogSystemInterface
+{
+public:
+    virtual ~AwsOrthancLogger() {}
+
+    /**
+     * Gets the currently configured log level for this logger.
+     */
+    virtual Aws::Utils::Logging::LogLevel GetLogLevel() const
+    {
+      return Aws::Utils::Logging::LogLevel::Trace;
+    }
+    /**
+     * Does a printf style output to the output stream. Don't use this, it's unsafe. See LogStream
+     */
+    virtual void Log(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const char* formatStr, ...)
+    {
+      Aws::StringStream ss;
+
+      va_list args;
+      va_start(args, formatStr);
+
+      va_list tmp_args; //unfortunately you cannot consume a va_list twice
+      va_copy(tmp_args, args); //so we have to copy it
+      #ifdef _WIN32
+          const int requiredLength = _vscprintf(formatStr, tmp_args) + 1;
+      #else
+          const int requiredLength = vsnprintf(nullptr, 0, formatStr, tmp_args) + 1;
+      #endif
+      va_end(tmp_args);
+
+      char outputBuff[requiredLength];
+      #ifdef _WIN32
+          vsnprintf_s(outputBuff, requiredLength, _TRUNCATE, formatStr, args);
+      #else
+          vsnprintf(outputBuff, requiredLength, formatStr, args);
+      #endif // _WIN32
+
+      if (logLevel == Aws::Utils::Logging::LogLevel::Debug || logLevel == Aws::Utils::Logging::LogLevel::Trace)
+      {
+        LOG(INFO) << reinterpret_cast<const char*>(&outputBuff[0]);
+      }
+      else if (logLevel == Aws::Utils::Logging::LogLevel::Warn)
+      {
+        LOG(WARNING) << reinterpret_cast<const char*>(&outputBuff[0]);
+      }
+      else
+      {
+        LOG(ERROR) << reinterpret_cast<const char*>(&outputBuff[0]);
+      }
+
+      va_end(args);
+    }
+    /**
+    * Writes the stream to the output stream.
+    */
+    virtual void LogStream(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const Aws::OStringStream &messageStream)
+    {
+      if (logLevel == Aws::Utils::Logging::LogLevel::Debug || logLevel == Aws::Utils::Logging::LogLevel::Trace)
+      {
+        LOG(INFO) << tag << messageStream.str();
+      }
+      else if (logLevel == Aws::Utils::Logging::LogLevel::Warn)
+      {
+        LOG(WARNING) << tag << messageStream.str();
+      }
+      else
+      {
+        LOG(ERROR) << tag << messageStream.str();
+      }
+
+    }
+    /**
+     * Writes any buffered messages to the underlying device if the logger supports buffering.
+     */
+    virtual void Flush() {}
+};
 
 IStorage* AwsS3StoragePluginFactory::CreateStorage(const std::string& nameForLogs, const OrthancPlugins::OrthancConfiguration& orthancConfig)
 {
@@ -386,14 +470,6 @@
     throw Orthanc::OrthancException(Orthanc::ErrorCode_BadSequenceOfCalls, "Cannot initialize twice");
   }
 
-  api_.reset(new Aws::Crt::ApiHandle);
-
-  sdkOptions_.reset(new Aws::SDKOptions);
-  sdkOptions_->cryptoOptions.initAndCleanupOpenSSL = false;  // Done by the Orthanc framework
-  sdkOptions_->httpOptions.initAndCleanupCurl = false;  // Done by the Orthanc framework
-  
-  Aws::InitAPI(*sdkOptions_);
-
   bool enableLegacyStorageStructure;
   bool storageContainsUnknownFiles;
 
@@ -432,8 +508,26 @@
   const unsigned int connectTimeout = pluginSection.GetUnsignedIntegerValue("ConnectTimeout", 30);
   const unsigned int requestTimeout = pluginSection.GetUnsignedIntegerValue("RequestTimeout", 1200);
   const bool virtualAddressing = pluginSection.GetBooleanValue("VirtualAddressing", true);
+  const bool enableAwsSdkLogs = pluginSection.GetBooleanValue("EnableAwsSdkLogs", false);
   const std::string caFile = orthancConfig.GetStringValue("HttpsCACertificates", "");
-  
+
+
+  api_.reset(new Aws::Crt::ApiHandle);
+
+  sdkOptions_.reset(new Aws::SDKOptions);
+  sdkOptions_->cryptoOptions.initAndCleanupOpenSSL = false;  // Done by the Orthanc framework
+  sdkOptions_->httpOptions.initAndCleanupCurl = false;  // Done by the Orthanc framework
+
+  if (enableAwsSdkLogs)
+  {
+    // Set up logging
+    Aws::Utils::Logging::InitializeAWSLogging(Aws::MakeShared<AwsOrthancLogger>(ALLOCATION_TAG));
+    // strangely, this seems to disable logging !!!! sdkOptions_->loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Trace;
+  }
+
+  Aws::InitAPI(*sdkOptions_);
+
+
   try
   {
     Aws::Client::ClientConfiguration configuration;
--- a/Aws/CMakeLists.txt	Tue Oct 17 13:10:29 2023 +0200
+++ b/Aws/CMakeLists.txt	Tue Nov 21 09:46:51 2023 +0100
@@ -19,13 +19,13 @@
 
 project(OrthancAwsS3Storage)
 
-set(PLUGIN_VERSION "mainline")
+set(PLUGIN_VERSION "2.3.0")
 
 if (PLUGIN_VERSION STREQUAL "mainline")
   set(ORTHANC_FRAMEWORK_DEFAULT_VERSION "mainline")
   set(ORTHANC_FRAMEWORK_DEFAULT_SOURCE "hg")
 else()
-  set(ORTHANC_FRAMEWORK_DEFAULT_VERSION "1.12.1")
+  set(ORTHANC_FRAMEWORK_DEFAULT_VERSION "daf4807631c5")
   set(ORTHANC_FRAMEWORK_DEFAULT_SOURCE "web")
 endif()
 
--- a/Azure/CMakeLists.txt	Tue Oct 17 13:10:29 2023 +0200
+++ b/Azure/CMakeLists.txt	Tue Nov 21 09:46:51 2023 +0100
@@ -19,12 +19,13 @@
 
 project(OrthancAzureBlobStorage)
 
-set(PLUGIN_VERSION "mainline")
+set(PLUGIN_VERSION "2.3.0")
 
 include(CheckIncludeFileCXX)
 
-set(ORTHANC_FRAMEWORK_SOURCE "hg" CACHE STRING "orthanc source")
-set(ORTHANC_FRAMEWORK_VERSION "mainline" CACHE STRING "orthanc framework version")  # TODO: update to 1.12.2 when availableset(ALLOW_DOWNLOADS ON)
+set(ORTHANC_FRAMEWORK_SOURCE "web" CACHE STRING "orthanc source")
+set(ORTHANC_FRAMEWORK_VERSION "daf4807631c5" CACHE STRING "orthanc framework version")  # TODO: update to 1.12.2 when available
+set(ALLOW_DOWNLOADS ON)
 
 # Download and setup the Orthanc framework
 
--- a/Common/Resources/Orthanc/CMake/DownloadOrthancFramework.cmake	Tue Oct 17 13:10:29 2023 +0200
+++ b/Common/Resources/Orthanc/CMake/DownloadOrthancFramework.cmake	Tue Nov 21 09:46:51 2023 +0100
@@ -271,7 +271,7 @@
   else()
     message("Forking the Orthanc source repository using Mercurial")
     execute_process(
-      COMMAND ${ORTHANC_FRAMEWORK_HG} clone "https://hg.orthanc-server.com/orthanc/"
+      COMMAND ${ORTHANC_FRAMEWORK_HG} clone "https://orthanc.uclouvain.be/hg/orthanc/"
       WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
       RESULT_VARIABLE Failure
       )    
--- a/Common/Resources/Orthanc/Plugins/OrthancPluginCppWrapper.cpp	Tue Oct 17 13:10:29 2023 +0200
+++ b/Common/Resources/Orthanc/Plugins/OrthancPluginCppWrapper.cpp	Tue Nov 21 09:46:51 2023 +0100
@@ -79,6 +79,10 @@
     }
   }
 
+  void ResetGlobalContext()
+  {
+    globalContext_ = NULL;
+  }
 
   bool HasGlobalContext()
   {
--- a/Common/Resources/Orthanc/Plugins/OrthancPluginCppWrapper.h	Tue Oct 17 13:10:29 2023 +0200
+++ b/Common/Resources/Orthanc/Plugins/OrthancPluginCppWrapper.h	Tue Nov 21 09:46:51 2023 +0100
@@ -137,6 +137,8 @@
 
   void SetGlobalContext(OrthancPluginContext* context);
 
+  void ResetGlobalContext();
+
   bool HasGlobalContext();
 
   OrthancPluginContext* GetGlobalContext();
--- a/Common/Resources/SyncOrthancFolder.py	Tue Oct 17 13:10:29 2023 +0200
+++ b/Common/Resources/SyncOrthancFolder.py	Tue Nov 21 09:46:51 2023 +0100
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 
 #
 # This maintenance script updates the content of the "Orthanc" folder
@@ -8,11 +8,11 @@
 import multiprocessing
 import os
 import stat
-import urllib2
+import urllib.request
 
 TARGET = os.path.join(os.path.dirname(__file__), 'Orthanc')
 PLUGIN_SDK_VERSION = '1.12.1'
-REPOSITORY = 'https://hg.orthanc-server.com/orthanc/raw-file'
+REPOSITORY = 'https://orthanc.uclouvain.be/hg/orthanc/raw-file'
 
 FILES = [
     ('OrthancFramework/Resources/CMake/AutoGeneratedCode.cmake', 'CMake'),
@@ -42,7 +42,7 @@
     branch = x[0]
     source = x[1]
     target = os.path.join(TARGET, x[2])
-    print target
+    print(target)
 
     try:
         os.makedirs(os.path.dirname(target))
@@ -51,9 +51,9 @@
 
     url = '%s/%s/%s' % (REPOSITORY, branch, source)
 
-    with open(target, 'w') as f:
+    with open(target, 'wb') as f:
         try:
-            f.write(urllib2.urlopen(url).read())
+            f.write(urllib.request.urlopen(url).read())
         except:
             print('ERROR %s' % url)
             raise
--- a/Common/StoragePlugin.cpp	Tue Oct 17 13:10:29 2023 +0200
+++ b/Common/StoragePlugin.cpp	Tue Nov 21 09:46:51 2023 +0100
@@ -88,6 +88,7 @@
 {
   try
   {
+    Orthanc::Toolbox::ElapsedTimer timer;
     OrthancPlugins::LogInfo(primaryStorage->GetNameForLogs() + ": creating attachment " + std::string(uuid) + " of type " + boost::lexical_cast<std::string>(type));
     std::unique_ptr<IStorage::IWriter> writer(primaryStorage->GetWriterForObject(uuid, type, cryptoEnabled));
 
@@ -111,6 +112,7 @@
     {
       writer->Write(reinterpret_cast<const char*>(content), size);
     }
+    OrthancPlugins::LogInfo(primaryStorage->GetNameForLogs() + ": created attachment " + std::string(uuid) + " (" + timer.GetHumanTransferSpeed(true, size) + ")");
   }
   catch (StoragePluginException& ex)
   {
@@ -133,11 +135,13 @@
 
   try
   {
+    Orthanc::Toolbox::ElapsedTimer timer;
     OrthancPlugins::LogInfo(storage->GetNameForLogs() + ": reading range of attachment " + std::string(uuid) + " of type " + boost::lexical_cast<std::string>(type));
     
     std::unique_ptr<IStorage::IReader> reader(storage->GetReaderForObject(uuid, type, cryptoEnabled));
     reader->ReadRange(reinterpret_cast<char*>(target->data), target->size, rangeStart);
     
+    OrthancPlugins::LogInfo(storage->GetNameForLogs() + ": read range of attachment " + std::string(uuid) + " (" + timer.GetHumanTransferSpeed(true, target->size) + ")");
     return OrthancPluginErrorCode_Success;
   }
   catch (StoragePluginException& ex)
@@ -181,6 +185,7 @@
 {
   try
   {
+    Orthanc::Toolbox::ElapsedTimer timer;
     OrthancPlugins::LogInfo(storage->GetNameForLogs() + ": reading whole attachment " + std::string(uuid) + " of type " + boost::lexical_cast<std::string>(type));
     std::unique_ptr<IStorage::IReader> reader(storage->GetReaderForObject(uuid, type, cryptoEnabled));
 
@@ -227,6 +232,8 @@
     {
       reader->ReadWhole(reinterpret_cast<char*>(target->data), fileSize);
     }
+
+    OrthancPlugins::LogInfo(storage->GetNameForLogs() + ": read whole attachment " + std::string(uuid) + " (" + timer.GetHumanTransferSpeed(true, fileSize) + ")");
   }
   catch (StoragePluginException& ex)
   {
--- a/Google/CMakeLists.txt	Tue Oct 17 13:10:29 2023 +0200
+++ b/Google/CMakeLists.txt	Tue Nov 21 09:46:51 2023 +0100
@@ -19,12 +19,12 @@
 
 project(OrthancGoogleCloudStorage)
 
-set(PLUGIN_VERSION "mainline")
+set(PLUGIN_VERSION "2.3.0")
 
 include(CheckIncludeFileCXX)
 
-set(ORTHANC_FRAMEWORK_SOURCE "hg" CACHE STRING "orthanc source")
-set(ORTHANC_FRAMEWORK_VERSION "mainline" CACHE STRING "orthanc framework version")  # TODO: update to 1.12.2 when available
+set(ORTHANC_FRAMEWORK_SOURCE "web" CACHE STRING "orthanc source")
+set(ORTHANC_FRAMEWORK_VERSION "daf4807631c5" CACHE STRING "orthanc framework version")  # TODO: update to 1.12.2 when available
 set(ALLOW_DOWNLOADS ON)
 
 # Download and setup the Orthanc framework
--- a/NEWS	Tue Oct 17 13:10:29 2023 +0200
+++ b/NEWS	Tue Nov 21 09:46:51 2023 +0100
@@ -1,8 +1,15 @@
+Pending changes in the mainline
+===============================
+
+* AWS plugin:
+  * New configuration "EnableAwsSdkLogs" to include AWS SDK logs in Orthanc logs.
+* Now displaying size, duration and transfer speed for each read/write operation.
+
 2023-10-17 - v 2.3.0
 ====================
 
 * AWS plugin:
-  * Added a new configuration "UseTransferMode" (false by default).
+  * Added a new configuration "UseTransferManager" (false by default).
     When set to true, the Transfer Manager mode is used to upload/download
     whole files to/from the bucket.  If set to false, the default "object"
     mode is used.  The Transfer Manager mode is supposed to be faster,