comparison Aws/AwsS3StoragePlugin.cpp @ 120:12ea59c97c40

sync orthanc folder + show timings + EnableAwsSdkLogs
author Alain Mazy <am@osimis.io>
date Tue, 21 Nov 2023 09:46:51 +0100
parents 78c075412ab4
children addccb7ad900 3e9cced85a5b
comparison
equal deleted inserted replaced
115:ca68456d789a 120:12ea59c97c40
15 * You should have received a copy of the GNU Affero General Public License 15 * You should have received a copy of the GNU Affero General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 **/ 17 **/
18 18
19 #include "AwsS3StoragePlugin.h" 19 #include "AwsS3StoragePlugin.h"
20 #include <Logging.h>
20 21
21 #include <aws/core/Aws.h> 22 #include <aws/core/Aws.h>
22 #include <aws/s3/S3Client.h> 23 #include <aws/s3/S3Client.h>
23 #include <aws/s3/model/PutObjectRequest.h> 24 #include <aws/s3/model/PutObjectRequest.h>
24 #include <aws/s3/model/GetObjectRequest.h> 25 #include <aws/s3/model/GetObjectRequest.h>
25 #include <aws/s3/model/ListObjectsRequest.h> 26 #include <aws/s3/model/ListObjectsRequest.h>
26 #include <aws/s3/model/DeleteObjectRequest.h> 27 #include <aws/s3/model/DeleteObjectRequest.h>
27 #include <aws/core/auth/AWSCredentialsProvider.h> 28 #include <aws/core/auth/AWSCredentialsProvider.h>
28 #include <aws/core/utils/HashingUtils.h> 29 #include <aws/core/utils/HashingUtils.h>
30 #include <aws/core/utils/logging/DefaultLogSystem.h>
31 #include <aws/core/utils/logging/DefaultCRTLogSystem.h>
32 #include <aws/core/utils/logging/AWSLogging.h>
29 #include <aws/core/utils/memory/stl/AWSStreamFwd.h> 33 #include <aws/core/utils/memory/stl/AWSStreamFwd.h>
30 #include <aws/core/utils/memory/stl/AWSStringStream.h> 34 #include <aws/core/utils/memory/stl/AWSStringStream.h>
31 #include <aws/core/utils/memory/AWSMemory.h> 35 #include <aws/core/utils/memory/AWSMemory.h>
32 #include <aws/core/utils/stream/PreallocatedStreamBuf.h> 36 #include <aws/core/utils/stream/PreallocatedStreamBuf.h>
33 #include <aws/core/utils/StringUtils.h> 37 #include <aws/core/utils/StringUtils.h>
34 #include <aws/transfer/TransferManager.h> 38 #include <aws/transfer/TransferManager.h>
35 #include <aws/crt/Api.h> 39 #include <aws/crt/Api.h>
40 #include <iostream>
36 #include <fstream> 41 #include <fstream>
37 42
38 #include <boost/lexical_cast.hpp> 43 #include <boost/lexical_cast.hpp>
39 #include <boost/interprocess/streams/bufferstream.hpp> 44 #include <boost/interprocess/streams/bufferstream.hpp>
40 #include <iostream> 45 #include <iostream>
376 } 381 }
377 382
378 static std::unique_ptr<Aws::Crt::ApiHandle> api_; 383 static std::unique_ptr<Aws::Crt::ApiHandle> api_;
379 static std::unique_ptr<Aws::SDKOptions> sdkOptions_; 384 static std::unique_ptr<Aws::SDKOptions> sdkOptions_;
380 385
386 #include <stdarg.h>
387
388 class AwsOrthancLogger : public Aws::Utils::Logging::LogSystemInterface
389 {
390 public:
391 virtual ~AwsOrthancLogger() {}
392
393 /**
394 * Gets the currently configured log level for this logger.
395 */
396 virtual Aws::Utils::Logging::LogLevel GetLogLevel() const
397 {
398 return Aws::Utils::Logging::LogLevel::Trace;
399 }
400 /**
401 * Does a printf style output to the output stream. Don't use this, it's unsafe. See LogStream
402 */
403 virtual void Log(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const char* formatStr, ...)
404 {
405 Aws::StringStream ss;
406
407 va_list args;
408 va_start(args, formatStr);
409
410 va_list tmp_args; //unfortunately you cannot consume a va_list twice
411 va_copy(tmp_args, args); //so we have to copy it
412 #ifdef _WIN32
413 const int requiredLength = _vscprintf(formatStr, tmp_args) + 1;
414 #else
415 const int requiredLength = vsnprintf(nullptr, 0, formatStr, tmp_args) + 1;
416 #endif
417 va_end(tmp_args);
418
419 char outputBuff[requiredLength];
420 #ifdef _WIN32
421 vsnprintf_s(outputBuff, requiredLength, _TRUNCATE, formatStr, args);
422 #else
423 vsnprintf(outputBuff, requiredLength, formatStr, args);
424 #endif // _WIN32
425
426 if (logLevel == Aws::Utils::Logging::LogLevel::Debug || logLevel == Aws::Utils::Logging::LogLevel::Trace)
427 {
428 LOG(INFO) << reinterpret_cast<const char*>(&outputBuff[0]);
429 }
430 else if (logLevel == Aws::Utils::Logging::LogLevel::Warn)
431 {
432 LOG(WARNING) << reinterpret_cast<const char*>(&outputBuff[0]);
433 }
434 else
435 {
436 LOG(ERROR) << reinterpret_cast<const char*>(&outputBuff[0]);
437 }
438
439 va_end(args);
440 }
441 /**
442 * Writes the stream to the output stream.
443 */
444 virtual void LogStream(Aws::Utils::Logging::LogLevel logLevel, const char* tag, const Aws::OStringStream &messageStream)
445 {
446 if (logLevel == Aws::Utils::Logging::LogLevel::Debug || logLevel == Aws::Utils::Logging::LogLevel::Trace)
447 {
448 LOG(INFO) << tag << messageStream.str();
449 }
450 else if (logLevel == Aws::Utils::Logging::LogLevel::Warn)
451 {
452 LOG(WARNING) << tag << messageStream.str();
453 }
454 else
455 {
456 LOG(ERROR) << tag << messageStream.str();
457 }
458
459 }
460 /**
461 * Writes any buffered messages to the underlying device if the logger supports buffering.
462 */
463 virtual void Flush() {}
464 };
381 465
382 IStorage* AwsS3StoragePluginFactory::CreateStorage(const std::string& nameForLogs, const OrthancPlugins::OrthancConfiguration& orthancConfig) 466 IStorage* AwsS3StoragePluginFactory::CreateStorage(const std::string& nameForLogs, const OrthancPlugins::OrthancConfiguration& orthancConfig)
383 { 467 {
384 if (sdkOptions_.get() != NULL) 468 if (sdkOptions_.get() != NULL)
385 { 469 {
386 throw Orthanc::OrthancException(Orthanc::ErrorCode_BadSequenceOfCalls, "Cannot initialize twice"); 470 throw Orthanc::OrthancException(Orthanc::ErrorCode_BadSequenceOfCalls, "Cannot initialize twice");
387 } 471 }
388
389 api_.reset(new Aws::Crt::ApiHandle);
390
391 sdkOptions_.reset(new Aws::SDKOptions);
392 sdkOptions_->cryptoOptions.initAndCleanupOpenSSL = false; // Done by the Orthanc framework
393 sdkOptions_->httpOptions.initAndCleanupCurl = false; // Done by the Orthanc framework
394
395 Aws::InitAPI(*sdkOptions_);
396 472
397 bool enableLegacyStorageStructure; 473 bool enableLegacyStorageStructure;
398 bool storageContainsUnknownFiles; 474 bool storageContainsUnknownFiles;
399 475
400 if (!orthancConfig.IsSection(GetConfigurationSectionName())) 476 if (!orthancConfig.IsSection(GetConfigurationSectionName()))
430 506
431 const std::string endpoint = pluginSection.GetStringValue("Endpoint", ""); 507 const std::string endpoint = pluginSection.GetStringValue("Endpoint", "");
432 const unsigned int connectTimeout = pluginSection.GetUnsignedIntegerValue("ConnectTimeout", 30); 508 const unsigned int connectTimeout = pluginSection.GetUnsignedIntegerValue("ConnectTimeout", 30);
433 const unsigned int requestTimeout = pluginSection.GetUnsignedIntegerValue("RequestTimeout", 1200); 509 const unsigned int requestTimeout = pluginSection.GetUnsignedIntegerValue("RequestTimeout", 1200);
434 const bool virtualAddressing = pluginSection.GetBooleanValue("VirtualAddressing", true); 510 const bool virtualAddressing = pluginSection.GetBooleanValue("VirtualAddressing", true);
511 const bool enableAwsSdkLogs = pluginSection.GetBooleanValue("EnableAwsSdkLogs", false);
435 const std::string caFile = orthancConfig.GetStringValue("HttpsCACertificates", ""); 512 const std::string caFile = orthancConfig.GetStringValue("HttpsCACertificates", "");
436 513
514
515 api_.reset(new Aws::Crt::ApiHandle);
516
517 sdkOptions_.reset(new Aws::SDKOptions);
518 sdkOptions_->cryptoOptions.initAndCleanupOpenSSL = false; // Done by the Orthanc framework
519 sdkOptions_->httpOptions.initAndCleanupCurl = false; // Done by the Orthanc framework
520
521 if (enableAwsSdkLogs)
522 {
523 // Set up logging
524 Aws::Utils::Logging::InitializeAWSLogging(Aws::MakeShared<AwsOrthancLogger>(ALLOCATION_TAG));
525 // strangely, this seems to disable logging !!!! sdkOptions_->loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Trace;
526 }
527
528 Aws::InitAPI(*sdkOptions_);
529
530
437 try 531 try
438 { 532 {
439 Aws::Client::ClientConfiguration configuration; 533 Aws::Client::ClientConfiguration configuration;
440 534
441 configuration.region = region.c_str(); 535 configuration.region = region.c_str();