annotate Aws/AwsS3StoragePlugin.cpp @ 70:16e419fe80c5

Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
author Alain Mazy <am@osimis.io>
date Mon, 06 Sep 2021 14:21:56 +0200
parents 58a03fce4897
children a25b4140e7e4
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
1 /**
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
2 * Cloud storage plugins for Orthanc
37
f55b2afdf53d upgrade to year 2021
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 33
diff changeset
3 * Copyright (C) 2020-2021 Osimis S.A., Belgium
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
4 *
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
5 * This program is free software: you can redistribute it and/or
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
6 * modify it under the terms of the GNU Affero General Public License
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
7 * as published by the Free Software Foundation, either version 3 of
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
8 * the License, or (at your option) any later version.
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
9 *
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
10 * This program is distributed in the hope that it will be useful, but
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
13 * Affero General Public License for more details.
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
14 *
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
15 * You should have received a copy of the GNU Affero General Public License
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
17 **/
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
18
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
19 #include "AwsS3StoragePlugin.h"
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
20
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
21 #include <aws/core/Aws.h>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
22 #include <aws/s3/S3Client.h>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
23 #include <aws/s3/model/PutObjectRequest.h>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
24 #include <aws/s3/model/GetObjectRequest.h>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
25 #include <aws/s3/model/ListObjectsRequest.h>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
26 #include <aws/s3/model/DeleteObjectRequest.h>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
27 #include <aws/core/auth/AWSCredentialsProvider.h>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
28 #include <aws/core/utils/memory/stl/AWSStringStream.h>
57
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
29 #include <aws/crt/Api.h>
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
30
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
31 #include <boost/lexical_cast.hpp>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
32 #include <iostream>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
33 #include <fstream>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
34
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
35 const char* ALLOCATION_TAG = "OrthancS3";
15
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
36 static const char* const PLUGIN_SECTION = "AwsS3Storage";
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
37
15
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
38 class AwsS3StoragePlugin : public BaseStoragePlugin
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
39 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
40 public:
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
41
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
42 Aws::S3::S3Client client_;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
43 std::string bucketName_;
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
44 bool storageContainsUnknownFiles_;
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
45
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
46 public:
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
47
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
48 AwsS3StoragePlugin(const Aws::S3::S3Client& client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles);
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
49
57
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
50 virtual ~AwsS3StoragePlugin();
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
51
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
52 virtual IWriter* GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
53 virtual IReader* GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
54 virtual void DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled);
15
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
55 virtual const char* GetConfigurationSectionName() {return PLUGIN_SECTION;};
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
56 };
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
57
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
58
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
59 class Writer : public IStoragePlugin::IWriter
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
60 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
61 std::string path_;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
62 Aws::S3::S3Client client_;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
63 std::string bucketName_;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
64
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
65 public:
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
66 Writer(const Aws::S3::S3Client& client, const std::string& bucketName, const std::string& path)
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
67 : path_(path),
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
68 client_(client),
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
69 bucketName_(bucketName)
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
70 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
71 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
72
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
73 virtual ~Writer()
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
74 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
75 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
76
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
77 virtual void Write(const char* data, size_t size)
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
78 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
79 Aws::S3::Model::PutObjectRequest putObjectRequest;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
80
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
81 putObjectRequest.SetBucket(bucketName_.c_str());
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
82 putObjectRequest.SetKey(path_.c_str());
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
83
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
84 std::shared_ptr<Aws::StringStream> stream = Aws::MakeShared<Aws::StringStream>(ALLOCATION_TAG, std::ios_base::in | std::ios_base::binary);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
85
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
86 stream->rdbuf()->pubsetbuf(const_cast<char*>(data), size);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
87 stream->rdbuf()->pubseekpos(size);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
88 stream->seekg(0);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
89
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
90 putObjectRequest.SetBody(stream);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
91
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
92 auto result = client_.PutObject(putObjectRequest);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
93
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
94 if (!result.IsSuccess())
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
95 {
62
d10696f94959 more detailed AWS errors
Alain Mazy <am@osimis.io>
parents: 57
diff changeset
96 throw StoragePluginException(std::string("error while writing file ") + path_ + ": response code = " + boost::lexical_cast<std::string>((int)result.GetError().GetResponseCode()) + " " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str());
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
97 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
98 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
99 };
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
100
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
101
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
102 class Reader : public IStoragePlugin::IReader
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
103 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
104 Aws::S3::S3Client client_;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
105 std::string bucketName_;
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
106 std::list<std::string> paths_;
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
107 std::string uuid_;
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
108
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
109 public:
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
110 Reader(const Aws::S3::S3Client& client, const std::string& bucketName, const std::list<std::string>& paths, const char* uuid)
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
111 : client_(client),
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
112 bucketName_(bucketName),
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
113 paths_(paths),
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
114 uuid_(uuid)
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
115 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
116 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
117
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
118 virtual ~Reader()
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
119 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
120
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
121 }
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
122
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
123 virtual size_t GetSize()
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
124 {
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
125 std::string firstExceptionMessage;
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
126
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
127 for (auto& path: paths_)
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
128 {
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
129 try
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
130 {
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
131 return _GetSize(path);
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
132 }
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
133 catch (StoragePluginException& ex)
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
134 {
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
135 if (firstExceptionMessage.empty())
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
136 {
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
137 firstExceptionMessage = ex.what();
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
138 }
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
139 //ignore to retry
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
140 }
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
141 }
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
142 throw StoragePluginException(firstExceptionMessage);
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
143 }
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
144
41
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
145 virtual void ReadWhole(char* data, size_t size)
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
146 {
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
147 _Read(data, size, 0, false);
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
148 }
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
149
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
150 virtual void ReadRange(char* data, size_t size, size_t fromOffset)
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
151 {
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
152 _Read(data, size, fromOffset, true);
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
153 }
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
154
70
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
155 private:
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
156
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
157 size_t _GetSize(const std::string& path)
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
158 {
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
159 Aws::S3::Model::ListObjectsRequest listObjectRequest;
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
160 listObjectRequest.SetBucket(bucketName_.c_str());
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
161 listObjectRequest.SetPrefix(path.c_str());
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
162
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
163 auto result = client_.ListObjects(listObjectRequest);
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
164
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
165 if (result.IsSuccess())
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
166 {
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
167 Aws::Vector<Aws::S3::Model::Object> objectList =
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
168 result.GetResult().GetContents();
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
169
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
170 if (objectList.size() == 1)
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
171 {
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
172 return objectList[0].GetSize();
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
173 }
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
174 else if (objectList.size() > 1)
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
175 {
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
176 throw StoragePluginException(std::string("error while reading file ") + path + ": multiple objet with same name !");
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
177 }
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
178 throw StoragePluginException(std::string("error while reading file ") + path + ": object not found !");
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
179 }
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
180 else
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
181 {
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
182 throw StoragePluginException(std::string("error while reading file ") + path + ": " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str());
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
183 }
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
184 }
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
185
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
186
41
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
187 void _Read(char* data, size_t size, size_t fromOffset, bool useRange)
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
188 {
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
189 std::string firstExceptionMessage;
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
190
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
191 for (auto& path: paths_)
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
192 {
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
193 try
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
194 {
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
195 return __Read(path, data, size, fromOffset, useRange);
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
196 }
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
197 catch (StoragePluginException& ex)
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
198 {
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
199 if (firstExceptionMessage.empty())
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
200 {
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
201 firstExceptionMessage = ex.what();
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
202 }
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
203 //ignore to retry
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
204 }
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
205 }
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
206 throw StoragePluginException(firstExceptionMessage);
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
207 }
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
208
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
209 void __Read(const std::string& path, char* data, size_t size, size_t fromOffset, bool useRange)
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
210 {
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
211 Aws::S3::Model::GetObjectRequest getObjectRequest;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
212 getObjectRequest.SetBucket(bucketName_.c_str());
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
213 getObjectRequest.SetKey(path.c_str());
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
214
41
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
215 if (useRange)
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
216 {
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
217 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
218 std::string range = std::string("bytes=") + boost::lexical_cast<std::string>(fromOffset) + "-" + boost::lexical_cast<std::string>(fromOffset + size -1);
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
219 getObjectRequest.SetRange(range.c_str());
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
220 }
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
221
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
222 getObjectRequest.SetResponseStreamFactory(
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
223 [data, size]()
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
224 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
225 std::unique_ptr<Aws::StringStream>
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
226 istream(Aws::New<Aws::StringStream>(ALLOCATION_TAG));
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
227
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
228 istream->rdbuf()->pubsetbuf(static_cast<char*>(data),
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
229 size);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
230
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
231 return istream.release();
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
232 });
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
233
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
234 // Get the object
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
235 auto result = client_.GetObject(getObjectRequest);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
236 if (result.IsSuccess())
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
237 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
238 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
239 else
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
240 {
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
241 throw StoragePluginException(std::string("error while reading file ") + path + ": response code = " + boost::lexical_cast<std::string>((int)result.GetError().GetResponseCode()) + " " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str());
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
242 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
243 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
244
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
245 };
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
246
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
247
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
248
41
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
249
d99afdf6d872 ReadRange for AWS
Alain Mazy <am@osimis.io>
parents: 37
diff changeset
250
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
251 const char* AwsS3StoragePluginFactory::GetStoragePluginName()
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
252 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
253 return "AWS S3 Storage";
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
254 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
255
57
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
256
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
257 static std::unique_ptr<Aws::Crt::ApiHandle> api_;
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
258 static std::unique_ptr<Aws::SDKOptions> sdkOptions_;
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
259
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
260
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
261 IStoragePlugin* AwsS3StoragePluginFactory::CreateStoragePlugin(const OrthancPlugins::OrthancConfiguration& orthancConfig)
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
262 {
57
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
263 if (sdkOptions_.get() != NULL)
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
264 {
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
265 throw Orthanc::OrthancException(Orthanc::ErrorCode_BadSequenceOfCalls, "Cannot initialize twice");
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
266 }
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
267
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
268 api_.reset(new Aws::Crt::ApiHandle);
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
269
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
270 sdkOptions_.reset(new Aws::SDKOptions);
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
271 sdkOptions_->cryptoOptions.initAndCleanupOpenSSL = false; // Done by the Orthanc framework
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
272 sdkOptions_->httpOptions.initAndCleanupCurl = false; // Done by the Orthanc framework
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
273
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
274 Aws::InitAPI(*sdkOptions_);
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
275
15
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
276 bool enableLegacyStorageStructure;
70
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
277 bool storageContainsUnknownFiles;
15
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
278
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
279 if (!orthancConfig.IsSection(PLUGIN_SECTION))
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
280 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
281 OrthancPlugins::LogWarning(std::string(GetStoragePluginName()) + " plugin, section missing. Plugin is not enabled.");
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
282 return nullptr;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
283 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
284
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
285 OrthancPlugins::OrthancConfiguration pluginSection;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
286 orthancConfig.GetSection(pluginSection, PLUGIN_SECTION);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
287
70
16e419fe80c5 Google & Azure: Added the "EnableLegacyUnknownFiles" configuration option
Alain Mazy <am@osimis.io>
parents: 68
diff changeset
288 if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, storageContainsUnknownFiles, pluginSection))
15
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
289 {
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
290 return nullptr;
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
291 }
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
292
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
293 std::string bucketName;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
294 std::string region;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
295 std::string accessKey;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
296 std::string secretKey;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
297
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
298 if (!pluginSection.LookupStringValue(bucketName, "BucketName"))
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
299 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
300 OrthancPlugins::LogError("AwsS3Storage/BucketName configuration missing. Unable to initialize plugin");
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
301 return nullptr;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
302 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
303
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
304 if (!pluginSection.LookupStringValue(region, "Region"))
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
305 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
306 OrthancPlugins::LogError("AwsS3Storage/Region configuration missing. Unable to initialize plugin");
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
307 return nullptr;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
308 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
309
57
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
310 const std::string endpoint = pluginSection.GetStringValue("Endpoint", "");
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
311 const unsigned int connectTimeout = pluginSection.GetUnsignedIntegerValue("ConnectTimeout", 30);
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
312 const unsigned int requestTimeout = pluginSection.GetUnsignedIntegerValue("RequestTimeout", 1200);
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
313 const bool virtualAddressing = pluginSection.GetBooleanValue("VirtualAddressing", true);
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
314 const std::string caFile = orthancConfig.GetStringValue("HttpsCACertificates", "");
27
e1f52b851827 Added "VirtualAddressing" configuration option in the AWS S3 plugin (for compatibility with minio)
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 15
diff changeset
315
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
316 try
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
317 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
318 Aws::SDKOptions options;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
319 Aws::InitAPI(options);
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
320
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
321 Aws::Client::ClientConfiguration configuration;
57
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
322
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
323 configuration.region = region.c_str();
6
393fcf337462 AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents: 1
diff changeset
324 configuration.scheme = Aws::Http::Scheme::HTTPS;
393fcf337462 AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents: 1
diff changeset
325 configuration.connectTimeoutMs = connectTimeout * 1000;
393fcf337462 AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents: 1
diff changeset
326 configuration.requestTimeoutMs = requestTimeout * 1000;
27
e1f52b851827 Added "VirtualAddressing" configuration option in the AWS S3 plugin (for compatibility with minio)
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 15
diff changeset
327 configuration.httpRequestTimeoutMs = requestTimeout * 1000;
6
393fcf337462 AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents: 1
diff changeset
328
393fcf337462 AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents: 1
diff changeset
329 if (!endpoint.empty())
393fcf337462 AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents: 1
diff changeset
330 {
393fcf337462 AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents: 1
diff changeset
331 configuration.endpointOverride = endpoint.c_str();
393fcf337462 AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents: 1
diff changeset
332 }
393fcf337462 AWS: added 3 configurations: Endpoint, ConnectionTimeout, RequestTimeout
Alain Mazy
parents: 1
diff changeset
333
57
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
334 if (!caFile.empty())
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
335 {
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
336 configuration.caFile = caFile;
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
337 }
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
338
53
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
339 if (pluginSection.LookupStringValue(accessKey, "AccessKey") && pluginSection.LookupStringValue(secretKey, "SecretKey"))
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
340 {
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
341 OrthancPlugins::LogInfo("AWS S3 Storage: using credentials from the configuration file");
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
342 Aws::Auth::AWSCredentials credentials(accessKey.c_str(), secretKey.c_str());
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
343
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
344 Aws::S3::S3Client client(credentials, configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing);
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
345
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
346 OrthancPlugins::LogInfo("AWS S3 storage initialized");
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
347
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
348 return new AwsS3StoragePlugin(client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles);
53
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
349 }
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
350 else
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
351 {
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
352 // when using default credentials, credentials are not checked at startup but only the first time you try to access the bucket !
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
353 OrthancPlugins::LogInfo("AWS S3 Storage: using default credentials provider");
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
354 Aws::S3::S3Client client(configuration, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, virtualAddressing);
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
355
53
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
356 OrthancPlugins::LogInfo("AWS S3 storage initialized");
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
357
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
358 return new AwsS3StoragePlugin(client, bucketName, enableLegacyStorageStructure, storageContainsUnknownFiles);
53
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
359 }
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
360 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
361 catch (const std::exception& e)
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
362 {
53
1c3e34f5c5c6 AWS S3: if no access & secret keys are provided, now getting the credentials from the default credentials manager
Alain Mazy <am@osimis.io>
parents: 41
diff changeset
363 OrthancPlugins::LogError(std::string("AWS S3 Storage plugin: failed to initialize plugin: ") + e.what());
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
364 return nullptr;
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
365 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
366
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
367 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
368
57
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
369
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
370 AwsS3StoragePlugin::~AwsS3StoragePlugin()
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
371 {
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
372 assert(sdkOptions_.get() != NULL);
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
373 Aws::ShutdownAPI(*sdkOptions_);
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
374 api_.reset();
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
375 }
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
376
ba1be668e475 fix initialization of the aws static library
Sebastien Jodogne <s.jodogne@gmail.com>
parents: 53
diff changeset
377
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
378 AwsS3StoragePlugin::AwsS3StoragePlugin(const Aws::S3::S3Client& client, const std::string& bucketName, bool enableLegacyStorageStructure, bool storageContainsUnknownFiles)
15
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
379 : BaseStoragePlugin(enableLegacyStorageStructure),
2a02b21f0a19 migration + storage structure
Alain Mazy
parents: 6
diff changeset
380 client_(client),
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
381 bucketName_(bucketName),
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
382 storageContainsUnknownFiles_(storageContainsUnknownFiles)
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
383 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
384 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
385
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
386 IStoragePlugin::IWriter* AwsS3StoragePlugin::GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled)
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
387 {
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
388 return new Writer(client_, bucketName_, GetPath(uuid, type, encryptionEnabled));
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
389 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
390
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
391 IStoragePlugin::IReader* AwsS3StoragePlugin::GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled)
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
392 {
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
393 std::list<std::string> paths;
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
394 paths.push_back(GetPath(uuid, type, encryptionEnabled, false));
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
395 if (storageContainsUnknownFiles_)
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
396 {
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
397 paths.push_back(GetPath(uuid, type, encryptionEnabled, true));
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
398 }
64
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
399
c4f56973a279 Fix reading/deleting DCM header files that were saved with plugin v 1.2.0 and Orthanc 1.9.3 which had a .unk extension
Alain Mazy <am@osimis.io>
parents: 62
diff changeset
400 return new Reader(client_, bucketName_, paths, uuid);
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
401 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
402
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
403 void AwsS3StoragePlugin::DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled)
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
404 {
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
405 std::string firstExceptionMessage;
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
406
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
407 std::list<std::string> paths;
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
408 paths.push_back(GetPath(uuid, type, encryptionEnabled, false));
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
409 if (storageContainsUnknownFiles_)
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
410 {
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
411 paths.push_back(GetPath(uuid, type, encryptionEnabled, true));
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
412 }
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
413
68
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
414 // DeleteObject succeeds even if the file does not exist -> we need to try to delete every path
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
415 for (auto& path: paths)
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
416 {
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
417 Aws::S3::Model::DeleteObjectRequest deleteObjectRequest;
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
418 deleteObjectRequest.SetBucket(bucketName_.c_str());
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
419 deleteObjectRequest.SetKey(path.c_str());
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
420
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
421 auto result = client_.DeleteObject(deleteObjectRequest);
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
422
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
423 if (!result.IsSuccess() && firstExceptionMessage.empty())
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
424 {
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
425 firstExceptionMessage = std::string("error while deleting file ") + path + ": response code = " + boost::lexical_cast<std::string>((int)result.GetError().GetResponseCode()) + " " + result.GetError().GetExceptionName().c_str() + " " + result.GetError().GetMessage().c_str();
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
426 }
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
427 }
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
428
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
429 if (!firstExceptionMessage.empty())
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
430 {
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
431 throw StoragePluginException(firstExceptionMessage);
58a03fce4897 new option for AWS: EnableLegacyUnknownFiles
Alain Mazy <am@osimis.io>
parents: 64
diff changeset
432 }
1
fc26a8fc54d5 initial release
Alain Mazy <alain@mazy.be>
parents:
diff changeset
433 }