Mercurial > hg > orthanc-object-storage
annotate Google/GoogleStoragePlugin.cpp @ 68:58a03fce4897
new option for AWS: EnableLegacyUnknownFiles
| author | Alain Mazy <am@osimis.io> |
|---|---|
| date | Fri, 09 Jul 2021 15:40:07 +0200 |
| parents | b922ae86bbe1 |
| children | 16e419fe80c5 |
| rev | line source |
|---|---|
| 1 | 1 /** |
| 2 * Cloud storage plugins for Orthanc | |
|
37
f55b2afdf53d
upgrade to year 2021
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
15
diff
changeset
|
3 * Copyright (C) 2020-2021 Osimis S.A., Belgium |
| 1 | 4 * |
| 5 * This program is free software: you can redistribute it and/or | |
| 6 * modify it under the terms of the GNU Affero General Public License | |
| 7 * as published by the Free Software Foundation, either version 3 of | |
| 8 * the License, or (at your option) any later version. | |
| 9 * | |
| 10 * This program is distributed in the hope that it will be useful, but | |
| 11 * WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| 13 * Affero General Public License for more details. | |
| 14 * | |
| 15 * You should have received a copy of the GNU Affero General Public License | |
| 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| 17 **/ | |
| 18 | |
|
56
b922ae86bbe1
full static linking against AWS SDK
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
40
diff
changeset
|
19 |
| 1 | 20 #include "GoogleStoragePlugin.h" |
| 21 | |
| 22 #include "google/cloud/storage/client.h" | |
| 23 | |
| 24 // Create aliases to make the code easier to read. | |
| 25 namespace gcs = google::cloud::storage; | |
| 15 | 26 static const char* const PLUGIN_SECTION = "GoogleCloudStorage"; |
| 27 | |
| 28 const char* GoogleStoragePlugin::GetConfigurationSectionName() | |
| 29 { | |
| 30 return PLUGIN_SECTION; | |
| 31 } | |
| 1 | 32 |
| 33 class Writer : public IStoragePlugin::IWriter | |
| 34 { | |
| 35 std::string path_; | |
| 36 gcs::Client client_; | |
| 37 std::string bucketName_; | |
| 38 gcs::ObjectWriteStream stream_; | |
| 39 public: | |
| 40 Writer(const std::string& bucketName, const std::string& path, gcs::Client& client) | |
| 41 : path_(path), | |
| 42 client_(client), | |
| 43 bucketName_(bucketName) | |
| 44 { | |
| 45 } | |
| 46 | |
| 47 virtual ~Writer() | |
| 48 { | |
| 49 } | |
| 50 | |
| 51 virtual void Write(const char* data, size_t size) | |
| 52 { | |
| 53 stream_ = client_.WriteObject(bucketName_, path_); | |
| 54 | |
| 55 if (stream_) | |
| 56 { | |
| 57 stream_.write(data, size); | |
| 58 stream_.Close(); | |
| 59 | |
| 60 if (!stream_.metadata()) | |
| 61 { | |
| 62 throw StoragePluginException("GoogleCloudStorage: error while writing file " + std::string(path_) + ": " + stream_.metadata().status().message()); | |
| 63 } | |
| 64 } | |
| 65 else | |
| 66 { | |
| 67 throw StoragePluginException("GoogleCloudStorage: error while opening/writing file " + std::string(path_) + ": " + stream_.metadata().status().message()); | |
| 68 } | |
| 69 } | |
| 70 }; | |
| 71 | |
| 72 | |
| 73 class Reader : public IStoragePlugin::IReader | |
| 74 { | |
| 75 std::string path_; | |
| 76 gcs::Client client_; | |
| 77 std::string bucketName_; | |
| 78 | |
| 79 public: | |
| 80 Reader(const std::string& bucketName, const std::string& path, gcs::Client& client) | |
| 81 : path_(path), | |
| 82 client_(client), | |
| 83 bucketName_(bucketName) | |
| 84 { | |
| 85 } | |
| 86 | |
| 87 virtual ~Reader() | |
| 88 { | |
| 89 | |
| 90 } | |
| 91 virtual size_t GetSize() | |
| 92 { | |
| 93 auto objectMetadata = client_.GetObjectMetadata(bucketName_, path_); | |
| 94 | |
| 95 if (objectMetadata) | |
| 96 { | |
| 97 std::uint64_t fileSize = static_cast<int64_t>(objectMetadata->size()); | |
| 98 | |
| 99 return fileSize; | |
| 100 } | |
| 101 else | |
| 102 { | |
| 103 throw StoragePluginException("error while getting the size of " + std::string(path_) + ": " + objectMetadata.status().message()); | |
| 104 } | |
| 105 } | |
| 106 | |
|
40
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
107 virtual void ReadWhole(char* data, size_t size) |
| 1 | 108 { |
| 109 auto reader = client_.ReadObject(bucketName_, path_); | |
| 110 | |
| 111 if (!reader) | |
| 112 { | |
| 113 throw StoragePluginException("error while opening/reading file " + std::string(path_) + ": " + reader.status().message()); | |
| 114 } | |
| 115 | |
| 116 reader.read(data, size); | |
| 117 | |
| 118 if (!reader) | |
| 119 { | |
| 120 throw StoragePluginException("error while reading file " + std::string(path_) + ": " + reader.status().message()); | |
| 121 } | |
| 122 } | |
| 123 | |
|
40
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
124 virtual void ReadRange(char* data, size_t size, size_t fromOffset) |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
125 { |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
126 auto reader = client_.ReadObject(bucketName_, path_, gcs::ReadRange(fromOffset, fromOffset + size)); |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
127 |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
128 if (!reader) |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
129 { |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
130 throw StoragePluginException("error while opening/reading file " + std::string(path_) + ": " + reader.status().message()); |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
131 } |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
132 |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
133 reader.read(data, size); |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
134 |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
135 if (!reader) |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
136 { |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
137 throw StoragePluginException("error while reading file " + std::string(path_) + ": " + reader.status().message()); |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
138 } |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
139 } |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
140 |
|
516b6e731bb5
added support for ReadRange in google plugin
Alain Mazy <am@osimis.io>
parents:
37
diff
changeset
|
141 |
| 1 | 142 }; |
| 143 | |
| 144 | |
| 145 | |
| 146 const char* GoogleStoragePluginFactory::GetStoragePluginName() | |
| 147 { | |
| 148 return "Google Cloud Storage"; | |
| 149 } | |
| 150 | |
| 151 IStoragePlugin* GoogleStoragePluginFactory::CreateStoragePlugin(const OrthancPlugins::OrthancConfiguration& orthancConfig) | |
| 152 { | |
| 15 | 153 bool enableLegacyStorageStructure; |
| 154 | |
| 1 | 155 if (!orthancConfig.IsSection(PLUGIN_SECTION)) |
| 156 { | |
| 157 OrthancPlugins::LogWarning(std::string(GetStoragePluginName()) + " plugin, section missing. Plugin is not enabled."); | |
| 158 return nullptr; | |
| 159 } | |
| 160 | |
| 161 OrthancPlugins::OrthancConfiguration pluginSection; | |
| 162 orthancConfig.GetSection(pluginSection, PLUGIN_SECTION); | |
| 163 | |
| 15 | 164 if (!BaseStoragePlugin::ReadCommonConfiguration(enableLegacyStorageStructure, pluginSection)) |
| 165 { | |
| 166 return nullptr; | |
| 167 } | |
| 168 | |
| 1 | 169 std::string pathToGoogleCredentials; |
| 170 | |
| 171 if (!pluginSection.LookupStringValue(pathToGoogleCredentials, "ServiceAccountFile")) | |
| 172 { | |
| 173 OrthancPlugins::LogError("GoogleCloudStorage/ServiceAccountFile configuration missing. Unable to initialize plugin"); | |
| 174 return nullptr; | |
| 175 } | |
| 176 | |
| 177 std::string googleBucketName; | |
| 178 if (!pluginSection.LookupStringValue(googleBucketName, "BucketName")) | |
| 179 { | |
| 180 OrthancPlugins::LogError("GoogleCloudStorage/BucketName configuration missing. Unable to initialize plugin"); | |
| 181 return nullptr; | |
| 182 } | |
| 183 | |
| 184 // Use service account credentials from a JSON keyfile: | |
| 185 auto creds = gcs::oauth2::CreateServiceAccountCredentialsFromJsonFilePath(pathToGoogleCredentials); | |
| 186 if (!creds) | |
| 187 { | |
| 188 OrthancPlugins::LogError("GoogleCloudStorage plugin: unable to validate credentials. Check the ServiceAccountFile: " + creds.status().message()); | |
| 189 return nullptr; | |
| 190 } | |
| 191 | |
| 192 // Create a client to communicate with Google Cloud Storage. | |
| 193 google::cloud::StatusOr<gcs::Client> mainClient = gcs::Client(gcs::ClientOptions(*creds)); | |
| 194 | |
| 195 if (!mainClient) | |
| 196 { | |
| 197 OrthancPlugins::LogError("GoogleCloudStorage plugin: unable to create client: " + mainClient.status().message()); | |
| 198 return nullptr; | |
| 199 } | |
| 200 | |
| 15 | 201 return new GoogleStoragePlugin(googleBucketName, mainClient.value(), enableLegacyStorageStructure); |
| 1 | 202 } |
| 203 | |
| 15 | 204 GoogleStoragePlugin::GoogleStoragePlugin(const std::string &bucketName, google::cloud::storage::Client& mainClient, bool enableLegacyStorageStructure) |
| 205 : BaseStoragePlugin(enableLegacyStorageStructure), | |
| 206 bucketName_(bucketName), | |
| 1 | 207 mainClient_(mainClient) |
| 208 { | |
| 209 | |
| 210 } | |
| 211 | |
| 212 IStoragePlugin::IWriter* GoogleStoragePlugin::GetWriterForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
| 213 { | |
| 214 return new Writer(bucketName_, GetPath(uuid, type, encryptionEnabled), mainClient_); | |
| 215 } | |
| 216 | |
| 217 IStoragePlugin::IReader* GoogleStoragePlugin::GetReaderForObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
| 218 { | |
| 219 return new Reader(bucketName_, GetPath(uuid, type, encryptionEnabled), mainClient_); | |
| 220 } | |
| 221 | |
| 222 void GoogleStoragePlugin::DeleteObject(const char* uuid, OrthancPluginContentType type, bool encryptionEnabled) | |
| 223 { | |
| 224 gcs::Client client(mainClient_); | |
| 225 | |
| 226 std::string path = GetPath(uuid, type, encryptionEnabled); | |
| 227 | |
| 228 auto deletionStatus = client.DeleteObject(bucketName_, path); | |
| 229 | |
| 230 if (!deletionStatus.ok()) | |
| 231 { | |
| 232 throw StoragePluginException("GoogleCloudStorage: error while deleting file " + std::string(path) + ": " + deletionStatus.message()); | |
| 233 } | |
| 234 | |
| 235 } |