Mercurial > hg > orthanc-transfers
annotate Framework/TransferScheduler.cpp @ 77:1e396fb509ca default
updated copyright, as Orthanc Team now replaces Osimis
| author | Sebastien Jodogne <s.jodogne@gmail.com> |
|---|---|
| date | Thu, 30 May 2024 22:44:10 +0200 |
| parents | 44a0430d7899 |
| children |
| rev | line source |
|---|---|
| 0 | 1 /** |
| 2 * Transfers accelerator plugin for Orthanc | |
|
77
1e396fb509ca
updated copyright, as Orthanc Team now replaces Osimis
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
33
diff
changeset
|
3 * Copyright (C) 2018-2023 Osimis S.A., Belgium |
|
1e396fb509ca
updated copyright, as Orthanc Team now replaces Osimis
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
33
diff
changeset
|
4 * Copyright (C) 2024-2024 Orthanc Team SRL, Belgium |
|
1e396fb509ca
updated copyright, as Orthanc Team now replaces Osimis
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
33
diff
changeset
|
5 * Copyright (C) 2021-2024 Sebastien Jodogne, ICTEAM UCLouvain, Belgium |
| 0 | 6 * |
| 7 * This program is free software: you can redistribute it and/or | |
| 8 * modify it under the terms of the GNU Affero General Public License | |
| 9 * as published by the Free Software Foundation, either version 3 of | |
| 10 * the License, or (at your option) any later version. | |
| 11 * | |
| 12 * This program is distributed in the hope that it will be useful, but | |
| 13 * WITHOUT ANY WARRANTY; without even the implied warranty of | |
| 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| 15 * Affero General Public License for more details. | |
| 16 * | |
| 17 * You should have received a copy of the GNU Affero General Public License | |
| 18 * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| 19 **/ | |
| 20 | |
| 21 | |
| 22 #include "TransferScheduler.h" | |
| 23 | |
| 22 | 24 #include "../Resources/Orthanc/Plugins/OrthancPluginCppWrapper.h" |
| 25 | |
| 20 | 26 #include <Logging.h> |
| 27 #include <OrthancException.h> | |
| 0 | 28 |
| 29 | |
| 30 namespace OrthancPlugins | |
| 31 { | |
| 32 void TransferScheduler::AddResource(OrthancInstancesCache& cache, | |
| 33 Orthanc::ResourceType level, | |
| 34 const std::string& id) | |
| 35 { | |
| 36 Json::Value resource; | |
| 37 | |
| 38 std::string base; | |
| 39 switch (level) | |
| 40 { | |
| 41 case Orthanc::ResourceType_Patient: | |
| 42 base = "patients"; | |
| 43 break; | |
| 44 | |
| 45 case Orthanc::ResourceType_Study: | |
| 46 base = "studies"; | |
| 47 break; | |
| 48 | |
| 49 case Orthanc::ResourceType_Series: | |
| 50 base = "series"; | |
| 51 break; | |
| 52 | |
| 53 default: | |
| 54 throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); | |
| 55 } | |
| 56 | |
|
8
4c3437217518
fix for compatibility with simplified OrthancPluginCppWrapper
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
0
diff
changeset
|
57 if (RestApiGet(resource, "/" + base + "/" + id + "/instances", false)) |
| 0 | 58 { |
| 59 if (resource.type() != Json::arrayValue) | |
| 60 { | |
| 61 throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); | |
| 62 } | |
| 63 | |
| 64 for (Json::Value::ArrayIndex i = 0; i < resource.size(); i++) | |
| 65 { | |
| 66 if (resource[i].type() != Json::objectValue || | |
| 67 !resource[i].isMember(KEY_ID) || | |
| 68 resource[i][KEY_ID].type() != Json::stringValue) | |
| 69 { | |
| 70 throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); | |
| 71 } | |
| 72 | |
| 73 AddInstance(cache, resource[i][KEY_ID].asString()); | |
| 74 } | |
| 75 } | |
| 76 else | |
| 77 { | |
| 78 std::string s = Orthanc::EnumerationToString(level); | |
| 79 Orthanc::Toolbox::ToLowerCase(s); | |
| 80 LOG(WARNING) << "Missing " << s << ": " << id; | |
| 81 throw Orthanc::OrthancException(Orthanc::ErrorCode_UnknownResource); | |
| 82 } | |
| 83 } | |
| 84 | |
| 85 | |
| 86 void TransferScheduler::ComputeBucketsInternal(std::vector<TransferBucket>& target, | |
| 87 size_t groupThreshold, | |
| 88 size_t separateThreshold, | |
| 89 const std::string& baseUrl, /* only needed in pull mode */ | |
| 90 BucketCompression compression /* only needed in pull mode */) const | |
| 91 { | |
| 92 if (groupThreshold > separateThreshold || | |
| 93 separateThreshold == 0) // (*) | |
| 94 { | |
| 95 throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); | |
| 96 } | |
| 97 | |
| 98 target.clear(); | |
| 99 | |
| 100 std::list<std::string> toGroup_; | |
| 101 | |
| 102 for (Instances::const_iterator it = instances_.begin(); | |
| 103 it != instances_.end(); ++it) | |
| 104 { | |
| 105 size_t size = it->second.GetSize(); | |
| 106 | |
| 107 if (size < groupThreshold) | |
| 108 { | |
| 109 toGroup_.push_back(it->first); | |
| 110 } | |
| 111 else if (size < separateThreshold) | |
| 112 { | |
| 113 // Send the whole instance as it is | |
| 114 TransferBucket bucket; | |
| 115 bucket.AddChunk(it->second, 0, size); | |
| 116 target.push_back(bucket); | |
| 117 } | |
| 118 else | |
| 119 { | |
| 120 // Divide this large instance as a set of chunks | |
| 121 size_t chunksCount; | |
| 122 | |
| 123 if (size % separateThreshold == 0) | |
| 124 { | |
| 125 chunksCount = size / separateThreshold; | |
| 126 } | |
| 127 else | |
| 128 { | |
| 129 chunksCount = size / separateThreshold + 1; | |
| 130 } | |
| 131 | |
| 132 assert(chunksCount != 0); // This follows from (*) | |
| 133 | |
| 134 size_t chunkSize = size / chunksCount; | |
| 135 size_t offset = 0; | |
| 136 | |
| 137 for (size_t i = 0; i < chunksCount; i++, offset += chunkSize) | |
| 138 { | |
| 139 TransferBucket bucket; | |
| 140 | |
| 141 if (i == chunksCount - 1) | |
| 142 { | |
| 143 // The last chunk must contain all the remaining bytes | |
| 144 // of the instance (correction of rounding effects) | |
| 145 bucket.AddChunk(it->second, offset, size - offset); | |
| 146 } | |
| 147 else | |
| 148 { | |
| 149 bucket.AddChunk(it->second, offset, chunkSize); | |
| 150 } | |
| 151 | |
| 152 target.push_back(bucket); | |
| 153 } | |
| 154 } | |
| 155 } | |
| 156 | |
| 157 // Grouping the remaining small instances, preventing the | |
| 158 // download URL from getting too long: "If you keep URLs under | |
| 159 // 2000 characters, they'll work in virtually any combination of | |
| 160 // client and server software." | |
| 161 // https://stackoverflow.com/a/417184/881731 | |
| 162 | |
| 163 static const size_t MAX_URL_LENGTH = 2000 - 44 /* size of an Orthanc identifier (SHA-1) */; | |
| 164 | |
| 165 TransferBucket bucket; | |
| 166 | |
| 167 for (std::list<std::string>::const_iterator it = toGroup_.begin(); | |
| 168 it != toGroup_.end(); ++it) | |
| 169 { | |
| 170 Instances::const_iterator instance = instances_.find(*it); | |
| 171 assert(instance != instances_.end()); | |
| 172 | |
| 173 bucket.AddChunk(instance->second, 0, instance->second.GetSize()); | |
| 174 | |
| 175 bool full = (bucket.GetTotalSize() >= groupThreshold); | |
| 176 | |
| 177 if (!full && !baseUrl.empty()) | |
| 178 { | |
| 179 std::string uri; | |
| 180 bucket.ComputePullUri(uri, compression); | |
| 181 | |
| 182 std::string url = baseUrl + uri; | |
| 183 full = (url.length() >= MAX_URL_LENGTH); | |
| 184 } | |
| 185 | |
| 186 if (full) | |
| 187 { | |
| 188 target.push_back(bucket); | |
| 189 bucket.Clear(); | |
| 190 } | |
| 191 } | |
| 192 | |
| 193 if (bucket.GetChunksCount() > 0) | |
| 194 { | |
| 195 target.push_back(bucket); | |
| 196 } | |
| 197 } | |
| 198 | |
| 199 | |
| 200 void TransferScheduler::AddInstance(OrthancInstancesCache& cache, | |
| 201 const std::string& instanceId) | |
| 202 { | |
| 203 size_t size; | |
| 204 std::string md5; | |
| 205 cache.GetInstanceInfo(size, md5, instanceId); | |
| 206 | |
| 207 AddInstance(DicomInstanceInfo(instanceId, size, md5)); | |
| 208 } | |
| 209 | |
| 210 | |
| 211 void TransferScheduler::AddInstance(const DicomInstanceInfo& info) | |
| 212 { | |
| 213 instances_[info.GetId()] = info; | |
| 214 } | |
| 215 | |
| 216 | |
| 217 void TransferScheduler::ParseListOfResources(OrthancInstancesCache& cache, | |
| 218 const Json::Value& resources) | |
| 219 { | |
| 220 if (resources.type() != Json::arrayValue) | |
| 221 { | |
| 222 throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat); | |
| 223 } | |
| 224 | |
| 225 for (Json::Value::ArrayIndex i = 0; i < resources.size(); i++) | |
| 226 { | |
| 227 if (resources[i].type() != Json::objectValue || | |
| 228 !resources[i].isMember(KEY_LEVEL) || | |
| 229 !resources[i].isMember(KEY_ID) || | |
| 230 resources[i][KEY_LEVEL].type() != Json::stringValue || | |
| 231 resources[i][KEY_ID].type() != Json::stringValue) | |
| 232 { | |
| 233 throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat); | |
| 234 } | |
| 235 else | |
| 236 { | |
| 237 Orthanc::ResourceType level = Orthanc::StringToResourceType(resources[i][KEY_LEVEL].asCString()); | |
| 238 | |
| 239 switch (level) | |
| 240 { | |
| 241 case Orthanc::ResourceType_Patient: | |
| 242 AddPatient(cache, resources[i][KEY_ID].asString()); | |
| 243 break; | |
| 244 | |
| 245 case Orthanc::ResourceType_Study: | |
| 246 AddStudy(cache, resources[i][KEY_ID].asString()); | |
| 247 break; | |
| 248 | |
| 249 case Orthanc::ResourceType_Series: | |
| 250 AddSeries(cache, resources[i][KEY_ID].asString()); | |
| 251 break; | |
| 252 | |
| 253 case Orthanc::ResourceType_Instance: | |
| 254 AddInstance(cache, resources[i][KEY_ID].asString()); | |
| 255 break; | |
| 256 | |
| 257 default: | |
| 258 throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); | |
| 259 } | |
| 260 } | |
| 261 } | |
| 262 } | |
| 263 | |
| 264 | |
| 265 void TransferScheduler::ListInstances(std::vector<DicomInstanceInfo>& target) const | |
| 266 { | |
| 267 target.clear(); | |
| 268 target.reserve(instances_.size()); | |
| 269 | |
| 270 for (Instances::const_iterator it = instances_.begin(); | |
| 271 it != instances_.end(); ++it) | |
| 272 { | |
| 273 assert(it->first == it->second.GetId()); | |
| 274 target.push_back(it->second); | |
| 275 } | |
| 276 } | |
| 277 | |
| 278 | |
| 279 size_t TransferScheduler::GetTotalSize() const | |
| 280 { | |
| 281 size_t size = 0; | |
| 282 | |
| 283 for (Instances::const_iterator it = instances_.begin(); | |
| 284 it != instances_.end(); ++it) | |
| 285 { | |
| 286 size += it->second.GetSize(); | |
| 287 } | |
| 288 | |
| 289 return size; | |
| 290 } | |
| 291 | |
| 292 | |
| 293 void TransferScheduler::ComputePullBuckets(std::vector<TransferBucket>& target, | |
| 294 size_t groupThreshold, | |
| 295 size_t separateThreshold, | |
| 296 const std::string& baseUrl, | |
| 297 BucketCompression compression) const | |
| 298 { | |
| 299 ComputeBucketsInternal(target, groupThreshold, separateThreshold, baseUrl, compression); | |
| 300 } | |
| 301 | |
| 302 | |
| 303 void TransferScheduler::FormatPushTransaction(Json::Value& target, | |
| 304 std::vector<TransferBucket>& buckets, | |
| 305 size_t groupThreshold, | |
| 306 size_t separateThreshold, | |
| 307 BucketCompression compression) const | |
| 308 { | |
| 309 ComputeBucketsInternal(buckets, groupThreshold, separateThreshold, "", BucketCompression_None); | |
| 310 | |
| 311 target = Json::objectValue; | |
| 312 | |
| 313 Json::Value tmp = Json::arrayValue; | |
| 314 | |
| 315 for (Instances::const_iterator it = instances_.begin(); | |
| 316 it != instances_.end(); ++it) | |
| 317 { | |
| 318 Json::Value item; | |
| 319 it->second.Serialize(item); | |
| 320 tmp.append(item); | |
| 321 } | |
| 322 | |
| 323 target[KEY_INSTANCES] = tmp; | |
| 324 | |
| 325 tmp = Json::arrayValue; | |
| 326 | |
| 327 for (size_t i = 0; i < buckets.size(); i++) | |
| 328 { | |
| 329 Json::Value item; | |
| 330 buckets[i].Serialize(item); | |
| 331 tmp.append(item); | |
| 332 } | |
| 333 | |
| 334 target[KEY_BUCKETS] = tmp; | |
| 335 | |
| 336 switch (compression) | |
| 337 { | |
| 338 case BucketCompression_Gzip: | |
| 339 target[KEY_COMPRESSION] = "gzip"; | |
| 340 break; | |
| 341 | |
| 342 case BucketCompression_None: | |
| 343 target[KEY_COMPRESSION] = "none"; | |
| 344 break; | |
| 345 | |
| 346 default: | |
| 347 throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); | |
| 348 } | |
| 349 } | |
| 350 } |