Mercurial > hg > orthanc-transfers
annotate Framework/TransferScheduler.cpp @ 72:d60d1a3a7357 OrthancTransfers-1.4
fix md5
author | Alain Mazy <am@osimis.io> |
---|---|
date | Tue, 26 Mar 2024 10:16:15 +0100 |
parents | 44a0430d7899 |
children | 1e396fb509ca |
rev | line source |
---|---|
0 | 1 /** |
2 * Transfers accelerator plugin for Orthanc | |
33
44a0430d7899
upgrade to year 2021
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
22
diff
changeset
|
3 * Copyright (C) 2018-2021 Osimis S.A., Belgium |
0 | 4 * |
5 * This program is free software: you can redistribute it and/or | |
6 * modify it under the terms of the GNU Affero General Public License | |
7 * as published by the Free Software Foundation, either version 3 of | |
8 * the License, or (at your option) any later version. | |
9 * | |
10 * This program is distributed in the hope that it will be useful, but | |
11 * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 * Affero General Public License for more details. | |
14 * | |
15 * You should have received a copy of the GNU Affero General Public License | |
16 * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
17 **/ | |
18 | |
19 | |
20 #include "TransferScheduler.h" | |
21 | |
22 | 22 #include "../Resources/Orthanc/Plugins/OrthancPluginCppWrapper.h" |
23 | |
20 | 24 #include <Logging.h> |
25 #include <OrthancException.h> | |
0 | 26 |
27 | |
28 namespace OrthancPlugins | |
29 { | |
30 void TransferScheduler::AddResource(OrthancInstancesCache& cache, | |
31 Orthanc::ResourceType level, | |
32 const std::string& id) | |
33 { | |
34 Json::Value resource; | |
35 | |
36 std::string base; | |
37 switch (level) | |
38 { | |
39 case Orthanc::ResourceType_Patient: | |
40 base = "patients"; | |
41 break; | |
42 | |
43 case Orthanc::ResourceType_Study: | |
44 base = "studies"; | |
45 break; | |
46 | |
47 case Orthanc::ResourceType_Series: | |
48 base = "series"; | |
49 break; | |
50 | |
51 default: | |
52 throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); | |
53 } | |
54 | |
8
4c3437217518
fix for compatibility with simplified OrthancPluginCppWrapper
Sebastien Jodogne <s.jodogne@gmail.com>
parents:
0
diff
changeset
|
55 if (RestApiGet(resource, "/" + base + "/" + id + "/instances", false)) |
0 | 56 { |
57 if (resource.type() != Json::arrayValue) | |
58 { | |
59 throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); | |
60 } | |
61 | |
62 for (Json::Value::ArrayIndex i = 0; i < resource.size(); i++) | |
63 { | |
64 if (resource[i].type() != Json::objectValue || | |
65 !resource[i].isMember(KEY_ID) || | |
66 resource[i][KEY_ID].type() != Json::stringValue) | |
67 { | |
68 throw Orthanc::OrthancException(Orthanc::ErrorCode_InternalError); | |
69 } | |
70 | |
71 AddInstance(cache, resource[i][KEY_ID].asString()); | |
72 } | |
73 } | |
74 else | |
75 { | |
76 std::string s = Orthanc::EnumerationToString(level); | |
77 Orthanc::Toolbox::ToLowerCase(s); | |
78 LOG(WARNING) << "Missing " << s << ": " << id; | |
79 throw Orthanc::OrthancException(Orthanc::ErrorCode_UnknownResource); | |
80 } | |
81 } | |
82 | |
83 | |
84 void TransferScheduler::ComputeBucketsInternal(std::vector<TransferBucket>& target, | |
85 size_t groupThreshold, | |
86 size_t separateThreshold, | |
87 const std::string& baseUrl, /* only needed in pull mode */ | |
88 BucketCompression compression /* only needed in pull mode */) const | |
89 { | |
90 if (groupThreshold > separateThreshold || | |
91 separateThreshold == 0) // (*) | |
92 { | |
93 throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); | |
94 } | |
95 | |
96 target.clear(); | |
97 | |
98 std::list<std::string> toGroup_; | |
99 | |
100 for (Instances::const_iterator it = instances_.begin(); | |
101 it != instances_.end(); ++it) | |
102 { | |
103 size_t size = it->second.GetSize(); | |
104 | |
105 if (size < groupThreshold) | |
106 { | |
107 toGroup_.push_back(it->first); | |
108 } | |
109 else if (size < separateThreshold) | |
110 { | |
111 // Send the whole instance as it is | |
112 TransferBucket bucket; | |
113 bucket.AddChunk(it->second, 0, size); | |
114 target.push_back(bucket); | |
115 } | |
116 else | |
117 { | |
118 // Divide this large instance as a set of chunks | |
119 size_t chunksCount; | |
120 | |
121 if (size % separateThreshold == 0) | |
122 { | |
123 chunksCount = size / separateThreshold; | |
124 } | |
125 else | |
126 { | |
127 chunksCount = size / separateThreshold + 1; | |
128 } | |
129 | |
130 assert(chunksCount != 0); // This follows from (*) | |
131 | |
132 size_t chunkSize = size / chunksCount; | |
133 size_t offset = 0; | |
134 | |
135 for (size_t i = 0; i < chunksCount; i++, offset += chunkSize) | |
136 { | |
137 TransferBucket bucket; | |
138 | |
139 if (i == chunksCount - 1) | |
140 { | |
141 // The last chunk must contain all the remaining bytes | |
142 // of the instance (correction of rounding effects) | |
143 bucket.AddChunk(it->second, offset, size - offset); | |
144 } | |
145 else | |
146 { | |
147 bucket.AddChunk(it->second, offset, chunkSize); | |
148 } | |
149 | |
150 target.push_back(bucket); | |
151 } | |
152 } | |
153 } | |
154 | |
155 // Grouping the remaining small instances, preventing the | |
156 // download URL from getting too long: "If you keep URLs under | |
157 // 2000 characters, they'll work in virtually any combination of | |
158 // client and server software." | |
159 // https://stackoverflow.com/a/417184/881731 | |
160 | |
161 static const size_t MAX_URL_LENGTH = 2000 - 44 /* size of an Orthanc identifier (SHA-1) */; | |
162 | |
163 TransferBucket bucket; | |
164 | |
165 for (std::list<std::string>::const_iterator it = toGroup_.begin(); | |
166 it != toGroup_.end(); ++it) | |
167 { | |
168 Instances::const_iterator instance = instances_.find(*it); | |
169 assert(instance != instances_.end()); | |
170 | |
171 bucket.AddChunk(instance->second, 0, instance->second.GetSize()); | |
172 | |
173 bool full = (bucket.GetTotalSize() >= groupThreshold); | |
174 | |
175 if (!full && !baseUrl.empty()) | |
176 { | |
177 std::string uri; | |
178 bucket.ComputePullUri(uri, compression); | |
179 | |
180 std::string url = baseUrl + uri; | |
181 full = (url.length() >= MAX_URL_LENGTH); | |
182 } | |
183 | |
184 if (full) | |
185 { | |
186 target.push_back(bucket); | |
187 bucket.Clear(); | |
188 } | |
189 } | |
190 | |
191 if (bucket.GetChunksCount() > 0) | |
192 { | |
193 target.push_back(bucket); | |
194 } | |
195 } | |
196 | |
197 | |
198 void TransferScheduler::AddInstance(OrthancInstancesCache& cache, | |
199 const std::string& instanceId) | |
200 { | |
201 size_t size; | |
202 std::string md5; | |
203 cache.GetInstanceInfo(size, md5, instanceId); | |
204 | |
205 AddInstance(DicomInstanceInfo(instanceId, size, md5)); | |
206 } | |
207 | |
208 | |
209 void TransferScheduler::AddInstance(const DicomInstanceInfo& info) | |
210 { | |
211 instances_[info.GetId()] = info; | |
212 } | |
213 | |
214 | |
215 void TransferScheduler::ParseListOfResources(OrthancInstancesCache& cache, | |
216 const Json::Value& resources) | |
217 { | |
218 if (resources.type() != Json::arrayValue) | |
219 { | |
220 throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat); | |
221 } | |
222 | |
223 for (Json::Value::ArrayIndex i = 0; i < resources.size(); i++) | |
224 { | |
225 if (resources[i].type() != Json::objectValue || | |
226 !resources[i].isMember(KEY_LEVEL) || | |
227 !resources[i].isMember(KEY_ID) || | |
228 resources[i][KEY_LEVEL].type() != Json::stringValue || | |
229 resources[i][KEY_ID].type() != Json::stringValue) | |
230 { | |
231 throw Orthanc::OrthancException(Orthanc::ErrorCode_BadFileFormat); | |
232 } | |
233 else | |
234 { | |
235 Orthanc::ResourceType level = Orthanc::StringToResourceType(resources[i][KEY_LEVEL].asCString()); | |
236 | |
237 switch (level) | |
238 { | |
239 case Orthanc::ResourceType_Patient: | |
240 AddPatient(cache, resources[i][KEY_ID].asString()); | |
241 break; | |
242 | |
243 case Orthanc::ResourceType_Study: | |
244 AddStudy(cache, resources[i][KEY_ID].asString()); | |
245 break; | |
246 | |
247 case Orthanc::ResourceType_Series: | |
248 AddSeries(cache, resources[i][KEY_ID].asString()); | |
249 break; | |
250 | |
251 case Orthanc::ResourceType_Instance: | |
252 AddInstance(cache, resources[i][KEY_ID].asString()); | |
253 break; | |
254 | |
255 default: | |
256 throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); | |
257 } | |
258 } | |
259 } | |
260 } | |
261 | |
262 | |
263 void TransferScheduler::ListInstances(std::vector<DicomInstanceInfo>& target) const | |
264 { | |
265 target.clear(); | |
266 target.reserve(instances_.size()); | |
267 | |
268 for (Instances::const_iterator it = instances_.begin(); | |
269 it != instances_.end(); ++it) | |
270 { | |
271 assert(it->first == it->second.GetId()); | |
272 target.push_back(it->second); | |
273 } | |
274 } | |
275 | |
276 | |
277 size_t TransferScheduler::GetTotalSize() const | |
278 { | |
279 size_t size = 0; | |
280 | |
281 for (Instances::const_iterator it = instances_.begin(); | |
282 it != instances_.end(); ++it) | |
283 { | |
284 size += it->second.GetSize(); | |
285 } | |
286 | |
287 return size; | |
288 } | |
289 | |
290 | |
291 void TransferScheduler::ComputePullBuckets(std::vector<TransferBucket>& target, | |
292 size_t groupThreshold, | |
293 size_t separateThreshold, | |
294 const std::string& baseUrl, | |
295 BucketCompression compression) const | |
296 { | |
297 ComputeBucketsInternal(target, groupThreshold, separateThreshold, baseUrl, compression); | |
298 } | |
299 | |
300 | |
301 void TransferScheduler::FormatPushTransaction(Json::Value& target, | |
302 std::vector<TransferBucket>& buckets, | |
303 size_t groupThreshold, | |
304 size_t separateThreshold, | |
305 BucketCompression compression) const | |
306 { | |
307 ComputeBucketsInternal(buckets, groupThreshold, separateThreshold, "", BucketCompression_None); | |
308 | |
309 target = Json::objectValue; | |
310 | |
311 Json::Value tmp = Json::arrayValue; | |
312 | |
313 for (Instances::const_iterator it = instances_.begin(); | |
314 it != instances_.end(); ++it) | |
315 { | |
316 Json::Value item; | |
317 it->second.Serialize(item); | |
318 tmp.append(item); | |
319 } | |
320 | |
321 target[KEY_INSTANCES] = tmp; | |
322 | |
323 tmp = Json::arrayValue; | |
324 | |
325 for (size_t i = 0; i < buckets.size(); i++) | |
326 { | |
327 Json::Value item; | |
328 buckets[i].Serialize(item); | |
329 tmp.append(item); | |
330 } | |
331 | |
332 target[KEY_BUCKETS] = tmp; | |
333 | |
334 switch (compression) | |
335 { | |
336 case BucketCompression_Gzip: | |
337 target[KEY_COMPRESSION] = "gzip"; | |
338 break; | |
339 | |
340 case BucketCompression_None: | |
341 target[KEY_COMPRESSION] = "none"; | |
342 break; | |
343 | |
344 default: | |
345 throw Orthanc::OrthancException(Orthanc::ErrorCode_ParameterOutOfRange); | |
346 } | |
347 } | |
348 } |