Mercurial > hg > orthanc

comparison OrthancServer/Plugins/Samples/DbOptimizer/Plugin.cpp @ 4961:1b76853e1797 more-tags

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
DbOptimizer plugin
author Alain Mazy <am@osimis.io>
date Wed, 23 Mar 2022 11:56:28 +0100
parents
children
comparison
equal deleted inserted replaced
4960:c68265bf1f94 4961:1b76853e1797
1 /**
2 * Orthanc - A Lightweight, RESTful DICOM Store
3 * Copyright (C) 2012-2016 Sebastien Jodogne, Medical Physics
4 * Department, University Hospital of Liege, Belgium
5 * Copyright (C) 2017-2022 Osimis S.A., Belgium
6 * Copyright (C) 2021-2022 Sebastien Jodogne, ICTEAM UCLouvain, Belgium
7 *
8 * This program is free software: you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License as
10 * published by the Free Software Foundation, either version 3 of the
11 * License, or (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 **/
21
22
23 #include "../../../../OrthancFramework/Sources/Compatibility.h"
24 #include "../Common/OrthancPluginCppWrapper.h"
25
26 #include <boost/thread.hpp>
27 #include <json/value.h>
28 #include <json/writer.h>
29 #include <string.h>
30 #include <iostream>
31 #include <algorithm>
32 #include <map>
33
34 static int globalPropertyId_ = 0;
35 static bool force_ = false;
36 static uint throttleDelay_ = 0;
37 static std::unique_ptr<boost::thread> workerThread_;
38 static bool workerThreadShouldStop = false;
39
40 struct DbConfiguration
41 {
42 std::string orthancVersion;
43 std::map<OrthancPluginResourceType, std::string> mainDicomTagsSignature;
44
45 DbConfiguration()
46 {
47 }
48
49 bool IsDefined() const
50 {
51 return !orthancVersion.empty() && mainDicomTagsSignature.size() == 4;
52 }
53
54 void Clear()
55 {
56 orthancVersion.clear();
57 mainDicomTagsSignature.clear();
58 }
59
60 void ToJson(Json::Value& target)
61 {
62 if (!IsDefined())
63 {
64 target = Json::nullValue;
65 }
66 else
67 {
68 Json::Value signatures;
69
70 target = Json::objectValue;
71
72 // default main dicom tags signature are the one from Orthanc 1.4.2 (last time the list was changed):
73 signatures["Patient"] = mainDicomTagsSignature[OrthancPluginResourceType_Patient];
74 signatures["Study"] = mainDicomTagsSignature[OrthancPluginResourceType_Study];
75 signatures["Series"] = mainDicomTagsSignature[OrthancPluginResourceType_Series];
76 signatures["Instance"] = mainDicomTagsSignature[OrthancPluginResourceType_Instance];
77
78 target["MainDicomTagsSignature"] = signatures;
79 target["OrthancVersion"] = orthancVersion;
80 }
81 }
82
83 void FromJson(Json::Value& source)
84 {
85 if (!source.isNull())
86 {
87 orthancVersion = source["OrthancVersion"].asString();
88
89 const Json::Value& signatures = source["MainDicomTagsSignature"];
90 mainDicomTagsSignature[OrthancPluginResourceType_Patient] = signatures["Patient"].asString();
91 mainDicomTagsSignature[OrthancPluginResourceType_Study] = signatures["Study"].asString();
92 mainDicomTagsSignature[OrthancPluginResourceType_Series] = signatures["Series"].asString();
93 mainDicomTagsSignature[OrthancPluginResourceType_Instance] = signatures["Instance"].asString();
94 }
95 }
96 };
97
98 struct PluginStatus
99 {
100 int statusVersion;
101 int64_t lastProcessedChange;
102 int64_t lastChangeToProcess;
103
104 DbConfiguration currentlyProcessingConfiguration; // last configuration being processed (has not reached last change yet)
105 DbConfiguration lastProcessedConfiguration; // last configuration that has been fully processed (till last change)
106
107 PluginStatus()
108 : statusVersion(1),
109 lastProcessedChange(-1),
110 lastChangeToProcess(-1)
111 {
112 }
113
114 void ToJson(Json::Value& target)
115 {
116 target = Json::objectValue;
117
118 target["Version"] = statusVersion;
119 target["LastProcessedChange"] = Json::Value::Int64(lastProcessedChange);
120 target["LastChangeToProcess"] = Json::Value::Int64(lastChangeToProcess);
121
122 currentlyProcessingConfiguration.ToJson(target["CurrentlyProcessingConfiguration"]);
123 lastProcessedConfiguration.ToJson(target["LastProcessedConfiguration"]);
124 }
125
126 void FromJson(Json::Value& source)
127 {
128 statusVersion = source["Version"].asInt();
129 lastProcessedChange = source["LastProcessedChange"].asInt64();
130 lastChangeToProcess = source["LastChangeToProcess"].asInt64();
131
132 Json::Value& current = source["CurrentlyProcessingConfiguration"];
133 Json::Value& last = source["LastProcessedConfiguration"];
134
135 currentlyProcessingConfiguration.FromJson(current);
136 lastProcessedConfiguration.FromJson(last);
137 }
138 };
139
140
141 static void ReadStatusFromDb(PluginStatus& pluginStatus)
142 {
143 OrthancPlugins::OrthancString globalPropertyContent;
144
145 globalPropertyContent.Assign(OrthancPluginGetGlobalProperty(OrthancPlugins::GetGlobalContext(),
146 globalPropertyId_,
147 ""));
148
149 if (!globalPropertyContent.IsNullOrEmpty())
150 {
151 Json::Value jsonStatus;
152 globalPropertyContent.ToJson(jsonStatus);
153 pluginStatus.FromJson(jsonStatus);
154 }
155 else
156 {
157 // default config
158 pluginStatus.statusVersion = 1;
159 pluginStatus.lastProcessedChange = -1;
160 pluginStatus.lastChangeToProcess = -1;
161
162 pluginStatus.currentlyProcessingConfiguration.orthancVersion = "1.9.0"; // when we don't know, we assume some files were stored with Orthanc 1.9.0 (last version saving the dicom-as-json files)
163
164 // default main dicom tags signature are the one from Orthanc 1.4.2 (last time the list was changed):
165 pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Patient] = "0010,0010;0010,0020;0010,0030;0010,0040;0010,1000";
166 pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Study] = "0008,0020;0008,0030;0008,0050;0008,0080;0008,0090;0008,1030;0020,000d;0020,0010;0032,1032;0032,1060";
167 pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Series] = "0008,0021;0008,0031;0008,0060;0008,0070;0008,1010;0008,103e;0008,1070;0018,0010;0018,0015;0018,0024;0018,1030;0018,1090;0018,1400;0020,000e;0020,0011;0020,0037;0020,0105;0020,1002;0040,0254;0054,0081;0054,0101;0054,1000";
168 pluginStatus.currentlyProcessingConfiguration.mainDicomTagsSignature[OrthancPluginResourceType_Instance] = "0008,0012;0008,0013;0008,0018;0020,0012;0020,0013;0020,0032;0020,0037;0020,0100;0020,4000;0028,0008;0054,1330";
169 }
170 }
171
172 static void SaveStatusInDb(PluginStatus& pluginStatus)
173 {
174 Json::Value jsonStatus;
175 pluginStatus.ToJson(jsonStatus);
176
177 Json::StreamWriterBuilder builder;
178 builder.settings_["indentation"] = " ";
179 std::string serializedStatus = Json::writeString(builder, jsonStatus);
180
181 OrthancPluginSetGlobalProperty(OrthancPlugins::GetGlobalContext(),
182 globalPropertyId_,
183 serializedStatus.c_str());
184 }
185
186 static void GetCurrentDbConfiguration(DbConfiguration& configuration)
187 {
188 Json::Value signatures;
189 Json::Value systemInfo;
190
191 OrthancPlugins::RestApiGet(systemInfo, "/system", false);
192 configuration.mainDicomTagsSignature[OrthancPluginResourceType_Patient] = systemInfo["MainDicomTags"]["Patient"].asString();
193 configuration.mainDicomTagsSignature[OrthancPluginResourceType_Study] = systemInfo["MainDicomTags"]["Study"].asString();
194 configuration.mainDicomTagsSignature[OrthancPluginResourceType_Series] = systemInfo["MainDicomTags"]["Series"].asString();
195 configuration.mainDicomTagsSignature[OrthancPluginResourceType_Instance] = systemInfo["MainDicomTags"]["Instance"].asString();
196
197 configuration.orthancVersion = OrthancPlugins::GetGlobalContext()->orthancVersion;
198 }
199
200 static bool NeedsProcessing(const DbConfiguration& current, const DbConfiguration& last)
201 {
202 if (!last.IsDefined())
203 {
204 return true;
205 }
206
207 const char* lastVersion = last.orthancVersion.c_str();
208 const std::map<OrthancPluginResourceType, std::string>& lastTags = last.mainDicomTagsSignature;
209 const std::map<OrthancPluginResourceType, std::string>& currentTags = current.mainDicomTagsSignature;
210 bool needsProcessing = false;
211
212 if (!OrthancPlugins::CheckMinimalVersion(lastVersion, 1, 9, 1))
213 {
214 OrthancPlugins::LogWarning("DbOptimizer: your storage might still contain some dicom-as-json files -> will reconstruct DB");
215 needsProcessing = true;
216 }
217
218 if (lastTags.at(OrthancPluginResourceType_Patient) != currentTags.at(OrthancPluginResourceType_Patient))
219 {
220 OrthancPlugins::LogWarning("DbOptimizer: Patient main dicom tags have changed, -> will reconstruct DB");
221 needsProcessing = true;
222 }
223
224 if (lastTags.at(OrthancPluginResourceType_Study) != currentTags.at(OrthancPluginResourceType_Study))
225 {
226 OrthancPlugins::LogWarning("DbOptimizer: Study main dicom tags have changed, -> will reconstruct DB");
227 needsProcessing = true;
228 }
229
230 if (lastTags.at(OrthancPluginResourceType_Series) != currentTags.at(OrthancPluginResourceType_Series))
231 {
232 OrthancPlugins::LogWarning("DbOptimizer: Series main dicom tags have changed, -> will reconstruct DB");
233 needsProcessing = true;
234 }
235
236 if (lastTags.at(OrthancPluginResourceType_Instance) != currentTags.at(OrthancPluginResourceType_Instance))
237 {
238 OrthancPlugins::LogWarning("DbOptimizer: Instance main dicom tags have changed, -> will reconstruct DB");
239 needsProcessing = true;
240 }
241
242 return needsProcessing;
243 }
244
245 static bool ProcessChanges(PluginStatus& pluginStatus, const DbConfiguration& currentDbConfiguration)
246 {
247 Json::Value changes;
248
249 pluginStatus.currentlyProcessingConfiguration = currentDbConfiguration;
250
251 OrthancPlugins::RestApiGet(changes, "/changes?since=" + boost::lexical_cast<std::string>(pluginStatus.lastProcessedChange) + "&limit=100", false);
252
253 for (Json::ArrayIndex i = 0; i < changes["Changes"].size(); i++)
254 {
255 const Json::Value& change = changes["Changes"][i];
256 int64_t seq = change["Seq"].asInt64();
257
258 if (change["ChangeType"] == "NewStudy") // some StableStudy might be missing if orthanc was shutdown during a StableAge -> consider only the NewStudy events that can not be missed
259 {
260 Json::Value result;
261 OrthancPlugins::RestApiPost(result, "/studies/" + change["ID"].asString() + "/reconstruct", std::string(""), false);
262 boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelay_*1000));
263 }
264
265 if (seq >= pluginStatus.lastChangeToProcess) // we are done !
266 {
267 return true;
268 }
269
270 pluginStatus.lastProcessedChange = seq;
271 }
272
273 return false;
274 }
275
276
277 static void WorkerThread()
278 {
279 PluginStatus pluginStatus;
280 DbConfiguration currentDbConfiguration;
281
282 OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "Starting DB optimizer worker thread");
283
284 ReadStatusFromDb(pluginStatus);
285 GetCurrentDbConfiguration(currentDbConfiguration);
286
287 if (!NeedsProcessing(currentDbConfiguration, pluginStatus.lastProcessedConfiguration))
288 {
289 OrthancPlugins::LogWarning("DbOptimizer: everything has been processed already !");
290 return;
291 }
292
293 if (force_ || NeedsProcessing(currentDbConfiguration, pluginStatus.currentlyProcessingConfiguration))
294 {
295 if (force_)
296 {
297 OrthancPlugins::LogWarning("DbOptimizer: forcing execution -> will reconstruct DB");
298 }
299 else
300 {
301 OrthancPlugins::LogWarning("DbOptimizer: the DB configuration has changed since last run, will reprocess the whole DB !");
302 }
303
304 Json::Value changes;
305 OrthancPlugins::RestApiGet(changes, "/changes?last", false);
306
307 pluginStatus.lastProcessedChange = 0;
308 pluginStatus.lastChangeToProcess = changes["Last"].asInt64(); // the last change is the last change at the time we start. We assume that every new ingested file will be constructed correctly
309 }
310 else
311 {
312 OrthancPlugins::LogWarning("DbOptimizer: the DB configuration has not changed since last run, will continue processing changes");
313 }
314
315 bool completed = pluginStatus.lastChangeToProcess == 0; // if the DB is empty at start, no need to process anyting
316 while (!workerThreadShouldStop && !completed)
317 {
318 completed = ProcessChanges(pluginStatus, currentDbConfiguration);
319 SaveStatusInDb(pluginStatus);
320
321 if (!completed)
322 {
323 OrthancPlugins::LogInfo("DbOptimizer: processed changes " +
324 boost::lexical_cast<std::string>(pluginStatus.lastProcessedChange) +
325 " / " + boost::lexical_cast<std::string>(pluginStatus.lastChangeToProcess));
326
327 boost::this_thread::sleep(boost::posix_time::milliseconds(throttleDelay_*100)); // wait 1/10 of the delay between changes
328 }
329 }
330
331 if (completed)
332 {
333 pluginStatus.lastProcessedConfiguration = currentDbConfiguration;
334 pluginStatus.currentlyProcessingConfiguration.Clear();
335
336 pluginStatus.lastProcessedChange = -1;
337 pluginStatus.lastChangeToProcess = -1;
338
339 SaveStatusInDb(pluginStatus);
340
341 OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "DbOptimizer: finished processing all changes");
342 }
343 }
344
345 extern "C"
346 {
347 OrthancPluginErrorCode OnChangeCallback(OrthancPluginChangeType changeType,
348 OrthancPluginResourceType resourceType,
349 const char* resourceId)
350 {
351 switch (changeType)
352 {
353 case OrthancPluginChangeType_OrthancStarted:
354 {
355 OrthancPluginLogWarning(OrthancPlugins::GetGlobalContext(), "Starting DB Optmizer worker thread");
356 workerThread_.reset(new boost::thread(WorkerThread));
357 return OrthancPluginErrorCode_Success;
358 }
359 case OrthancPluginChangeType_OrthancStopped:
360 {
361 if (workerThread_ && workerThread_->joinable())
362 {
363 workerThreadShouldStop = true;
364 workerThread_->join();
365 }
366 }
367 default:
368 return OrthancPluginErrorCode_Success;
369 }
370 }
371
372 ORTHANC_PLUGINS_API int32_t OrthancPluginInitialize(OrthancPluginContext* c)
373 {
374 OrthancPlugins::SetGlobalContext(c);
375
376 /* Check the version of the Orthanc core */
377 if (OrthancPluginCheckVersion(c) == 0)
378 {
379 OrthancPlugins::ReportMinimalOrthancVersion(ORTHANC_PLUGINS_MINIMAL_MAJOR_NUMBER,
380 ORTHANC_PLUGINS_MINIMAL_MINOR_NUMBER,
381 ORTHANC_PLUGINS_MINIMAL_REVISION_NUMBER);
382 return -1;
383 }
384
385 OrthancPlugins::LogWarning("DB Optimizer plugin is initializing");
386 OrthancPluginSetDescription(c, "Optimizes your DB and storage.");
387
388 OrthancPlugins::OrthancConfiguration configuration;
389
390 OrthancPlugins::OrthancConfiguration dbOptimizer;
391 configuration.GetSection(dbOptimizer, "DbOptimizer");
392
393 bool enabled = dbOptimizer.GetBooleanValue("Enable", false);
394 if (enabled)
395 {
396 globalPropertyId_ = dbOptimizer.GetIntegerValue("GlobalPropertyId", 1025);
397 force_ = dbOptimizer.GetBooleanValue("Force", false);
398 throttleDelay_ = dbOptimizer.GetUnsignedIntegerValue("ThrottleDelay", 0);
399 OrthancPluginRegisterOnChangeCallback(c, OnChangeCallback);
400 }
401 else
402 {
403 OrthancPlugins::LogWarning("DB Optimizer plugin is disabled by the configuration file");
404 }
405
406 return 0;
407 }
408
409
410 ORTHANC_PLUGINS_API void OrthancPluginFinalize()
411 {
412 OrthancPlugins::LogWarning("DB Optimizer plugin is finalizing");
413 }
414
415
416 ORTHANC_PLUGINS_API const char* OrthancPluginGetName()
417 {
418 return "db-optimizer";
419 }
420
421
422 ORTHANC_PLUGINS_API const char* OrthancPluginGetVersion()
423 {
424 return DB_OPTIMIZER_VERSION;
425 }
426 }