Mercurial > hg > orthanc

comparison OrthancServer/Sources/ServerIndex.cpp @ 4577:a114a5db2afe db-changes

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
end of refactoring read-write transactions
author Sebastien Jodogne <s.jodogne@gmail.com>
date Tue, 09 Mar 2021 11:52:07 +0100
parents f6cd49af7526
children 710748828b6a
comparison
equal deleted inserted replaced
4576:f6cd49af7526 4577:a114a5db2afe
110 std::string remainingPublicId_; 110 std::string remainingPublicId_;
111 std::list<FileToRemove> pendingFilesToRemove_; 111 std::list<FileToRemove> pendingFilesToRemove_;
112 std::list<ServerIndexChange> pendingChanges_; 112 std::list<ServerIndexChange> pendingChanges_;
113 uint64_t sizeOfFilesToRemove_; 113 uint64_t sizeOfFilesToRemove_;
114 bool insideTransaction_; 114 bool insideTransaction_;
115 uint64_t sizeOfAddedAttachments_;
115 116
116 void Reset() 117 void Reset()
117 { 118 {
118 sizeOfFilesToRemove_ = 0; 119 sizeOfFilesToRemove_ = 0;
119 hasRemainingLevel_ = false; 120 hasRemainingLevel_ = false;
120 pendingFilesToRemove_.clear(); 121 pendingFilesToRemove_.clear();
121 pendingChanges_.clear(); 122 pendingChanges_.clear();
123 sizeOfAddedAttachments_ = 0;
122 } 124 }
123 125
124 public: 126 public:
125 explicit Listener(ServerContext& context) : 127 explicit Listener(ServerContext& context) :
126 context_(context), 128 context_(context),
127 insideTransaction_(false) 129 insideTransaction_(false),
130 sizeOfAddedAttachments_(0)
128 { 131 {
129 Reset(); 132 Reset();
130 assert(ResourceType_Patient < ResourceType_Study && 133 assert(ResourceType_Patient < ResourceType_Study &&
131 ResourceType_Study < ResourceType_Series && 134 ResourceType_Study < ResourceType_Series &&
132 ResourceType_Series < ResourceType_Instance); 135 ResourceType_Series < ResourceType_Instance);
224 { 227 {
225 context_.SignalChange(change); 228 context_.SignalChange(change);
226 } 229 }
227 } 230 }
228 231
232 void SignalAttachmentsAdded(uint64_t compressedSize)
233 {
234 sizeOfAddedAttachments_ += compressedSize;
235 }
236
229 bool HasRemainingLevel() const 237 bool HasRemainingLevel() const
230 { 238 {
231 return hasRemainingLevel_; 239 return hasRemainingLevel_;
232 } 240 }
233 241
239 247
240 const std::string& GetRemainingPublicId() const 248 const std::string& GetRemainingPublicId() const
241 { 249 {
242 assert(HasRemainingLevel()); 250 assert(HasRemainingLevel());
243 return remainingPublicId_; 251 return remainingPublicId_;
244 } 252 }
253
254 uint64_t GetSizeOfAddedAttachments() const
255 {
256 return sizeOfAddedAttachments_;
257 }
245 }; 258 };
246 259
247 260
248 class ServerIndex::Transaction 261 class ServerIndex::Transaction
249 { 262 {
250 private: 263 private:
251 ServerIndex& index_; 264 ServerIndex& index_;
252 std::unique_ptr<IDatabaseWrapper::ITransaction> transaction_; 265 std::unique_ptr<IDatabaseWrapper::ITransaction> transaction_;
253 bool isCommitted_; 266 bool isCommitted_;
254 uint64_t sizeOfAddedAttachments_;
255 267
256 public: 268 public:
257 explicit Transaction(ServerIndex& index, 269 explicit Transaction(ServerIndex& index,
258 TransactionType type) : 270 TransactionType type) :
259 index_(index), 271 index_(index),
260 isCommitted_(false), 272 isCommitted_(false)
261 sizeOfAddedAttachments_(0)
262 { 273 {
263 transaction_.reset(index_.db_.StartTransaction(type)); 274 transaction_.reset(index_.db_.StartTransaction(type));
264 index_.listener_->StartTransaction(); 275 index_.listener_->StartTransaction();
265 } 276 }
266 277
272 { 283 {
273 transaction_->Rollback(); 284 transaction_->Rollback();
274 } 285 }
275 } 286 }
276 287
277 void SignalAttachmentsAdded(uint64_t compressedSize)
278 {
279 sizeOfAddedAttachments_ += compressedSize;
280 }
281
282 void Commit() 288 void Commit()
283 { 289 {
284 if (!isCommitted_) 290 if (!isCommitted_)
285 { 291 {
286 int64_t delta = (static_cast<int64_t>(sizeOfAddedAttachments_) - 292 int64_t delta = (static_cast<int64_t>(index_.listener_->GetSizeOfAddedAttachments()) -
287 static_cast<int64_t>(index_.listener_->GetSizeOfFilesToRemove())); 293 static_cast<int64_t>(index_.listener_->GetSizeOfFilesToRemove()));
288 294
289 transaction_->Commit(delta); 295 transaction_->Commit(delta);
290 296
291 // We can remove the files once the SQLite transaction has 297 // We can remove the files once the SQLite transaction has
450 unsigned int threadSleep) 456 unsigned int threadSleep)
451 { 457 {
452 // By default, wait for 10 seconds before flushing 458 // By default, wait for 10 seconds before flushing
453 unsigned int sleep = 10; 459 unsigned int sleep = 10;
454 460
461 // TODO - REMOVE THIS
455 try 462 try
456 { 463 {
457 boost::mutex::scoped_lock lock(that->mutex_); 464 boost::mutex::scoped_lock lock(that->monitoringMutex_);
458 std::string sleepString; 465 std::string sleepString;
459 466
460 if (that->db_.LookupGlobalProperty(sleepString, GlobalProperty_FlushSleep) && 467 if (that->db_.LookupGlobalProperty(sleepString, GlobalProperty_FlushSleep) &&
461 Toolbox::IsInteger(sleepString)) 468 Toolbox::IsInteger(sleepString))
462 { 469 {
480 continue; 487 continue;
481 } 488 }
482 489
483 Logging::Flush(); 490 Logging::Flush();
484 491
485 boost::mutex::scoped_lock lock(that->mutex_); 492 boost::mutex::scoped_lock lock(that->monitoringMutex_);
486 493
487 try 494 try
488 { 495 {
489 that->db_.FlushToDisk(); 496 that->db_.FlushToDisk();
490 } 497 }
633 listener_.reset(new Listener(context)); 640 listener_.reset(new Listener(context));
634 db_.SetListener(*listener_); 641 db_.SetListener(*listener_);
635 642
636 // Initial recycling if the parameters have changed since the last 643 // Initial recycling if the parameters have changed since the last
637 // execution of Orthanc 644 // execution of Orthanc
638 StandaloneRecycling(); 645 StandaloneRecycling(maximumStorageSize_, maximumPatients_);
639 646
640 if (db.HasFlushToDisk()) 647 if (db.HasFlushToDisk())
641 { 648 {
642 flushThread_ = boost::thread(FlushThread, this, threadSleep); 649 flushThread_ = boost::thread(FlushThread, this, threadSleep);
643 } 650 }
677 } 684 }
678 } 685 }
679 } 686 }
680 687
681 688
682 static void SetInstanceMetadata(ResourcesContent& content, 689
683 std::map<MetadataType, std::string>& instanceMetadata, 690 SeriesStatus ServerIndex::ReadOnlyTransaction::GetSeriesStatus(int64_t id,
684 int64_t instance, 691 int64_t expectedNumberOfInstances)
685 MetadataType metadata,
686 const std::string& value)
687 {
688 content.AddMetadata(instance, metadata, value);
689 instanceMetadata[metadata] = value;
690 }
691
692
693 StoreStatus ServerIndex::Store(std::map<MetadataType, std::string>& instanceMetadata,
694 const DicomMap& dicomSummary,
695 const Attachments& attachments,
696 const MetadataMap& metadata,
697 const DicomInstanceOrigin& origin,
698 bool overwrite,
699 bool hasTransferSyntax,
700 DicomTransferSyntax transferSyntax,
701 bool hasPixelDataOffset,
702 uint64_t pixelDataOffset)
703 {
704 boost::mutex::scoped_lock lock(mutex_);
705
706 int64_t expectedInstances;
707 const bool hasExpectedInstances =
708 ComputeExpectedNumberOfInstances(expectedInstances, dicomSummary);
709
710 instanceMetadata.clear();
711
712 DicomInstanceHasher hasher(dicomSummary);
713 const std::string hashPatient = hasher.HashPatient();
714 const std::string hashStudy = hasher.HashStudy();
715 const std::string hashSeries = hasher.HashSeries();
716 const std::string hashInstance = hasher.HashInstance();
717
718 try
719 {
720 Transaction t(*this, TransactionType_ReadWrite);
721
722 IDatabaseWrapper::CreateInstanceResult status;
723 int64_t instanceId;
724
725 // Check whether this instance is already stored
726 if (!db_.CreateInstance(status, instanceId, hashPatient,
727 hashStudy, hashSeries, hashInstance))
728 {
729 // The instance already exists
730
731 if (overwrite)
732 {
733 // Overwrite the old instance
734 LOG(INFO) << "Overwriting instance: " << hashInstance;
735 db_.DeleteResource(instanceId);
736
737 // Re-create the instance, now that the old one is removed
738 if (!db_.CreateInstance(status, instanceId, hashPatient,
739 hashStudy, hashSeries, hashInstance))
740 {
741 throw OrthancException(ErrorCode_InternalError);
742 }
743 }
744 else
745 {
746 // Do nothing if the instance already exists and overwriting is disabled
747 db_.GetAllMetadata(instanceMetadata, instanceId);
748 return StoreStatus_AlreadyStored;
749 }
750 }
751
752
753 // Warn about the creation of new resources. The order must be
754 // from instance to patient.
755
756 // NB: In theory, could be sped up by grouping the underlying
757 // calls to "db_.LogChange()". However, this would only have an
758 // impact when new patient/study/series get created, which
759 // occurs far less often that creating new instances. The
760 // positive impact looks marginal in practice.
761 LogChange(instanceId, ChangeType_NewInstance, ResourceType_Instance, hashInstance);
762
763 if (status.isNewSeries_)
764 {
765 LogChange(status.seriesId_, ChangeType_NewSeries, ResourceType_Series, hashSeries);
766 }
767
768 if (status.isNewStudy_)
769 {
770 LogChange(status.studyId_, ChangeType_NewStudy, ResourceType_Study, hashStudy);
771 }
772
773 if (status.isNewPatient_)
774 {
775 LogChange(status.patientId_, ChangeType_NewPatient, ResourceType_Patient, hashPatient);
776 }
777
778
779 // Ensure there is enough room in the storage for the new instance
780 uint64_t instanceSize = 0;
781 for (Attachments::const_iterator it = attachments.begin();
782 it != attachments.end(); ++it)
783 {
784 instanceSize += it->GetCompressedSize();
785 }
786
787 Recycle(instanceSize, hashPatient /* don't consider the current patient for recycling */);
788
789
790 // Attach the files to the newly created instance
791 for (Attachments::const_iterator it = attachments.begin();
792 it != attachments.end(); ++it)
793 {
794 db_.AddAttachment(instanceId, *it);
795 }
796
797
798 {
799 ResourcesContent content;
800
801 // Populate the tags of the newly-created resources
802
803 content.AddResource(instanceId, ResourceType_Instance, dicomSummary);
804
805 if (status.isNewSeries_)
806 {
807 content.AddResource(status.seriesId_, ResourceType_Series, dicomSummary);
808 }
809
810 if (status.isNewStudy_)
811 {
812 content.AddResource(status.studyId_, ResourceType_Study, dicomSummary);
813 }
814
815 if (status.isNewPatient_)
816 {
817 content.AddResource(status.patientId_, ResourceType_Patient, dicomSummary);
818 }
819
820
821 // Attach the user-specified metadata
822
823 for (MetadataMap::const_iterator
824 it = metadata.begin(); it != metadata.end(); ++it)
825 {
826 switch (it->first.first)
827 {
828 case ResourceType_Patient:
829 content.AddMetadata(status.patientId_, it->first.second, it->second);
830 break;
831
832 case ResourceType_Study:
833 content.AddMetadata(status.studyId_, it->first.second, it->second);
834 break;
835
836 case ResourceType_Series:
837 content.AddMetadata(status.seriesId_, it->first.second, it->second);
838 break;
839
840 case ResourceType_Instance:
841 SetInstanceMetadata(content, instanceMetadata, instanceId,
842 it->first.second, it->second);
843 break;
844
845 default:
846 throw OrthancException(ErrorCode_ParameterOutOfRange);
847 }
848 }
849
850
851 // Attach the auto-computed metadata for the patient/study/series levels
852 std::string now = SystemToolbox::GetNowIsoString(true /* use UTC time (not local time) */);
853 content.AddMetadata(status.seriesId_, MetadataType_LastUpdate, now);
854 content.AddMetadata(status.studyId_, MetadataType_LastUpdate, now);
855 content.AddMetadata(status.patientId_, MetadataType_LastUpdate, now);
856
857 if (status.isNewSeries_)
858 {
859 if (hasExpectedInstances)
860 {
861 content.AddMetadata(status.seriesId_, MetadataType_Series_ExpectedNumberOfInstances,
862 boost::lexical_cast<std::string>(expectedInstances));
863 }
864
865 // New in Orthanc 1.9.0
866 content.AddMetadata(status.seriesId_, MetadataType_RemoteAet,
867 origin.GetRemoteAetC());
868 }
869
870
871 // Attach the auto-computed metadata for the instance level,
872 // reflecting these additions into the input metadata map
873 SetInstanceMetadata(content, instanceMetadata, instanceId,
874 MetadataType_Instance_ReceptionDate, now);
875 SetInstanceMetadata(content, instanceMetadata, instanceId, MetadataType_RemoteAet,
876 origin.GetRemoteAetC());
877 SetInstanceMetadata(content, instanceMetadata, instanceId, MetadataType_Instance_Origin,
878 EnumerationToString(origin.GetRequestOrigin()));
879
880
881 if (hasTransferSyntax)
882 {
883 // New in Orthanc 1.2.0
884 SetInstanceMetadata(content, instanceMetadata, instanceId,
885 MetadataType_Instance_TransferSyntax,
886 GetTransferSyntaxUid(transferSyntax));
887 }
888
889 {
890 std::string s;
891
892 if (origin.LookupRemoteIp(s))
893 {
894 // New in Orthanc 1.4.0
895 SetInstanceMetadata(content, instanceMetadata, instanceId,
896 MetadataType_Instance_RemoteIp, s);
897 }
898
899 if (origin.LookupCalledAet(s))
900 {
901 // New in Orthanc 1.4.0
902 SetInstanceMetadata(content, instanceMetadata, instanceId,
903 MetadataType_Instance_CalledAet, s);
904 }
905
906 if (origin.LookupHttpUsername(s))
907 {
908 // New in Orthanc 1.4.0
909 SetInstanceMetadata(content, instanceMetadata, instanceId,
910 MetadataType_Instance_HttpUsername, s);
911 }
912 }
913
914 if (hasPixelDataOffset)
915 {
916 // New in Orthanc 1.9.1
917 SetInstanceMetadata(content, instanceMetadata, instanceId,
918 MetadataType_Instance_PixelDataOffset,
919 boost::lexical_cast<std::string>(pixelDataOffset));
920 }
921
922 const DicomValue* value;
923 if ((value = dicomSummary.TestAndGetValue(DICOM_TAG_SOP_CLASS_UID)) != NULL &&
924 !value->IsNull() &&
925 !value->IsBinary())
926 {
927 SetInstanceMetadata(content, instanceMetadata, instanceId,
928 MetadataType_Instance_SopClassUid, value->GetContent());
929 }
930
931
932 if ((value = dicomSummary.TestAndGetValue(DICOM_TAG_INSTANCE_NUMBER)) != NULL ||
933 (value = dicomSummary.TestAndGetValue(DICOM_TAG_IMAGE_INDEX)) != NULL)
934 {
935 if (!value->IsNull() &&
936 !value->IsBinary())
937 {
938 SetInstanceMetadata(content, instanceMetadata, instanceId,
939 MetadataType_Instance_IndexInSeries, Toolbox::StripSpaces(value->GetContent()));
940 }
941 }
942
943
944 db_.SetResourcesContent(content);
945 }
946
947
948 // Check whether the series of this new instance is now completed
949 int64_t expectedNumberOfInstances;
950 if (ComputeExpectedNumberOfInstances(expectedNumberOfInstances, dicomSummary))
951 {
952 SeriesStatus seriesStatus = GetSeriesStatus(db_, status.seriesId_, expectedNumberOfInstances);
953 if (seriesStatus == SeriesStatus_Complete)
954 {
955 LogChange(status.seriesId_, ChangeType_CompletedSeries, ResourceType_Series, hashSeries);
956 }
957 }
958
959
960 // Mark the parent resources of this instance as unstable
961 MarkAsUnstable(status.seriesId_, ResourceType_Series, hashSeries);
962 MarkAsUnstable(status.studyId_, ResourceType_Study, hashStudy);
963 MarkAsUnstable(status.patientId_, ResourceType_Patient, hashPatient);
964
965 t.SignalAttachmentsAdded(instanceSize);
966 t.Commit();
967
968 return StoreStatus_Success;
969 }
970 catch (OrthancException& e)
971 {
972 LOG(ERROR) << "EXCEPTION [" << e.What() << "]";
973 }
974
975 return StoreStatus_Failure;
976 }
977
978
979 SeriesStatus ServerIndex::GetSeriesStatus(IDatabaseWrapper& db,
980 int64_t id,
981 int64_t expectedNumberOfInstances)
982 { 692 {
983 std::list<std::string> values; 693 std::list<std::string> values;
984 db.GetChildrenMetadata(values, id, MetadataType_Instance_IndexInSeries); 694 db_.GetChildrenMetadata(values, id, MetadataType_Instance_IndexInSeries);
985 695
986 std::set<int64_t> instances; 696 std::set<int64_t> instances;
987 697
988 for (std::list<std::string>::const_iterator 698 for (std::list<std::string>::const_iterator
989 it = values.begin(); it != values.end(); ++it) 699 it = values.begin(); it != values.end(); ++it)
1090 800
1091 target["Last"] = static_cast<int>(last); 801 target["Last"] = static_cast<int>(last);
1092 } 802 }
1093 803
1094 804
1095 bool ServerIndex::IsRecyclingNeeded(uint64_t instanceSize) 805 void ServerIndex::SetMaximumPatientCount(unsigned int count)
1096 { 806 {
1097 if (maximumStorageSize_ != 0) 807 {
1098 { 808 boost::mutex::scoped_lock lock(monitoringMutex_);
1099 assert(maximumStorageSize_ >= instanceSize); 809 maximumPatients_ = count;
1100 810
1101 if (db_.IsDiskSizeAbove(maximumStorageSize_ - instanceSize)) 811 if (count == 0)
1102 { 812 {
1103 return true; 813 LOG(WARNING) << "No limit on the number of stored patients";
1104 }
1105 }
1106
1107 if (maximumPatients_ != 0)
1108 {
1109 uint64_t patientCount = db_.GetResourceCount(ResourceType_Patient);
1110 if (patientCount > maximumPatients_)
1111 {
1112 return true;
1113 }
1114 }
1115
1116 return false;
1117 }
1118
1119
1120 void ServerIndex::Recycle(uint64_t instanceSize,
1121 const std::string& newPatientId)
1122 {
1123 if (IsRecyclingNeeded(instanceSize))
1124 {
1125 // Check whether other DICOM instances from this patient are
1126 // already stored
1127 int64_t patientToAvoid;
1128 bool hasPatientToAvoid;
1129
1130 if (newPatientId.empty())
1131 {
1132 hasPatientToAvoid = false;
1133 } 814 }
1134 else 815 else
1135 { 816 {
1136 ResourceType type; 817 LOG(WARNING) << "At most " << count << " patients will be stored";
1137 hasPatientToAvoid = db_.LookupResource(patientToAvoid, type, newPatientId); 818 }
1138 if (type != ResourceType_Patient) 819 }
1139 { 820
1140 throw OrthancException(ErrorCode_InternalError); 821 StandaloneRecycling(maximumStorageSize_, maximumPatients_);
1141 } 822 }
1142 } 823
1143 824 void ServerIndex::SetMaximumStorageSize(uint64_t size)
1144 // Iteratively select patient to remove until there is enough 825 {
1145 // space in the DICOM store 826 {
1146 int64_t patientToRecycle; 827 boost::mutex::scoped_lock lock(monitoringMutex_);
1147 while (true) 828 maximumStorageSize_ = size;
1148 {
1149 // If other instances of this patient are already in the store,
1150 // we must avoid to recycle them
1151 bool ok = (hasPatientToAvoid ?
1152 db_.SelectPatientToRecycle(patientToRecycle, patientToAvoid) :
1153 db_.SelectPatientToRecycle(patientToRecycle));
1154
1155 if (!ok)
1156 {
1157 throw OrthancException(ErrorCode_FullStorage);
1158 }
1159 829
1160 LOG(TRACE) << "Recycling one patient"; 830 if (size == 0)
1161 db_.DeleteResource(patientToRecycle); 831 {
1162 832 LOG(WARNING) << "No limit on the size of the storage area";
1163 if (!IsRecyclingNeeded(instanceSize)) 833 }
1164 { 834 else
1165 // OK, we're done 835 {
1166 break; 836 LOG(WARNING) << "At most " << (size / MEGA_BYTES) << "MB will be used for the storage area";
1167 } 837 }
1168 } 838 }
1169 } 839
1170 } 840 StandaloneRecycling(maximumStorageSize_, maximumPatients_);
1171
1172
1173 void ServerIndex::SetMaximumPatientCount(unsigned int count)
1174 {
1175 boost::mutex::scoped_lock lock(mutex_);
1176 maximumPatients_ = count;
1177
1178 if (count == 0)
1179 {
1180 LOG(WARNING) << "No limit on the number of stored patients";
1181 }
1182 else
1183 {
1184 LOG(WARNING) << "At most " << count << " patients will be stored";
1185 }
1186
1187 StandaloneRecycling();
1188 }
1189
1190 void ServerIndex::SetMaximumStorageSize(uint64_t size)
1191 {
1192 boost::mutex::scoped_lock lock(mutex_);
1193 maximumStorageSize_ = size;
1194
1195 if (size == 0)
1196 {
1197 LOG(WARNING) << "No limit on the size of the storage area";
1198 }
1199 else
1200 {
1201 LOG(WARNING) << "At most " << (size / MEGA_BYTES) << "MB will be used for the storage area";
1202 }
1203
1204 StandaloneRecycling();
1205 }
1206
1207
1208 void ServerIndex::StandaloneRecycling()
1209 {
1210 // WARNING: No mutex here, do not include this as a public method
1211 Transaction t(*this, TransactionType_ReadWrite);
1212 Recycle(0, "");
1213 t.Commit();
1214 } 841 }
1215 842
1216 843
1217 void ServerIndex::UnstableResourcesMonitorThread(ServerIndex* that, 844 void ServerIndex::UnstableResourcesMonitorThread(ServerIndex* that,
1218 unsigned int threadSleep) 845 unsigned int threadSleep)
1234 while (!that->done_) 861 while (!that->done_)
1235 { 862 {
1236 // Check for stable resources each few seconds 863 // Check for stable resources each few seconds
1237 boost::this_thread::sleep(boost::posix_time::milliseconds(threadSleep)); 864 boost::this_thread::sleep(boost::posix_time::milliseconds(threadSleep));
1238 865
1239 boost::mutex::scoped_lock lock(that->mutex_); 866 boost::mutex::scoped_lock lock(that->monitoringMutex_);
1240 867
1241 while (!that->unstableResources_.IsEmpty() && 868 while (!that->unstableResources_.IsEmpty() &&
1242 that->unstableResources_.GetOldestPayload().GetAge() > static_cast<unsigned int>(stableAge)) 869 that->unstableResources_.GetOldestPayload().GetAge() > static_cast<unsigned int>(stableAge))
1243 { 870 {
1244 // This DICOM resource has not received any new instance for 871 // This DICOM resource has not received any new instance for
1279 906
1280 void ServerIndex::MarkAsUnstable(int64_t id, 907 void ServerIndex::MarkAsUnstable(int64_t id,
1281 Orthanc::ResourceType type, 908 Orthanc::ResourceType type,
1282 const std::string& publicId) 909 const std::string& publicId)
1283 { 910 {
1284 // WARNING: Before calling this method, "mutex_" must be locked. 911 // WARNING: Before calling this method, "monitoringMutex_" must be locked.
1285 912
1286 assert(type == Orthanc::ResourceType_Patient || 913 assert(type == Orthanc::ResourceType_Patient ||
1287 type == Orthanc::ResourceType_Study || 914 type == Orthanc::ResourceType_Study ||
1288 type == Orthanc::ResourceType_Series); 915 type == Orthanc::ResourceType_Series);
1289 916
1292 //LOG(INFO) << "Unstable resource: " << EnumerationToString(type) << " " << id; 919 //LOG(INFO) << "Unstable resource: " << EnumerationToString(type) << " " << id;
1293 920
1294 LogChange(id, ChangeType_NewChildInstance, type, publicId); 921 LogChange(id, ChangeType_NewChildInstance, type, publicId);
1295 } 922 }
1296 923
1297
1298
1299 StoreStatus ServerIndex::AddAttachment(const FileInfo& attachment,
1300 const std::string& publicId)
1301 {
1302 boost::mutex::scoped_lock lock(mutex_);
1303
1304 Transaction t(*this, TransactionType_ReadWrite);
1305
1306 ResourceType resourceType;
1307 int64_t resourceId;
1308 if (!db_.LookupResource(resourceId, resourceType, publicId))
1309 {
1310 return StoreStatus_Failure; // Inexistent resource
1311 }
1312
1313 // Remove possible previous attachment
1314 db_.DeleteAttachment(resourceId, attachment.GetContentType());
1315
1316 // Locate the patient of the target resource
1317 int64_t patientId = resourceId;
1318 for (;;)
1319 {
1320 int64_t parent;
1321 if (db_.LookupParent(parent, patientId))
1322 {
1323 // We have not reached the patient level yet
1324 patientId = parent;
1325 }
1326 else
1327 {
1328 // We have reached the patient level
1329 break;
1330 }
1331 }
1332
1333 // Possibly apply the recycling mechanism while preserving this patient
1334 assert(db_.GetResourceType(patientId) == ResourceType_Patient);
1335 Recycle(attachment.GetCompressedSize(), db_.GetPublicId(patientId));
1336
1337 db_.AddAttachment(resourceId, attachment);
1338
1339 if (IsUserContentType(attachment.GetContentType()))
1340 {
1341 LogChange(resourceId, ChangeType_UpdatedAttachment, resourceType, publicId);
1342 }
1343
1344 t.SignalAttachmentsAdded(attachment.GetCompressedSize());
1345 t.Commit();
1346
1347 return StoreStatus_Success;
1348 }
1349 924
1350 925
1351 void ServerIndex::NormalizeLookup(std::vector<DatabaseConstraint>& target, 926 void ServerIndex::NormalizeLookup(std::vector<DatabaseConstraint>& target,
1352 const DatabaseLookup& source, 927 const DatabaseLookup& source,
1353 ResourceType queryLevel) const 928 ResourceType queryLevel) const
1599 1174
1600 for (;;) 1175 for (;;)
1601 { 1176 {
1602 try 1177 try
1603 { 1178 {
1604 boost::mutex::scoped_lock lock(mutex_); // TODO - REMOVE 1179 boost::mutex::scoped_lock lock(monitoringMutex_); // TODO - REMOVE
1605 1180
1606 if (readOperations != NULL) 1181 if (readOperations != NULL)
1607 { 1182 {
1608 /** 1183 /**
1609 * IMPORTANT: In Orthanc <= 1.9.1, there was no transaction 1184 * IMPORTANT: In Orthanc <= 1.9.1, there was no transaction
3459 }; 3034 };
3460 3035
3461 Operations operations(dicom); 3036 Operations operations(dicom);
3462 Apply(operations); 3037 Apply(operations);
3463 } 3038 }
3039
3040
3041 static bool IsRecyclingNeeded(IDatabaseWrapper& db,
3042 uint64_t maximumStorageSize,
3043 unsigned int maximumPatients,
3044 uint64_t addedInstanceSize)
3045 {
3046 if (maximumStorageSize != 0)
3047 {
3048 if (maximumStorageSize < addedInstanceSize)
3049 {
3050 throw OrthancException(ErrorCode_FullStorage, "Cannot store an instance of size " +
3051 boost::lexical_cast<std::string>(addedInstanceSize) +
3052 " bytes in a storage area limited to " +
3053 boost::lexical_cast<std::string>(maximumStorageSize));
3054 }
3055
3056 if (db.IsDiskSizeAbove(maximumStorageSize - addedInstanceSize))
3057 {
3058 return true;
3059 }
3060 }
3061
3062 if (maximumPatients != 0)
3063 {
3064 uint64_t patientCount = db.GetResourceCount(ResourceType_Patient);
3065 if (patientCount > maximumPatients)
3066 {
3067 return true;
3068 }
3069 }
3070
3071 return false;
3072 }
3073
3074
3075 void ServerIndex::ReadWriteTransaction::Recycle(uint64_t maximumStorageSize,
3076 unsigned int maximumPatients,
3077 uint64_t addedInstanceSize,
3078 const std::string& newPatientId)
3079 {
3080 // TODO - Performance: Avoid calls to "IsRecyclingNeeded()"
3081
3082 if (IsRecyclingNeeded(db_, maximumStorageSize, maximumPatients, addedInstanceSize))
3083 {
3084 // Check whether other DICOM instances from this patient are
3085 // already stored
3086 int64_t patientToAvoid;
3087 bool hasPatientToAvoid;
3088
3089 if (newPatientId.empty())
3090 {
3091 hasPatientToAvoid = false;
3092 }
3093 else
3094 {
3095 ResourceType type;
3096 hasPatientToAvoid = db_.LookupResource(patientToAvoid, type, newPatientId);
3097 if (type != ResourceType_Patient)
3098 {
3099 throw OrthancException(ErrorCode_InternalError);
3100 }
3101 }
3102
3103 // Iteratively select patient to remove until there is enough
3104 // space in the DICOM store
3105 int64_t patientToRecycle;
3106 while (true)
3107 {
3108 // If other instances of this patient are already in the store,
3109 // we must avoid to recycle them
3110 bool ok = (hasPatientToAvoid ?
3111 db_.SelectPatientToRecycle(patientToRecycle, patientToAvoid) :
3112 db_.SelectPatientToRecycle(patientToRecycle));
3113
3114 if (!ok)
3115 {
3116 throw OrthancException(ErrorCode_FullStorage);
3117 }
3118
3119 LOG(TRACE) << "Recycling one patient";
3120 db_.DeleteResource(patientToRecycle);
3121
3122 if (!IsRecyclingNeeded(db_, maximumStorageSize, maximumPatients, addedInstanceSize))
3123 {
3124 // OK, we're done
3125 return;
3126 }
3127 }
3128 }
3129 }
3130
3131
3132 void ServerIndex::StandaloneRecycling(uint64_t maximumStorageSize,
3133 unsigned int maximumPatientCount)
3134 {
3135 class Operations : public IReadWriteOperations
3136 {
3137 private:
3138 uint64_t maximumStorageSize_;
3139 unsigned int maximumPatientCount_;
3140
3141 public:
3142 Operations(uint64_t maximumStorageSize,
3143 unsigned int maximumPatientCount) :
3144 maximumStorageSize_(maximumStorageSize),
3145 maximumPatientCount_(maximumPatientCount)
3146 {
3147 }
3148
3149 virtual void Apply(ReadWriteTransaction& transaction) ORTHANC_OVERRIDE
3150 {
3151 transaction.Recycle(maximumStorageSize_, maximumPatientCount_, 0, "");
3152 }
3153 };
3154
3155 Operations operations(maximumStorageSize, maximumPatientCount);
3156 Apply(operations);
3157 }
3158
3159
3160 static void SetInstanceMetadata(ResourcesContent& content,
3161 std::map<MetadataType, std::string>& instanceMetadata,
3162 int64_t instance,
3163 MetadataType metadata,
3164 const std::string& value)
3165 {
3166 content.AddMetadata(instance, metadata, value);
3167 instanceMetadata[metadata] = value;
3168 }
3169
3170
3171 StoreStatus ServerIndex::Store(std::map<MetadataType, std::string>& instanceMetadata,
3172 const DicomMap& dicomSummary,
3173 const Attachments& attachments,
3174 const MetadataMap& metadata,
3175 const DicomInstanceOrigin& origin,
3176 bool overwrite,
3177 bool hasTransferSyntax,
3178 DicomTransferSyntax transferSyntax,
3179 bool hasPixelDataOffset,
3180 uint64_t pixelDataOffset)
3181 {
3182 class Operations : public IReadWriteOperations
3183 {
3184 private:
3185 StoreStatus storeStatus_;
3186 std::map<MetadataType, std::string>& instanceMetadata_;
3187 ServerIndex& index_;
3188 const DicomMap& dicomSummary_;
3189 const Attachments& attachments_;
3190 const MetadataMap& metadata_;
3191 const DicomInstanceOrigin& origin_;
3192 bool overwrite_;
3193 bool hasTransferSyntax_;
3194 DicomTransferSyntax transferSyntax_;
3195 bool hasPixelDataOffset_;
3196 uint64_t pixelDataOffset_;
3197 uint64_t maximumStorageSize_;
3198 unsigned int maximumPatientCount_;
3199
3200 // Auto-computed fields
3201 bool hasExpectedInstances_;
3202 int64_t expectedInstances_;
3203 std::string hashPatient_;
3204 std::string hashStudy_;
3205 std::string hashSeries_;
3206 std::string hashInstance_;
3207
3208 public:
3209 Operations(std::map<MetadataType, std::string>& instanceMetadata,
3210 ServerIndex& index,
3211 const DicomMap& dicomSummary,
3212 const Attachments& attachments,
3213 const MetadataMap& metadata,
3214 const DicomInstanceOrigin& origin,
3215 bool overwrite,
3216 bool hasTransferSyntax,
3217 DicomTransferSyntax transferSyntax,
3218 bool hasPixelDataOffset,
3219 uint64_t pixelDataOffset,
3220 uint64_t maximumStorageSize,
3221 unsigned int maximumPatientCount) :
3222 storeStatus_(StoreStatus_Failure),
3223 instanceMetadata_(instanceMetadata),
3224 index_(index),
3225 dicomSummary_(dicomSummary),
3226 attachments_(attachments),
3227 metadata_(metadata),
3228 origin_(origin),
3229 overwrite_(overwrite),
3230 hasTransferSyntax_(hasTransferSyntax),
3231 transferSyntax_(transferSyntax),
3232 hasPixelDataOffset_(hasPixelDataOffset),
3233 pixelDataOffset_(pixelDataOffset),
3234 maximumStorageSize_(maximumStorageSize),
3235 maximumPatientCount_(maximumPatientCount)
3236 {
3237 hasExpectedInstances_ = ComputeExpectedNumberOfInstances(expectedInstances_, dicomSummary);
3238
3239 instanceMetadata_.clear();
3240
3241 DicomInstanceHasher hasher(dicomSummary);
3242 hashPatient_ = hasher.HashPatient();
3243 hashStudy_ = hasher.HashStudy();
3244 hashSeries_ = hasher.HashSeries();
3245 hashInstance_ = hasher.HashInstance();
3246 }
3247
3248 StoreStatus GetStoreStatus() const
3249 {
3250 return storeStatus_;
3251 }
3252
3253 virtual void Apply(ReadWriteTransaction& transaction) ORTHANC_OVERRIDE
3254 {
3255 try
3256 {
3257 IDatabaseWrapper::CreateInstanceResult status;
3258 int64_t instanceId;
3259
3260 // Check whether this instance is already stored
3261 if (!transaction.CreateInstance(status, instanceId, hashPatient_,
3262 hashStudy_, hashSeries_, hashInstance_))
3263 {
3264 // The instance already exists
3265
3266 if (overwrite_)
3267 {
3268 // Overwrite the old instance
3269 LOG(INFO) << "Overwriting instance: " << hashInstance_;
3270 transaction.DeleteResource(instanceId);
3271
3272 // Re-create the instance, now that the old one is removed
3273 if (!transaction.CreateInstance(status, instanceId, hashPatient_,
3274 hashStudy_, hashSeries_, hashInstance_))
3275 {
3276 throw OrthancException(ErrorCode_InternalError);
3277 }
3278 }
3279 else
3280 {
3281 // Do nothing if the instance already exists and overwriting is disabled
3282 transaction.GetAllMetadata(instanceMetadata_, instanceId);
3283 storeStatus_ = StoreStatus_AlreadyStored;
3284 return;
3285 }
3286 }
3287
3288
3289 // Warn about the creation of new resources. The order must be
3290 // from instance to patient.
3291
3292 // NB: In theory, could be sped up by grouping the underlying
3293 // calls to "transaction.LogChange()". However, this would only have an
3294 // impact when new patient/study/series get created, which
3295 // occurs far less often that creating new instances. The
3296 // positive impact looks marginal in practice.
3297 transaction.LogChange(instanceId, ChangeType_NewInstance, ResourceType_Instance, hashInstance_);
3298
3299 if (status.isNewSeries_)
3300 {
3301 transaction.LogChange(status.seriesId_, ChangeType_NewSeries, ResourceType_Series, hashSeries_);
3302 }
3303
3304 if (status.isNewStudy_)
3305 {
3306 transaction.LogChange(status.studyId_, ChangeType_NewStudy, ResourceType_Study, hashStudy_);
3307 }
3308
3309 if (status.isNewPatient_)
3310 {
3311 transaction.LogChange(status.patientId_, ChangeType_NewPatient, ResourceType_Patient, hashPatient_);
3312 }
3313
3314
3315 // Ensure there is enough room in the storage for the new instance
3316 uint64_t instanceSize = 0;
3317 for (Attachments::const_iterator it = attachments_.begin();
3318 it != attachments_.end(); ++it)
3319 {
3320 instanceSize += it->GetCompressedSize();
3321 }
3322
3323 transaction.Recycle(maximumStorageSize_, maximumPatientCount_,
3324 instanceSize, hashPatient_ /* don't consider the current patient for recycling */);
3325
3326
3327 // Attach the files to the newly created instance
3328 for (Attachments::const_iterator it = attachments_.begin();
3329 it != attachments_.end(); ++it)
3330 {
3331 transaction.AddAttachment(instanceId, *it);
3332 }
3333
3334
3335 {
3336 ResourcesContent content;
3337
3338 // Populate the tags of the newly-created resources
3339
3340 content.AddResource(instanceId, ResourceType_Instance, dicomSummary_);
3341
3342 if (status.isNewSeries_)
3343 {
3344 content.AddResource(status.seriesId_, ResourceType_Series, dicomSummary_);
3345 }
3346
3347 if (status.isNewStudy_)
3348 {
3349 content.AddResource(status.studyId_, ResourceType_Study, dicomSummary_);
3350 }
3351
3352 if (status.isNewPatient_)
3353 {
3354 content.AddResource(status.patientId_, ResourceType_Patient, dicomSummary_);
3355 }
3356
3357
3358 // Attach the user-specified metadata
3359
3360 for (MetadataMap::const_iterator
3361 it = metadata_.begin(); it != metadata_.end(); ++it)
3362 {
3363 switch (it->first.first)
3364 {
3365 case ResourceType_Patient:
3366 content.AddMetadata(status.patientId_, it->first.second, it->second);
3367 break;
3368
3369 case ResourceType_Study:
3370 content.AddMetadata(status.studyId_, it->first.second, it->second);
3371 break;
3372
3373 case ResourceType_Series:
3374 content.AddMetadata(status.seriesId_, it->first.second, it->second);
3375 break;
3376
3377 case ResourceType_Instance:
3378 SetInstanceMetadata(content, instanceMetadata_, instanceId,
3379 it->first.second, it->second);
3380 break;
3381
3382 default:
3383 throw OrthancException(ErrorCode_ParameterOutOfRange);
3384 }
3385 }
3386
3387
3388 // Attach the auto-computed metadata for the patient/study/series levels
3389 std::string now = SystemToolbox::GetNowIsoString(true /* use UTC time (not local time) */);
3390 content.AddMetadata(status.seriesId_, MetadataType_LastUpdate, now);
3391 content.AddMetadata(status.studyId_, MetadataType_LastUpdate, now);
3392 content.AddMetadata(status.patientId_, MetadataType_LastUpdate, now);
3393
3394 if (status.isNewSeries_)
3395 {
3396 if (hasExpectedInstances_)
3397 {
3398 content.AddMetadata(status.seriesId_, MetadataType_Series_ExpectedNumberOfInstances,
3399 boost::lexical_cast<std::string>(expectedInstances_));
3400 }
3401
3402 // New in Orthanc 1.9.0
3403 content.AddMetadata(status.seriesId_, MetadataType_RemoteAet,
3404 origin_.GetRemoteAetC());
3405 }
3406
3407
3408 // Attach the auto-computed metadata for the instance level,
3409 // reflecting these additions into the input metadata map
3410 SetInstanceMetadata(content, instanceMetadata_, instanceId,
3411 MetadataType_Instance_ReceptionDate, now);
3412 SetInstanceMetadata(content, instanceMetadata_, instanceId, MetadataType_RemoteAet,
3413 origin_.GetRemoteAetC());
3414 SetInstanceMetadata(content, instanceMetadata_, instanceId, MetadataType_Instance_Origin,
3415 EnumerationToString(origin_.GetRequestOrigin()));
3416
3417
3418 if (hasTransferSyntax_)
3419 {
3420 // New in Orthanc 1.2.0
3421 SetInstanceMetadata(content, instanceMetadata_, instanceId,
3422 MetadataType_Instance_TransferSyntax,
3423 GetTransferSyntaxUid(transferSyntax_));
3424 }
3425
3426 {
3427 std::string s;
3428
3429 if (origin_.LookupRemoteIp(s))
3430 {
3431 // New in Orthanc 1.4.0
3432 SetInstanceMetadata(content, instanceMetadata_, instanceId,
3433 MetadataType_Instance_RemoteIp, s);
3434 }
3435
3436 if (origin_.LookupCalledAet(s))
3437 {
3438 // New in Orthanc 1.4.0
3439 SetInstanceMetadata(content, instanceMetadata_, instanceId,
3440 MetadataType_Instance_CalledAet, s);
3441 }
3442
3443 if (origin_.LookupHttpUsername(s))
3444 {
3445 // New in Orthanc 1.4.0
3446 SetInstanceMetadata(content, instanceMetadata_, instanceId,
3447 MetadataType_Instance_HttpUsername, s);
3448 }
3449 }
3450
3451 if (hasPixelDataOffset_)
3452 {
3453 // New in Orthanc 1.9.1
3454 SetInstanceMetadata(content, instanceMetadata_, instanceId,
3455 MetadataType_Instance_PixelDataOffset,
3456 boost::lexical_cast<std::string>(pixelDataOffset_));
3457 }
3458
3459 const DicomValue* value;
3460 if ((value = dicomSummary_.TestAndGetValue(DICOM_TAG_SOP_CLASS_UID)) != NULL &&
3461 !value->IsNull() &&
3462 !value->IsBinary())
3463 {
3464 SetInstanceMetadata(content, instanceMetadata_, instanceId,
3465 MetadataType_Instance_SopClassUid, value->GetContent());
3466 }
3467
3468
3469 if ((value = dicomSummary_.TestAndGetValue(DICOM_TAG_INSTANCE_NUMBER)) != NULL ||
3470 (value = dicomSummary_.TestAndGetValue(DICOM_TAG_IMAGE_INDEX)) != NULL)
3471 {
3472 if (!value->IsNull() &&
3473 !value->IsBinary())
3474 {
3475 SetInstanceMetadata(content, instanceMetadata_, instanceId,
3476 MetadataType_Instance_IndexInSeries, Toolbox::StripSpaces(value->GetContent()));
3477 }
3478 }
3479
3480
3481 transaction.SetResourcesContent(content);
3482 }
3483
3484
3485 // Check whether the series of this new instance is now completed
3486 int64_t expectedNumberOfInstances;
3487 if (ComputeExpectedNumberOfInstances(expectedNumberOfInstances, dicomSummary_))
3488 {
3489 SeriesStatus seriesStatus = transaction.GetSeriesStatus(status.seriesId_, expectedNumberOfInstances);
3490 if (seriesStatus == SeriesStatus_Complete)
3491 {
3492 transaction.LogChange(status.seriesId_, ChangeType_CompletedSeries, ResourceType_Series, hashSeries_);
3493 }
3494 }
3495
3496
3497 // Mark the parent resources of this instance as unstable
3498 index_.MarkAsUnstable(status.seriesId_, ResourceType_Series, hashSeries_);
3499 index_.MarkAsUnstable(status.studyId_, ResourceType_Study, hashStudy_);
3500 index_.MarkAsUnstable(status.patientId_, ResourceType_Patient, hashPatient_);
3501
3502 transaction.GetListener().SignalAttachmentsAdded(instanceSize);
3503
3504 storeStatus_ = StoreStatus_Success;
3505 }
3506 catch (OrthancException& e)
3507 {
3508 LOG(ERROR) << "EXCEPTION [" << e.What() << "]";
3509 storeStatus_ = StoreStatus_Failure;
3510 }
3511 }
3512 };
3513
3514
3515 std::unique_ptr<Operations> operations;
3516
3517 {
3518 boost::mutex::scoped_lock lock(monitoringMutex_);
3519 operations.reset(new Operations(instanceMetadata, *this, dicomSummary, attachments, metadata, origin,
3520 overwrite, hasTransferSyntax, transferSyntax, hasPixelDataOffset,
3521 pixelDataOffset, maximumStorageSize_, maximumPatients_));
3522 }
3523
3524 Apply(*operations);
3525 return operations->GetStoreStatus();
3526 }
3527
3528
3529 StoreStatus ServerIndex::AddAttachment(const FileInfo& attachment,
3530 const std::string& publicId)
3531 {
3532 class Operations : public IReadWriteOperations
3533 {
3534 private:
3535 StoreStatus status_;
3536 const FileInfo& attachment_;
3537 const std::string& publicId_;
3538 uint64_t maximumStorageSize_;
3539 unsigned int maximumPatientCount_;
3540
3541 public:
3542 Operations(const FileInfo& attachment,
3543 const std::string& publicId,
3544 uint64_t maximumStorageSize,
3545 unsigned int maximumPatientCount) :
3546 status_(StoreStatus_Failure),
3547 attachment_(attachment),
3548 publicId_(publicId),
3549 maximumStorageSize_(maximumStorageSize),
3550 maximumPatientCount_(maximumPatientCount)
3551 {
3552 }
3553
3554 StoreStatus GetStatus() const
3555 {
3556 return status_;
3557 }
3558
3559 virtual void Apply(ReadWriteTransaction& transaction) ORTHANC_OVERRIDE
3560 {
3561 ResourceType resourceType;
3562 int64_t resourceId;
3563 if (!transaction.LookupResource(resourceId, resourceType, publicId_))
3564 {
3565 status_ = StoreStatus_Failure; // Inexistent resource
3566 }
3567 else
3568 {
3569 // Remove possible previous attachment
3570 transaction.DeleteAttachment(resourceId, attachment_.GetContentType());
3571
3572 // Locate the patient of the target resource
3573 int64_t patientId = resourceId;
3574 for (;;)
3575 {
3576 int64_t parent;
3577 if (transaction.LookupParent(parent, patientId))
3578 {
3579 // We have not reached the patient level yet
3580 patientId = parent;
3581 }
3582 else
3583 {
3584 // We have reached the patient level
3585 break;
3586 }
3587 }
3588
3589 // Possibly apply the recycling mechanism while preserving this patient
3590 assert(transaction.GetResourceType(patientId) == ResourceType_Patient);
3591 transaction.Recycle(maximumStorageSize_, maximumPatientCount_,
3592 attachment_.GetCompressedSize(), transaction.GetPublicId(patientId));
3593
3594 transaction.AddAttachment(resourceId, attachment_);
3595
3596 if (IsUserContentType(attachment_.GetContentType()))
3597 {
3598 transaction.LogChange(resourceId, ChangeType_UpdatedAttachment, resourceType, publicId_);
3599 }
3600
3601 transaction.GetListener().SignalAttachmentsAdded(attachment_.GetCompressedSize());
3602
3603 status_ = StoreStatus_Success;
3604 }
3605 }
3606 };
3607
3608
3609 std::unique_ptr<Operations> operations;
3610
3611 {
3612 boost::mutex::scoped_lock lock(monitoringMutex_);
3613 operations.reset(new Operations(attachment, publicId, maximumStorageSize_, maximumPatients_));
3614 }
3615
3616 Apply(*operations);
3617 return operations->GetStatus();
3618 }
3464 } 3619 }