Mercurial > hg > orthanc-tests

comparison NewTests/Concurrency/test_concurrency.py @ 620:8ba9b20ae95f debug-pg-transactions

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
debug pg transactions tests
author Alain Mazy <am@osimis.io>
date Mon, 05 Feb 2024 22:32:39 +0100
parents 6ba2ff41ea52
children
comparison
equal deleted inserted replaced
619:79812e0df162 620:8ba9b20ae95f
100 "Username": "postgres", 100 "Username": "postgres",
101 "Password": "postgres", 101 "Password": "postgres",
102 "IndexConnectionsCount": 10, 102 "IndexConnectionsCount": 10,
103 "MaximumConnectionRetries" : 2000, 103 "MaximumConnectionRetries" : 2000,
104 "ConnectionRetryInterval" : 5, 104 "ConnectionRetryInterval" : 5,
105 "TransactionMode": "ReadCommitted", 105 "TransactionMode": "READ COMMITTED"
106 #"TransactionMode": "Serializable", 106 #"TransactionMode": "Serializable",
107 "EnableVerboseLogs": True 107 # "EnableVerboseLogs": True
108 }, 108 },
109 "AuthenticationEnabled": False, 109 "AuthenticationEnabled": False,
110 "OverwriteInstances": True, 110 "OverwriteInstances": True,
111 "JobsEngineThreadsCount" : { 111 "JobsEngineThreadsCount" : {
112 "ResourceModification": 8 112 "ResourceModification": 8
147 cls.o = OrthancApiClient(cls.o._root_url) 147 cls.o = OrthancApiClient(cls.o._root_url)
148 cls.o.wait_started() 148 cls.o.wait_started()
149 cls.o.delete_all_content() 149 cls.o.delete_all_content()
150 150
151 def check_is_empty(self): 151 def check_is_empty(self):
152 print("checking is empty")
153
152 self.assertEqual(0, len(self.o.studies.get_all_ids())) 154 self.assertEqual(0, len(self.o.studies.get_all_ids()))
153 self.assertEqual(0, len(self.o.series.get_all_ids())) 155 self.assertEqual(0, len(self.o.series.get_all_ids()))
154 self.assertEqual(0, len(self.o.instances.get_all_ids())) 156 self.assertEqual(0, len(self.o.instances.get_all_ids()))
155 157
158 print("checking is empty (2)")
156 stats = self.o.get_json("/statistics") 159 stats = self.o.get_json("/statistics")
160 print("checking is empty (3)")
157 self.assertEqual(0, stats.get("CountPatients")) 161 self.assertEqual(0, stats.get("CountPatients"))
158 self.assertEqual(0, stats.get("CountStudies")) 162 self.assertEqual(0, stats.get("CountStudies"))
159 self.assertEqual(0, stats.get("CountSeries")) 163 self.assertEqual(0, stats.get("CountSeries"))
160 self.assertEqual(0, stats.get("CountInstances")) 164 self.assertEqual(0, stats.get("CountInstances"))
161 self.assertEqual(0, int(stats.get("TotalDiskSize"))) 165 self.assertEqual(0, int(stats.get("TotalDiskSize")))
176 t.start() 180 t.start()
177 181
178 for t in workers: 182 for t in workers:
179 t.join() 183 t.join()
180 184
181 def test_concurrent_uploads_same_study(self): 185 # def test_concurrent_uploads_same_study(self):
182 self.o.delete_all_content() 186 # self.o.delete_all_content()
183 self.clear_storage(storage_name=self._storage_name) 187 # self.clear_storage(storage_name=self._storage_name)
184 188
185 start_time = time.time() 189 # start_time = time.time()
186 workers_count = 20 190 # workers_count = 20
187 repeat_count = 1 191 # repeat_count = 1
188 192
189 # massively reupload the same study multiple times with OverwriteInstances set to true 193 # # massively reupload the same study multiple times with OverwriteInstances set to true
190 # Make sure the studies, series and instances are created only once 194 # # Make sure the studies, series and instances are created only once
191 self.execute_workers( 195 # self.execute_workers(
192 worker_func=worker_upload_folder, 196 # worker_func=worker_upload_folder,
193 worker_args=(self.o._root_url, here / "../../Database/Knee", repeat_count,), 197 # worker_args=(self.o._root_url, here / "../../Database/Knee", repeat_count,),
194 workers_count=workers_count) 198 # workers_count=workers_count)
195 199
196 elapsed = time.time() - start_time 200 # elapsed = time.time() - start_time
197 print(f"TIMING test_concurrent_uploads_same_study with {workers_count} workers and {repeat_count}x repeat: {elapsed:.3f} s") 201 # print(f"TIMING test_concurrent_uploads_same_study with {workers_count} workers and {repeat_count}x repeat: {elapsed:.3f} s")
198 202
199 self.assertTrue(self.o.is_alive()) 203 # self.assertTrue(self.o.is_alive())
200 204
201 self.assertEqual(1, len(self.o.studies.get_all_ids())) 205 # self.assertEqual(1, len(self.o.studies.get_all_ids()))
202 self.assertEqual(2, len(self.o.series.get_all_ids())) 206 # self.assertEqual(2, len(self.o.series.get_all_ids()))
203 self.assertEqual(50, len(self.o.instances.get_all_ids())) 207 # self.assertEqual(50, len(self.o.instances.get_all_ids()))
204 208
205 stats = self.o.get_json("/statistics") 209 # stats = self.o.get_json("/statistics")
206 self.assertEqual(1, stats.get("CountPatients")) 210 # self.assertEqual(1, stats.get("CountPatients"))
207 self.assertEqual(1, stats.get("CountStudies")) 211 # self.assertEqual(1, stats.get("CountStudies"))
208 self.assertEqual(2, stats.get("CountSeries")) 212 # self.assertEqual(2, stats.get("CountSeries"))
209 self.assertEqual(50, stats.get("CountInstances")) 213 # self.assertEqual(50, stats.get("CountInstances"))
210 self.assertEqual(4118738, int(stats.get("TotalDiskSize"))) 214 # self.assertEqual(4118738, int(stats.get("TotalDiskSize")))
211 215
212 self.o.instances.delete(orthanc_ids=self.o.instances.get_all_ids()) 216 # self.o.instances.delete(orthanc_ids=self.o.instances.get_all_ids())
213 217
214 self.check_is_empty() 218 # self.check_is_empty()
215 219
216 def test_concurrent_anonymize_same_study(self): 220 def test_concurrent_anonymize_same_study(self):
217 self.o.delete_all_content() 221 self.o.delete_all_content()
218 self.clear_storage(storage_name=self._storage_name) 222 self.clear_storage(storage_name=self._storage_name)
219 223
220 self.o.upload_folder(here / "../../Database/Knee") 224 self.o.upload_folder(here / "../../Database/Knee")
221 study_id = self.o.studies.get_all_ids()[0] 225 study_id = self.o.studies.get_all_ids()[0]
222 226
223 start_time = time.time() 227 start_time = time.time()
224 workers_count = 4 228 workers_count = 4
225 repeat_count = 10 229 repeat_count = 2
226 230
227 # massively anonymize the same study. This generates new studies and is a 231 # massively anonymize the same study. This generates new studies and is a
228 # good way to simulate ingestion of new studies 232 # good way to simulate ingestion of new studies
229 self.execute_workers( 233 self.execute_workers(
230 worker_func=worker_anonymize_study, 234 worker_func=worker_anonymize_study,
238 242
239 self.assertEqual(1 + workers_count * repeat_count, len(self.o.studies.get_all_ids())) 243 self.assertEqual(1 + workers_count * repeat_count, len(self.o.studies.get_all_ids()))
240 self.assertEqual(2 * (1 + workers_count * repeat_count), len(self.o.series.get_all_ids())) 244 self.assertEqual(2 * (1 + workers_count * repeat_count), len(self.o.series.get_all_ids()))
241 self.assertEqual(50 * (1 + workers_count * repeat_count), len(self.o.instances.get_all_ids())) 245 self.assertEqual(50 * (1 + workers_count * repeat_count), len(self.o.instances.get_all_ids()))
242 246
247 print("get stats")
243 stats = self.o.get_json("/statistics") 248 stats = self.o.get_json("/statistics")
249 print("get stats (2)")
244 self.assertEqual(1 + workers_count * repeat_count, stats.get("CountPatients")) 250 self.assertEqual(1 + workers_count * repeat_count, stats.get("CountPatients"))
245 self.assertEqual(1 + workers_count * repeat_count, stats.get("CountStudies")) 251 self.assertEqual(1 + workers_count * repeat_count, stats.get("CountStudies"))
246 self.assertEqual(2 * (1 + workers_count * repeat_count), stats.get("CountSeries")) 252 self.assertEqual(2 * (1 + workers_count * repeat_count), stats.get("CountSeries"))
247 self.assertEqual(50 * (1 + workers_count * repeat_count), stats.get("CountInstances")) 253 self.assertEqual(50 * (1 + workers_count * repeat_count), stats.get("CountInstances"))
254 print("get changes")
248 changes, last_change, done = self.o.get_changes(since=0, limit=100000) 255 changes, last_change, done = self.o.get_changes(since=0, limit=100000)
256 print("get changes (2)")
249 self.assertTrue(done) 257 self.assertTrue(done)
250 258
251 self.assertEqual(1 + workers_count * repeat_count, count_changes(changes, ChangeType.NEW_PATIENT)) 259 self.assertEqual(1 + workers_count * repeat_count, count_changes(changes, ChangeType.NEW_PATIENT))
252 self.assertEqual(1 + workers_count * repeat_count, count_changes(changes, ChangeType.NEW_STUDY)) 260 self.assertEqual(1 + workers_count * repeat_count, count_changes(changes, ChangeType.NEW_STUDY))
253 self.assertEqual(2 * (1 + workers_count * repeat_count), count_changes(changes, ChangeType.NEW_SERIES)) 261 self.assertEqual(2 * (1 + workers_count * repeat_count), count_changes(changes, ChangeType.NEW_SERIES))
254 self.assertEqual(50 * (1 + workers_count * repeat_count), count_changes(changes, ChangeType.NEW_INSTANCE)) 262 self.assertEqual(50 * (1 + workers_count * repeat_count), count_changes(changes, ChangeType.NEW_INSTANCE))
255 263
256 start_time = time.time() 264 start_time = time.time()
257 265
258 self.o.instances.delete(orthanc_ids=self.o.instances.get_all_ids()) 266 print("deleting")
267 all_instances_ids = self.o.instances.get_all_ids()
268 print("deleting (2)")
269 self.o.instances.delete(orthanc_ids=all_instances_ids)
270 print("deleted")
259 271
260 elapsed = time.time() - start_time 272 elapsed = time.time() - start_time
261 print(f"TIMING test_concurrent_anonymize_same_study deletion took: {elapsed:.3f} s") 273 print(f"TIMING test_concurrent_anonymize_same_study deletion took: {elapsed:.3f} s")
262 274
263 self.check_is_empty() 275 self.check_is_empty()