# HG changeset patch # User Alain Mazy # Date 1707168759 -3600 # Node ID 8ba9b20ae95ff9955e2edb781e1f82fa5e484208 # Parent 79812e0df1627a3b8596ce7a46c49116d2ba7302 debug pg transactions tests diff -r 79812e0df162 -r 8ba9b20ae95f NewTests/Concurrency/docker-compose-transfers-concurrency.yml --- a/NewTests/Concurrency/docker-compose-transfers-concurrency.yml Mon Feb 05 17:12:07 2024 +0100 +++ b/NewTests/Concurrency/docker-compose-transfers-concurrency.yml Mon Feb 05 22:32:39 2024 +0100 @@ -14,7 +14,7 @@ ORTHANC__POSTGRESQL: | { "Host": "pg-a", - "TransactionMode": "ReadCommitted" + "TransactionMode": "READ COMMITTED" } ORTHANC__AUTHENTICATION_ENABLED: "false" ORTHANC__ORTHANC_PEERS: | @@ -42,7 +42,7 @@ ORTHANC__POSTGRESQL: | { "Host": "pg-b", - "TransactionMode": "ReadCommitted" + "TransactionMode": "READ COMMITTED" } ORTHANC__AUTHENTICATION_ENABLED: "false" ORTHANC__ORTHANC_PEERS: | diff -r 79812e0df162 -r 8ba9b20ae95f NewTests/Concurrency/test_concurrency.py --- a/NewTests/Concurrency/test_concurrency.py Mon Feb 05 17:12:07 2024 +0100 +++ b/NewTests/Concurrency/test_concurrency.py Mon Feb 05 22:32:39 2024 +0100 @@ -102,9 +102,9 @@ "IndexConnectionsCount": 10, "MaximumConnectionRetries" : 2000, "ConnectionRetryInterval" : 5, - "TransactionMode": "ReadCommitted", + "TransactionMode": "READ COMMITTED" #"TransactionMode": "Serializable", - "EnableVerboseLogs": True + # "EnableVerboseLogs": True }, "AuthenticationEnabled": False, "OverwriteInstances": True, @@ -149,11 +149,15 @@ cls.o.delete_all_content() def check_is_empty(self): + print("checking is empty") + self.assertEqual(0, len(self.o.studies.get_all_ids())) self.assertEqual(0, len(self.o.series.get_all_ids())) self.assertEqual(0, len(self.o.instances.get_all_ids())) + print("checking is empty (2)") stats = self.o.get_json("/statistics") + print("checking is empty (3)") self.assertEqual(0, stats.get("CountPatients")) self.assertEqual(0, stats.get("CountStudies")) self.assertEqual(0, stats.get("CountSeries")) @@ -178,40 +182,40 @@ for t in workers: t.join() - def test_concurrent_uploads_same_study(self): - self.o.delete_all_content() - self.clear_storage(storage_name=self._storage_name) + # def test_concurrent_uploads_same_study(self): + # self.o.delete_all_content() + # self.clear_storage(storage_name=self._storage_name) - start_time = time.time() - workers_count = 20 - repeat_count = 1 + # start_time = time.time() + # workers_count = 20 + # repeat_count = 1 - # massively reupload the same study multiple times with OverwriteInstances set to true - # Make sure the studies, series and instances are created only once - self.execute_workers( - worker_func=worker_upload_folder, - worker_args=(self.o._root_url, here / "../../Database/Knee", repeat_count,), - workers_count=workers_count) + # # massively reupload the same study multiple times with OverwriteInstances set to true + # # Make sure the studies, series and instances are created only once + # self.execute_workers( + # worker_func=worker_upload_folder, + # worker_args=(self.o._root_url, here / "../../Database/Knee", repeat_count,), + # workers_count=workers_count) - elapsed = time.time() - start_time - print(f"TIMING test_concurrent_uploads_same_study with {workers_count} workers and {repeat_count}x repeat: {elapsed:.3f} s") + # elapsed = time.time() - start_time + # print(f"TIMING test_concurrent_uploads_same_study with {workers_count} workers and {repeat_count}x repeat: {elapsed:.3f} s") - self.assertTrue(self.o.is_alive()) + # self.assertTrue(self.o.is_alive()) - self.assertEqual(1, len(self.o.studies.get_all_ids())) - self.assertEqual(2, len(self.o.series.get_all_ids())) - self.assertEqual(50, len(self.o.instances.get_all_ids())) + # self.assertEqual(1, len(self.o.studies.get_all_ids())) + # self.assertEqual(2, len(self.o.series.get_all_ids())) + # self.assertEqual(50, len(self.o.instances.get_all_ids())) - stats = self.o.get_json("/statistics") - self.assertEqual(1, stats.get("CountPatients")) - self.assertEqual(1, stats.get("CountStudies")) - self.assertEqual(2, stats.get("CountSeries")) - self.assertEqual(50, stats.get("CountInstances")) - self.assertEqual(4118738, int(stats.get("TotalDiskSize"))) + # stats = self.o.get_json("/statistics") + # self.assertEqual(1, stats.get("CountPatients")) + # self.assertEqual(1, stats.get("CountStudies")) + # self.assertEqual(2, stats.get("CountSeries")) + # self.assertEqual(50, stats.get("CountInstances")) + # self.assertEqual(4118738, int(stats.get("TotalDiskSize"))) - self.o.instances.delete(orthanc_ids=self.o.instances.get_all_ids()) + # self.o.instances.delete(orthanc_ids=self.o.instances.get_all_ids()) - self.check_is_empty() + # self.check_is_empty() def test_concurrent_anonymize_same_study(self): self.o.delete_all_content() @@ -222,7 +226,7 @@ start_time = time.time() workers_count = 4 - repeat_count = 10 + repeat_count = 2 # massively anonymize the same study. This generates new studies and is a # good way to simulate ingestion of new studies @@ -240,12 +244,16 @@ self.assertEqual(2 * (1 + workers_count * repeat_count), len(self.o.series.get_all_ids())) self.assertEqual(50 * (1 + workers_count * repeat_count), len(self.o.instances.get_all_ids())) + print("get stats") stats = self.o.get_json("/statistics") + print("get stats (2)") self.assertEqual(1 + workers_count * repeat_count, stats.get("CountPatients")) self.assertEqual(1 + workers_count * repeat_count, stats.get("CountStudies")) self.assertEqual(2 * (1 + workers_count * repeat_count), stats.get("CountSeries")) self.assertEqual(50 * (1 + workers_count * repeat_count), stats.get("CountInstances")) + print("get changes") changes, last_change, done = self.o.get_changes(since=0, limit=100000) + print("get changes (2)") self.assertTrue(done) self.assertEqual(1 + workers_count * repeat_count, count_changes(changes, ChangeType.NEW_PATIENT)) @@ -255,7 +263,11 @@ start_time = time.time() - self.o.instances.delete(orthanc_ids=self.o.instances.get_all_ids()) + print("deleting") + all_instances_ids = self.o.instances.get_all_ids() + print("deleting (2)") + self.o.instances.delete(orthanc_ids=all_instances_ids) + print("deleted") elapsed = time.time() - start_time print(f"TIMING test_concurrent_anonymize_same_study deletion took: {elapsed:.3f} s") diff -r 79812e0df162 -r 8ba9b20ae95f NewTests/PostgresUpgrades/docker-compose.yml --- a/NewTests/PostgresUpgrades/docker-compose.yml Mon Feb 05 17:12:07 2024 +0100 +++ b/NewTests/PostgresUpgrades/docker-compose.yml Mon Feb 05 22:32:39 2024 +0100 @@ -71,7 +71,7 @@ ORTHANC__POSTGRESQL: | { "Host": "pg-15", - "TransactionMode": "ReadCommitted" + "TransactionMode": "READ COMMITTED" } ORTHANC__AUTHENTICATION_ENABLED: "false" @@ -106,7 +106,7 @@ ORTHANC__POSTGRESQL: | { "Host": "pg-9", - "TransactionMode": "ReadCommitted" + "TransactionMode": "READ COMMITTED" } ORTHANC__AUTHENTICATION_ENABLED: "false" diff -r 79812e0df162 -r 8ba9b20ae95f NewTests/helpers.py --- a/NewTests/helpers.py Mon Feb 05 17:12:07 2024 +0100 +++ b/NewTests/helpers.py Mon Feb 05 22:32:39 2024 +0100 @@ -164,6 +164,8 @@ @classmethod def clear_storage(cls, storage_name: str): storage_path = cls.get_storage_path(storage_name=storage_name) + print("clearing storage") + if Helpers.is_exe(): # clear the directory but keep it ! @@ -174,6 +176,8 @@ shutil.rmtree(os.path.join(root, d)) shutil.rmtree(storage_path, ignore_errors=True) else: + # subprocess.run(["sudo", "rm", "-rf", storage_path], check=True) + # docker run --rm -v /home/alain/o/orthanc-tests/NewTests/storages/concurrency/:/var/lib/orthanc/db/ debian:12-slim rm -rf /var/lib/orthanc/db/* cmd = [ "docker", "run", "--rm", "-v", f"{storage_path}:/var/lib/orthanc/db/", @@ -183,11 +187,21 @@ ] subprocess.run(cmd, check=True) + + if not cls.is_storage_empty(storage_name): + print("Could not clear storage") + exit(-12) + print("cleared storage") + @classmethod def is_storage_empty(cls, storage_name: str): + print("checking storage empty") storage_path = cls.get_storage_path(storage_name=storage_name) - return len(glob.glob(os.path.join(storage_path, '*'))) == 0 + res = len(glob.glob(os.path.join(storage_path, '*'))) == 0 + print("checked storage empty") + + return res @classmethod def create_docker_network(cls, network: str): @@ -284,8 +298,8 @@ cmd = [ "docker", "run", "--rm", - "-e", "VERBOSE_ENABLED=true" if enable_verbose else "VERBOSE_ENABLED=false", - "-e", "VERBOSE_STARTUP=true" if enable_verbose else "VERBOSE_STARTUP=false", + "-e", "VERBOSE_ENABLED=true", # if enable_verbose else "VERBOSE_ENABLED=false", + "-e", "VERBOSE_STARTUP=true", # always set it !!!! if enable_verbose else "VERBOSE_STARTUP=false", "-v", f"{config_path}:/etc/orthanc/orthanc.json", "-v", f"{storage_path}:/var/lib/orthanc/db/", "--name", config_name,