Mercurial > hg > orthanc-tests

changeset 620:8ba9b20ae95f debug-pg-transactions

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
debug pg transactions tests
author Alain Mazy <am@osimis.io>
date Mon, 05 Feb 2024 22:32:39 +0100
parents 79812e0df162
children
files NewTests/Concurrency/docker-compose-transfers-concurrency.yml NewTests/Concurrency/test_concurrency.py NewTests/PostgresUpgrades/docker-compose.yml NewTests/helpers.py
diffstat 4 files changed, 63 insertions(+), 37 deletions(-) [+]
line wrap: on
line diff
--- a/NewTests/Concurrency/docker-compose-transfers-concurrency.yml	Mon Feb 05 17:12:07 2024 +0100
+++ b/NewTests/Concurrency/docker-compose-transfers-concurrency.yml	Mon Feb 05 22:32:39 2024 +0100
@@ -14,7 +14,7 @@
       ORTHANC__POSTGRESQL: |
         {
           "Host": "pg-a",
-          "TransactionMode": "ReadCommitted"
+          "TransactionMode": "READ COMMITTED"
         }
       ORTHANC__AUTHENTICATION_ENABLED: "false"
       ORTHANC__ORTHANC_PEERS: |
@@ -42,7 +42,7 @@
       ORTHANC__POSTGRESQL: |
         {
           "Host": "pg-b",
-          "TransactionMode": "ReadCommitted"
+          "TransactionMode": "READ COMMITTED"
         }
       ORTHANC__AUTHENTICATION_ENABLED: "false"
       ORTHANC__ORTHANC_PEERS: |
--- a/NewTests/Concurrency/test_concurrency.py	Mon Feb 05 17:12:07 2024 +0100
+++ b/NewTests/Concurrency/test_concurrency.py	Mon Feb 05 22:32:39 2024 +0100
@@ -102,9 +102,9 @@
                 "IndexConnectionsCount": 10,
                 "MaximumConnectionRetries" : 2000,
                 "ConnectionRetryInterval" : 5,
-                "TransactionMode": "ReadCommitted",
+                "TransactionMode": "READ COMMITTED"
                 #"TransactionMode": "Serializable",
-                "EnableVerboseLogs": True
+                # "EnableVerboseLogs": True
             },
             "AuthenticationEnabled": False,
             "OverwriteInstances": True,
@@ -149,11 +149,15 @@
         cls.o.delete_all_content()
 
     def check_is_empty(self):
+        print("checking is empty")
+
         self.assertEqual(0, len(self.o.studies.get_all_ids()))
         self.assertEqual(0, len(self.o.series.get_all_ids()))
         self.assertEqual(0, len(self.o.instances.get_all_ids()))
 
+        print("checking is empty (2)")
         stats = self.o.get_json("/statistics")
+        print("checking is empty (3)")
         self.assertEqual(0, stats.get("CountPatients"))
         self.assertEqual(0, stats.get("CountStudies"))
         self.assertEqual(0, stats.get("CountSeries"))
@@ -178,40 +182,40 @@
         for t in workers:
             t.join()
 
-    def test_concurrent_uploads_same_study(self):
-        self.o.delete_all_content()
-        self.clear_storage(storage_name=self._storage_name)
+    # def test_concurrent_uploads_same_study(self):
+    #     self.o.delete_all_content()
+    #     self.clear_storage(storage_name=self._storage_name)
 
-        start_time = time.time()
-        workers_count = 20
-        repeat_count = 1
+    #     start_time = time.time()
+    #     workers_count = 20
+    #     repeat_count = 1
 
-        # massively reupload the same study multiple times with OverwriteInstances set to true
-        # Make sure the studies, series and instances are created only once
-        self.execute_workers(
-            worker_func=worker_upload_folder,
-            worker_args=(self.o._root_url, here / "../../Database/Knee", repeat_count,),
-            workers_count=workers_count)
+    #     # massively reupload the same study multiple times with OverwriteInstances set to true
+    #     # Make sure the studies, series and instances are created only once
+    #     self.execute_workers(
+    #         worker_func=worker_upload_folder,
+    #         worker_args=(self.o._root_url, here / "../../Database/Knee", repeat_count,),
+    #         workers_count=workers_count)
 
-        elapsed = time.time() - start_time
-        print(f"TIMING test_concurrent_uploads_same_study with {workers_count} workers and {repeat_count}x repeat: {elapsed:.3f} s")
+    #     elapsed = time.time() - start_time
+    #     print(f"TIMING test_concurrent_uploads_same_study with {workers_count} workers and {repeat_count}x repeat: {elapsed:.3f} s")
 
-        self.assertTrue(self.o.is_alive())
+    #     self.assertTrue(self.o.is_alive())
 
-        self.assertEqual(1, len(self.o.studies.get_all_ids()))
-        self.assertEqual(2, len(self.o.series.get_all_ids()))
-        self.assertEqual(50, len(self.o.instances.get_all_ids()))
+    #     self.assertEqual(1, len(self.o.studies.get_all_ids()))
+    #     self.assertEqual(2, len(self.o.series.get_all_ids()))
+    #     self.assertEqual(50, len(self.o.instances.get_all_ids()))
 
-        stats = self.o.get_json("/statistics")
-        self.assertEqual(1, stats.get("CountPatients"))
-        self.assertEqual(1, stats.get("CountStudies"))
-        self.assertEqual(2, stats.get("CountSeries"))
-        self.assertEqual(50, stats.get("CountInstances"))
-        self.assertEqual(4118738, int(stats.get("TotalDiskSize")))
+    #     stats = self.o.get_json("/statistics")
+    #     self.assertEqual(1, stats.get("CountPatients"))
+    #     self.assertEqual(1, stats.get("CountStudies"))
+    #     self.assertEqual(2, stats.get("CountSeries"))
+    #     self.assertEqual(50, stats.get("CountInstances"))
+    #     self.assertEqual(4118738, int(stats.get("TotalDiskSize")))
 
-        self.o.instances.delete(orthanc_ids=self.o.instances.get_all_ids())
+    #     self.o.instances.delete(orthanc_ids=self.o.instances.get_all_ids())
 
-        self.check_is_empty()
+    #     self.check_is_empty()
 
     def test_concurrent_anonymize_same_study(self):
         self.o.delete_all_content()
@@ -222,7 +226,7 @@
 
         start_time = time.time()
         workers_count = 4
-        repeat_count = 10
+        repeat_count = 2
 
         # massively anonymize the same study.  This generates new studies and is a
         # good way to simulate ingestion of new studies
@@ -240,12 +244,16 @@
         self.assertEqual(2 * (1 + workers_count * repeat_count), len(self.o.series.get_all_ids()))
         self.assertEqual(50 * (1 + workers_count * repeat_count), len(self.o.instances.get_all_ids()))
 
+        print("get stats")
         stats = self.o.get_json("/statistics")
+        print("get stats (2)")
         self.assertEqual(1 + workers_count * repeat_count, stats.get("CountPatients"))
         self.assertEqual(1 + workers_count * repeat_count, stats.get("CountStudies"))
         self.assertEqual(2 * (1 + workers_count * repeat_count), stats.get("CountSeries"))
         self.assertEqual(50 * (1 + workers_count * repeat_count), stats.get("CountInstances"))
+        print("get changes")
         changes, last_change, done = self.o.get_changes(since=0, limit=100000)
+        print("get changes (2)")
         self.assertTrue(done)
 
         self.assertEqual(1 + workers_count * repeat_count, count_changes(changes, ChangeType.NEW_PATIENT))
@@ -255,7 +263,11 @@
 
         start_time = time.time()
 
-        self.o.instances.delete(orthanc_ids=self.o.instances.get_all_ids())
+        print("deleting")
+        all_instances_ids = self.o.instances.get_all_ids()
+        print("deleting (2)")
+        self.o.instances.delete(orthanc_ids=all_instances_ids)
+        print("deleted")
 
         elapsed = time.time() - start_time
         print(f"TIMING test_concurrent_anonymize_same_study deletion took: {elapsed:.3f} s")
--- a/NewTests/PostgresUpgrades/docker-compose.yml	Mon Feb 05 17:12:07 2024 +0100
+++ b/NewTests/PostgresUpgrades/docker-compose.yml	Mon Feb 05 22:32:39 2024 +0100
@@ -71,7 +71,7 @@
       ORTHANC__POSTGRESQL: |
         {
           "Host": "pg-15",
-          "TransactionMode": "ReadCommitted"
+          "TransactionMode": "READ COMMITTED"
         }
       ORTHANC__AUTHENTICATION_ENABLED: "false"
 
@@ -106,7 +106,7 @@
       ORTHANC__POSTGRESQL: |
         {
           "Host": "pg-9",
-          "TransactionMode": "ReadCommitted"
+          "TransactionMode": "READ COMMITTED"
         }
       ORTHANC__AUTHENTICATION_ENABLED: "false"
 
--- a/NewTests/helpers.py	Mon Feb 05 17:12:07 2024 +0100
+++ b/NewTests/helpers.py	Mon Feb 05 22:32:39 2024 +0100
@@ -164,6 +164,8 @@
     @classmethod
     def clear_storage(cls, storage_name: str):
         storage_path = cls.get_storage_path(storage_name=storage_name)
+        print("clearing storage")
+
         if Helpers.is_exe():
 
             # clear the directory but keep it !
@@ -174,6 +176,8 @@
                     shutil.rmtree(os.path.join(root, d))
                     shutil.rmtree(storage_path, ignore_errors=True)
         else:
+            # subprocess.run(["sudo", "rm", "-rf", storage_path], check=True)
+            # docker run --rm -v /home/alain/o/orthanc-tests/NewTests/storages/concurrency/:/var/lib/orthanc/db/ debian:12-slim rm -rf /var/lib/orthanc/db/*
             cmd = [
                     "docker", "run", "--rm", 
                     "-v", f"{storage_path}:/var/lib/orthanc/db/",
@@ -183,11 +187,21 @@
                 ]
 
             subprocess.run(cmd, check=True)
+        
+        if not cls.is_storage_empty(storage_name):
+            print("Could not clear storage")
+            exit(-12)
+        print("cleared storage")
+
 
     @classmethod
     def is_storage_empty(cls, storage_name: str):
+        print("checking storage empty")
         storage_path = cls.get_storage_path(storage_name=storage_name)
-        return len(glob.glob(os.path.join(storage_path, '*'))) == 0
+        res =  len(glob.glob(os.path.join(storage_path, '*'))) == 0
+        print("checked storage empty")
+
+        return res
 
     @classmethod
     def create_docker_network(cls, network: str):
@@ -284,8 +298,8 @@
 
             cmd = [
                     "docker", "run", "--rm", 
-                    "-e", "VERBOSE_ENABLED=true" if enable_verbose else "VERBOSE_ENABLED=false",
-                    "-e", "VERBOSE_STARTUP=true" if enable_verbose else "VERBOSE_STARTUP=false", 
+                    "-e", "VERBOSE_ENABLED=true", # if enable_verbose else "VERBOSE_ENABLED=false",
+                    "-e", "VERBOSE_STARTUP=true", # always set it !!!! if enable_verbose else "VERBOSE_STARTUP=false", 
                     "-v", f"{config_path}:/etc/orthanc/orthanc.json",
                     "-v", f"{storage_path}:/var/lib/orthanc/db/",
                     "--name", config_name,