Mercurial > hg > orthanc-tests

changeset 601:3e15e950c462

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
new transfer tests in the concurrency section
author Alain Mazy <am@osimis.io>
date Fri, 19 Jan 2024 15:03:47 +0100
parents 58384ae69f41
children d88b0fc15f08
files NewTests/Concurrency/docker-compose-transfers-concurrency.yml NewTests/Concurrency/test_transfer.py NewTests/PostgresUpgrades/docker-compose.yml NewTests/PostgresUpgrades/test_pg_upgrades.py NewTests/helpers.py NewTests/requirements.txt
diffstat 6 files changed, 227 insertions(+), 40 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/NewTests/Concurrency/docker-compose-transfers-concurrency.yml	Fri Jan 19 15:03:47 2024 +0100
@@ -0,0 +1,82 @@
+version: "3"
+services:
+
+  orthanc-pg-a:
+    image: ${ORTHANC_IMAGE_UNDER_TESTS:-osimis/orthanc:latest}
+    container_name: orthanc-pg-a
+    depends_on: [pg-a]
+    restart: unless-stopped
+    ports: ["8052:8042"]
+    volumes: ["storage-orthanc-a:/var/lib/orthanc/db"]
+    environment:
+      # VERBOSE_ENABLED: "true"
+      TRANSFERS_PLUGIN_ENABLED: "true"
+      ORTHANC__POSTGRESQL: |
+        {
+          "Host": "pg-a",
+          "TransactionMode": "READ COMMITTED"
+        }
+      ORTHANC__AUTHENTICATION_ENABLED: "false"
+      ORTHANC__ORTHANC_PEERS: |
+        {
+          "a": {
+            "Url": "http://orthanc-pg-a:8042/",
+            "RemoteSelf": "b"
+          },
+          "b": {
+            "Url": "http://orthanc-pg-b:8042/"
+          }
+        }
+      ORTHANC__OVERWRITE_INSTANCES: "true"
+
+  orthanc-pg-b:
+    image: ${ORTHANC_IMAGE_UNDER_TESTS:-osimis/orthanc:latest}
+    container_name: orthanc-pg-b
+    depends_on: [pg-b]
+    restart: unless-stopped
+    ports: ["8053:8042"]
+    volumes: ["storage-orthanc-b:/var/lib/orthanc/db"]
+    environment:
+      # VERBOSE_ENABLED: "true"
+      TRANSFERS_PLUGIN_ENABLED: "true"
+      ORTHANC__POSTGRESQL: |
+        {
+          "Host": "pg-b",
+          "TransactionMode": "READ COMMITTED"
+        }
+      ORTHANC__AUTHENTICATION_ENABLED: "false"
+      ORTHANC__ORTHANC_PEERS: |
+        {
+          "a": {
+            "Url": "http://orthanc-pg-a:8042/",
+            "RemoteSelf": "b"
+          },
+          "b": {
+            "Url": "http://orthanc-pg-b:8042/"
+          }
+        }
+      ORTHANC__OVERWRITE_INSTANCES: "true"
+
+  pg-a:
+    image: postgres:15
+    container_name: pg-a
+    restart: unless-stopped
+    volumes: 
+      - "storage-pg-a:/var/lib/postgresql/data"
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: "trust"
+
+  pg-b:
+    image: postgres:15
+    container_name: pg-b
+    restart: unless-stopped
+    volumes: 
+      - "storage-pg-b:/var/lib/postgresql/data"
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: "trust"
+
+volumes:
+  storage-orthanc-a:
+  storage-orthanc-b:
+  storage-pg-a:
+  storage-pg-b:
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/NewTests/Concurrency/test_transfer.py	Fri Jan 19 15:03:47 2024 +0100
@@ -0,0 +1,109 @@
+import subprocess
+import time
+import unittest
+from orthanc_api_client import OrthancApiClient, ResourceType
+from orthanc_tools import OrthancTestDbPopulator
+from helpers import Helpers, wait_container_healthy
+
+import pathlib
+import os
+here = pathlib.Path(__file__).parent.resolve()
+
+
+
+
+class TestConcurrencyTransfers(unittest.TestCase):
+
+    @classmethod
+    def cleanup(cls):
+        os.chdir(here)
+        print("Cleaning old compose")
+        subprocess.run(["docker", "compose", "-f", "docker-compose-transfers-concurrency.yml", "down", "-v", "--remove-orphans"], check=True)
+
+    @classmethod
+    def compose_up(cls):
+        print("Pullling containers")
+        subprocess.run(["docker", "compose", "-f", "docker-compose-transfers-concurrency.yml", "pull"], check=True)        
+
+        print("Compose up")
+        subprocess.run(["docker", "compose", "-f", "docker-compose-transfers-concurrency.yml", "up", "-d"], check=True)        
+
+    @classmethod
+    def setUpClass(cls):
+        cls.cleanup()
+        cls.compose_up()
+
+    @classmethod
+    def tearDownClass(cls):
+        #cls.cleanup()
+        pass
+
+    def clean_start(self):
+        oa = OrthancApiClient("http://localhost:8052")
+        ob = OrthancApiClient("http://localhost:8053")
+
+        oa.wait_started()
+        ob.wait_started()
+
+        oa.delete_all_content()
+        ob.delete_all_content()
+
+        return oa, ob
+
+    def test_push(self):
+        oa, ob = self.clean_start()
+
+        populator = OrthancTestDbPopulator(oa, studies_count=5, random_seed=65)
+        populator.execute()
+
+        all_studies_ids = oa.studies.get_all_ids()
+        instances_count = oa.get_statistics().instances_count
+        disk_size = oa.get_statistics().total_disk_size
+        repeat_count = 2
+
+        for compression in [True, False]:
+            start_time = time.time()
+
+            for i in range(0, repeat_count):
+                oa.transfers.send(target_peer='b',
+                                resources_ids=all_studies_ids,
+                                resource_type=ResourceType.STUDY,
+                                compress=compression)
+                
+                self.assertEqual(instances_count, ob.get_statistics().instances_count)
+                self.assertEqual(disk_size, ob.get_statistics().total_disk_size)
+                ob.delete_all_content()
+
+            elapsed = time.time() - start_time
+            print(f"TIMING test_push (compression={compression}) with {instances_count} instances for a total of {disk_size/(1024*1024)} MB (repeat {repeat_count}x): {elapsed:.3f} s")
+
+
+    def test_pull(self):
+        oa, ob = self.clean_start()
+
+        populator = OrthancTestDbPopulator(ob, studies_count=5, random_seed=65)
+        populator.execute()
+
+        all_studies_ids = ob.studies.get_all_ids()
+        instances_count = ob.get_statistics().instances_count
+        disk_size = ob.get_statistics().total_disk_size
+        repeat_count = 2
+
+        for compression in [True, False]:
+            start_time = time.time()
+
+            for i in range(0, repeat_count):
+                remote_job = ob.transfers.send_async(target_peer='a',
+                                                    resources_ids=all_studies_ids,
+                                                    resource_type=ResourceType.STUDY,
+                                                    compress=True)
+                job = oa.jobs.get(orthanc_id=remote_job.remote_job_id)
+                job.wait_completed(polling_interval=0.1)
+
+                self.assertEqual(instances_count, oa.get_statistics().instances_count)
+                self.assertEqual(disk_size, oa.get_statistics().total_disk_size)
+                oa.delete_all_content()
+
+
+            elapsed = time.time() - start_time
+            print(f"TIMING test_pull (compression={compression}) with {instances_count} instances for a total of {disk_size/(1024*1024)} MB (repeat {repeat_count}x): {elapsed:.3f} s")
--- a/NewTests/PostgresUpgrades/docker-compose.yml	Fri Jan 19 09:15:57 2024 +0100
+++ b/NewTests/PostgresUpgrades/docker-compose.yml	Fri Jan 19 15:03:47 2024 +0100
@@ -26,10 +26,8 @@
     volumes: ["storage-orthanc-pg-15:/var/lib/orthanc/db"]
     environment:
       VERBOSE_ENABLED: "true"
-      PG_HOST: "pg-15"
-      PG_LOCK: "false"
-      PG_INDEX_ENABLED: "true"
-      AC_AUTHENTICATION_ENABLED: "false"
+      ORTHANC__POSTGRESQL__HOST: "pg-15"
+      ORTHANC__AUTHENTICATION_ENABLED: "false"
 
   # Orthanc previous version to run the integration tests
   orthanc-pg-15-61-for-integ-tests:
@@ -46,10 +44,8 @@
 
     environment:
       VERBOSE_ENABLED: "true"
-      PG_HOST: "pg-15"
-      PG_LOCK: "false"
-      PG_INDEX_ENABLED: "true"
-      AC_AUTHENTICATION_ENABLED: "false"
+      ORTHANC__POSTGRESQL__HOST: "pg-15"
+      ORTHANC__AUTHENTICATION_ENABLED: "false"
 
   orthanc-tests:
     image: jodogne/orthanc-tests
@@ -72,19 +68,18 @@
     volumes: ["storage-orthanc-pg-15:/var/lib/orthanc/db"]
     environment:
       VERBOSE_ENABLED: "true"
-      ORTHANC__AUTHENTICATION_ENABLED: "false"
       ORTHANC__POSTGRESQL: |
         {
           "Host": "pg-15",
           "TransactionMode": "READ COMMITTED"
         }
+      ORTHANC__AUTHENTICATION_ENABLED: "false"
 
 
   pg-15:
     image: postgres:15
     container_name: pg-15
     restart: unless-stopped
-    ports: ["5439:5432"]
     volumes: 
       - "storage-pg-15:/var/lib/postgresql/data"
       - "./downgrade.sh:/scripts/downgrade.sh"
@@ -108,19 +103,18 @@
     volumes: ["storage-orthanc-pg-9:/var/lib/orthanc/db"]
     environment:
       VERBOSE_ENABLED: "true"
-      ORTHANC__AUTHENTICATION_ENABLED: "false"
       ORTHANC__POSTGRESQL: |
         {
           "Host": "pg-9",
           "TransactionMode": "READ COMMITTED"
         }
+      ORTHANC__AUTHENTICATION_ENABLED: "false"
 
 
   pg-9:
     image: postgres:9.5
     container_name: pg-9
     restart: unless-stopped
-    ports: ["5440:5432"]
     volumes: ["storage-pg-9:/var/lib/postgresql/data"]
     environment:
       POSTGRES_HOST_AUTH_METHOD: "trust"
--- a/NewTests/PostgresUpgrades/test_pg_upgrades.py	Fri Jan 19 09:15:57 2024 +0100
+++ b/NewTests/PostgresUpgrades/test_pg_upgrades.py	Fri Jan 19 15:03:47 2024 +0100
@@ -2,37 +2,13 @@
 import time
 import unittest
 from orthanc_api_client import OrthancApiClient
-from helpers import Helpers
+from helpers import Helpers, wait_container_healthy
 
 import pathlib
 import os
 here = pathlib.Path(__file__).parent.resolve()
 
 
-def get_container_health(container_name):
-    try:
-        # Run the docker inspect command
-        result = subprocess.run(
-            ["docker", "inspect", "--format", "{{.State.Health.Status}}", container_name],
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        
-        # Extract the health status from the command output
-        return result.stdout.strip()
-
-    except subprocess.CalledProcessError as e:
-        print(f"Error checking container health: {e}")
-        return None
-
-def wait_container_healthy(container_name):
-    retry = 0
-
-    while (get_container_health(container_name) != "healthy" and retry < 200):
-        print(f"Waiting for {container_name} to be healty")
-        time.sleep(1)
-
 class TestPgUpgrades(unittest.TestCase):
 
     @classmethod
@@ -48,8 +24,7 @@
 
     @classmethod
     def tearDownClass(cls):
-        pass
-        # cls.cleanup()
+        cls.cleanup()
 
 
     def test_upgrades_downgrades_with_pg_15(self):
--- a/NewTests/helpers.py	Fri Jan 19 09:15:57 2024 +0100
+++ b/NewTests/helpers.py	Fri Jan 19 15:03:47 2024 +0100
@@ -6,6 +6,7 @@
 import typing
 import shutil
 import glob
+import time
 from threading import Thread
 
 
@@ -19,6 +20,32 @@
 }
 
 
+def get_container_health(container_name):
+    try:
+        # Run the docker inspect command
+        result = subprocess.run(
+            ["docker", "inspect", "--format", "{{.State.Health.Status}}", container_name],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        
+        # Extract the health status from the command output
+        return result.stdout.strip()
+
+    except subprocess.CalledProcessError as e:
+        print(f"Error checking container health: {e}")
+        return None
+
+def wait_container_healthy(container_name):
+    retry = 0
+
+    while (get_container_health(container_name) != "healthy" and retry < 200):
+        print(f"Waiting for {container_name} to be healty")
+        time.sleep(1)
+
+
+
 class Helpers:
 
     orthanc_under_tests_hostname: str = 'localhost'
--- a/NewTests/requirements.txt	Fri Jan 19 09:15:57 2024 +0100
+++ b/NewTests/requirements.txt	Fri Jan 19 15:03:47 2024 +0100
@@ -1,3 +1,3 @@
-orthanc-api-client>=0.14.4
+orthanc-api-client>=0.14.6
 orthanc-tools>=0.8.9
 uvicorn
\ No newline at end of file