Mercurial > hg > orthanc-tests

changeset 926:43b07d7e6fd2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
non regression perfs for s3 too
author Alain Mazy <am@orthanc.team>
date Fri, 12 Jun 2026 14:17:35 +0200
parents afc7edf1dff8
children fe40df898a4d
files NewTests/NonRegressionPerfs/docker-compose.yml NewTests/NonRegressionPerfs/test_non_regression_perfs.py NewTests/requirements.txt
diffstat 3 files changed, 230 insertions(+), 43 deletions(-) [+]
line wrap: on
line diff
--- a/NewTests/NonRegressionPerfs/docker-compose.yml	Thu Jun 11 10:24:23 2026 +0200
+++ b/NewTests/NonRegressionPerfs/docker-compose.yml	Fri Jun 12 14:17:35 2026 +0200
@@ -49,8 +49,130 @@
     environment:
       POSTGRES_HOST_AUTH_METHOD: "trust"
 
+
+  orthanc-ref-s3:
+    image: orthancteam/orthanc:26.1.0
+    depends_on: [orthanc-ref-db-s3]
+    restart: unless-stopped
+    ports: ["8242:8042"]
+    environment:
+      VERBOSE_ENABLED: "true"
+      ORTHANC__POSTGRESQL__HOST: "orthanc-ref-db-s3"
+      ORTHANC_JSON: |
+        {
+          "Name": "Orthanc-ref-s3",
+          "AuthenticationEnabled": false,
+          "OverwriteInstances": true,
+          "AwsS3Storage": {
+            "BucketName": "default-bucket",
+            "Region": "eu-west-1",
+            "AccessKey": "minio",
+            "SecretKey": "miniopwd",
+            "Endpoint": "http://toxiproxy-ref-s3:9000",
+            "VirtualAddressing": false
+          }
+        }
+
+  orthanc-new-s3:
+    image: ${ORTHANC_IMAGE_UNDER_TESTS:-orthancteam/orthanc:latest}
+    depends_on: [orthanc-new-db-s3]
+    restart: unless-stopped
+    ports: ["8243:8042"]
+    environment:
+      VERBOSE_ENABLED: "true"
+      ORTHANC__POSTGRESQL__HOST: "orthanc-new-db-s3"
+      ORTHANC_JSON: |
+        {
+          "Name": "Orthanc-under-tests-s3",
+          "AuthenticationEnabled": false,
+          "OverwriteInstances": true,
+          "AwsS3Storage": {
+            "BucketName": "default-bucket",
+            "Region": "eu-west-1",
+            "AccessKey": "minio",
+            "SecretKey": "miniopwd",
+            "Endpoint": "http://toxiproxy-new-s3:9000",
+            "VirtualAddressing": false
+          }
+        }
+
+
+  orthanc-ref-db-s3:
+    image: postgres:18
+    restart: unless-stopped
+    volumes: 
+      - "storage-db-ref-s3:/var/lib/postgresql/18/docker"
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: "trust"
+
+  orthanc-new-db-s3:
+    image: postgres:18
+    restart: unless-stopped
+    volumes: 
+      - "storage-db-new-s3:/var/lib/postgresql/18/docker"
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: "trust"
+
+
+  minio-ref:
+      image: minio/minio:RELEASE.2025-09-07T16-13-09Z
+      # don't expose these ports in a real world setup
+      # ports: [9000:9000, 9001:9001]
+      environment:
+          - MINIO_REGION=eu-west-1
+          - MINIO_ROOT_USER=minio
+          - MINIO_ROOT_PASSWORD=miniopwd
+      volumes:
+          - storage-minio-ref-s3:/data
+      entrypoint: sh
+      command: -c 'mkdir -p /data/default-bucket && /usr/bin/docker-entrypoint.sh server /data --console-address ":9001"'
+
+  minio-new:
+      image: minio/minio:RELEASE.2025-09-07T16-13-09Z
+      # don't expose these ports in a real world setup
+      # ports: [9000:9000, 9001:9001]
+      environment:
+          - MINIO_REGION=eu-west-1
+          - MINIO_ROOT_USER=minio
+          - MINIO_ROOT_PASSWORD=miniopwd
+      volumes:
+          - storage-minio-new-s3:/data
+      entrypoint: sh
+      command: -c 'mkdir -p /data/default-bucket && /usr/bin/docker-entrypoint.sh server /data --console-address ":9001"'
+
+  toxiproxy-ref-s3:
+      depends_on: 
+          - minio-ref
+      # ports: ["19200:9000"]
+      image: ghcr.io/shopify/toxiproxy
+
+  toxiproxy-new-s3:
+      depends_on: 
+          - minio-new
+      # ports: ["19201:9000"]
+      image: ghcr.io/shopify/toxiproxy
+
+
+  toxiproxy-config:
+      image: curlimages/curl
+      depends_on: 
+          - toxiproxy-ref-s3
+          - toxiproxy-new-s3
+      restart: on-failure
+      entrypoint: >
+          sh -c "sleep 3;
+          curl -sf -X POST http://toxiproxy-ref-s3:8474/proxies -H 'Content-Type: application/json' -d '{\"name\":\"minio\",\"listen\":\"0.0.0.0:9000\",\"upstream\":\"minio-ref:9000\"}';
+          curl -sf -X POST http://toxiproxy-ref-s3:8474/proxies/minio/toxics -H 'Content-Type: application/json' -d '{\"type\":\"latency\",\"attributes\":{\"latency\":10,\"jitter\":0}}';
+          curl -sf -X POST http://toxiproxy-new-s3:8474/proxies -H 'Content-Type: application/json' -d '{\"name\":\"minio\",\"listen\":\"0.0.0.0:9000\",\"upstream\":\"minio-new:9000\"}';
+          curl -sf -X POST http://toxiproxy-new-s3:8474/proxies/minio/toxics -H 'Content-Type: application/json' -d '{\"type\":\"latency\",\"attributes\":{\"latency\":10,\"jitter\":0}}';
+          "
+
 volumes:
   storage-orthanc-ref:
   storage-orthanc-new:
   storage-db-ref:
-  storage-db-new:
\ No newline at end of file
+  storage-db-new:
+  storage-minio-ref-s3:
+  storage-minio-new-s3:
+  storage-db-ref-s3:
+  storage-db-new-s3:  
\ No newline at end of file
--- a/NewTests/NonRegressionPerfs/test_non_regression_perfs.py	Thu Jun 11 10:24:23 2026 +0200
+++ b/NewTests/NonRegressionPerfs/test_non_regression_perfs.py	Fri Jun 12 14:17:35 2026 +0200
@@ -15,16 +15,22 @@
 here = pathlib.Path(__file__).parent.resolve()
 
 
-test_configs = {
-    "ref": {
-        "orthanc-url": "http://localhost:8142"
-    },
-    "new": {
-        "orthanc-url": "http://localhost:8143"
-    }
-}
+# test_configs = {
+#     "ref": {
+#         "orthanc-url": "http://localhost:8142"
+#     },
+#     "new": {
+#         "orthanc-url": "http://localhost:8143"
+#     },
+#     "ref-s3": {
+#         "orthanc-url": "http://localhost:8242"
+#     },
+#     "new-s3": {
+#         "orthanc-url": "http://localhost:8243"
+#     }
+# }
 
-test_results = {}
+# test_results = {}
 
 # Download a file localy (only the first time) and return its local_path and content
 def download_test_file(url: str) -> Tuple[str, bytes]:
@@ -49,23 +55,23 @@
         subprocess.run(["docker", "compose", "down", "-v", "--remove-orphans"], check=True)
 
 
-    @classmethod
-    def setUpClass(cls):
-        os.chdir(here)
-        subprocesss_env = os.environ.copy()
-        subprocesss_env["ORTHANC_IMAGE_UNDER_TESTS"] = Helpers.orthanc_under_tests_docker_image
+    # @classmethod
+    # def setUpClass(cls):
+    #     os.chdir(here)
+    #     subprocesss_env = os.environ.copy()
+    #     subprocesss_env["ORTHANC_IMAGE_UNDER_TESTS"] = Helpers.orthanc_under_tests_docker_image
 
-        # print("Pullling containers")
-        # subprocess.run(["docker", "compose", "pull"], env=subprocesss_env, check=True)
+    #     # print("Pullling containers")
+    #     # subprocess.run(["docker", "compose", "pull"], env=subprocesss_env, check=True)
 
-        print("Launching containers")
-        subprocess.run(["docker", "compose", "up", "-d"], env=subprocesss_env, check=True)
+    #     print("Launching containers")
+    #     subprocess.run(["docker", "compose", "up", "-d"], env=subprocesss_env, check=True)
         
-        o_ref = OrthancApiClient(test_configs["ref"]["orthanc-url"])
-        o_new = OrthancApiClient(test_configs["new"]["orthanc-url"])
+    #     o_ref = OrthancApiClient(test_configs["ref"]["orthanc-url"])
+    #     o_new = OrthancApiClient(test_configs["new"]["orthanc-url"])
 
-        o_ref.wait_started()
-        o_new.wait_started()
+    #     o_ref.wait_started()
+    #     o_new.wait_started()
 
 
     @classmethod
@@ -73,9 +79,7 @@
         cls.cleanup()
 
 
-    def measure(self, test_name: str, perform_test: Callable[[OrthancApiClient], None], reapeat_count: int = 1, tolerance_pct = 0.25) -> None:
-        global test_configs
-        global test_results
+    def measure(self, test_name: str, perform_test: Callable[[OrthancApiClient], None], test_configs, test_results, reapeat_count: int = 1, tolerance_pct = 0.25) -> None:
 
         test_results[test_name] = {}
 
@@ -108,39 +112,100 @@
         test_results[test_name]["success"] = not failed
 
 
-    def test_non_regression(self):
-        print("Launching tests")
+    def test_non_regression_s3(self):
+        print("Launching tests (s3)")
+
+        test_configs = {
+            "ref": {
+                "orthanc-url": "http://localhost:8242"
+            },
+            "new": {
+                "orthanc-url": "http://localhost:8243"
+            }
+        }
+        test_results = {}
+        self.compare(config_name='s3'
+                     test_configs=test_configs,
+                     test_results=test_results)
+
+
+    def test_non_regression_classic(self):
+        print("Launching tests (classic)")
 
-        print("")
+        test_configs = {
+            "ref": {
+                "orthanc-url": "http://localhost:8142"
+            },
+            "new": {
+                "orthanc-url": "http://localhost:8143"
+            }
+        }
+        test_results = {}
+        self.compare(config_name='file-system'
+                     test_configs=test_configs,
+                     test_results=test_results)
+
+
+    def compare(self, config_name, test_configs, test_results):
+
+        os.chdir(here)
+        subprocesss_env = os.environ.copy()
+        subprocesss_env["ORTHANC_IMAGE_UNDER_TESTS"] = Helpers.orthanc_under_tests_docker_image
+
+        # print("Pullling containers")
+        # subprocess.run(["docker", "compose", "pull"], env=subprocesss_env, check=True)
+
+        print("Launching containers")
+        subprocess.run(["docker", "compose", "up", "-d"], env=subprocesss_env, check=True)
+        
+        o_ref = OrthancApiClient(test_configs["ref"]["orthanc-url"])
+        o_new = OrthancApiClient(test_configs["new"]["orthanc-url"])
+
+        o_ref.wait_started()
+        o_new.wait_started()
+
+        print(f"---------- {config_name} -----------")
         print(f"{'TEST NAME':<50} | {'REF ORTHANC [ms]':>20} | {'NEW ORTHANC [ms]':>20} | {'DELTA [PCT]':>20}")
         print(f"{'-'*119}")
 
-        self.measure(test_name="populate with 5 workers",
+        self.measure(test_name="populate 3000 instances with 5 workers",
                      perform_test=lambda o: OrthancTestDbPopulator(o, studies_count=5, series_count=3, instances_count=120, random_seed=65, worker_threads_count=5).execute(),
-                     reapeat_count=1)
+                     reapeat_count=1,
+                     test_configs=test_configs,
+                     test_results=test_results)
 
-        self.measure(test_name="studies statistics",
+        self.measure(test_name="studies statistics 5x5",
                      perform_test=lambda o: [o.studies.get_json_statistics(i) for i in o.studies.get_all_ids()],
-                     reapeat_count=5)
-
+                     reapeat_count=5,
+                     test_configs=test_configs,
+                     test_results=test_results)
+        
         reg_of_path, reg_of_content = download_test_file("https://public-files.orthanc.team/test-files/429_MB_REG_OF.dcm")
         reg_ow_path, reg_ow_content = download_test_file("https://public-files.orthanc.team/test-files/429_MB_REG_OW.dcm")
 
         self.measure(test_name="upload large Reg file with OW VectorGridData",
                      perform_test=lambda o: o.upload(reg_ow_content),
-                     reapeat_count=1)
-
+                     reapeat_count=1,
+                     test_configs=test_configs,
+                     test_results=test_results)
+        
         self.measure(test_name="upload large Reg file with OF VectorGridData",
                      perform_test=lambda o: o.upload(reg_of_content),
-                     reapeat_count=1)
+                     reapeat_count=1,
+                     test_configs=test_configs,
+                     test_results=test_results)
 
-        self.measure(test_name="upload same file",
+        self.measure(test_name="upload same file 50x",
                      perform_test=lambda o: o.upload_file(here / "../../Database/Knee/T1/IM-0001-0001.dcm"),
-                     reapeat_count=50)
+                     reapeat_count=50,
+                     test_configs=test_configs,
+                     test_results=test_results)
 
-        self.measure(test_name="upload same file SEG",
+        self.measure(test_name="upload same file SEG 50x",
                      perform_test=lambda o: o.upload_file(here / "../../Database/DicomSeg.dcm"),
-                     reapeat_count=50)
+                     reapeat_count=50,
+                     test_configs=test_configs,
+                     test_results=test_results)
 
         print("Stopping containers")
         subprocess.run(["docker", "compose", "down"], check=True)
--- a/NewTests/requirements.txt	Thu Jun 11 10:24:23 2026 +0200
+++ b/NewTests/requirements.txt	Fri Jun 12 14:17:35 2026 +0200
@@ -1,4 +1,4 @@
-orthanc-api-client>=0.24.0
-orthanc-tools>=0.16.5
+orthanc-api-client>=0.24.1
+orthanc-tools>=0.18.2
 uvicorn
 fastapi
\ No newline at end of file