Mercurial > hg > orthanc-tests
changeset 920:a2051215a485
added new NonRegressionPerfs tests
| author | Alain Mazy <am@orthanc.team> |
|---|---|
| date | Tue, 02 Jun 2026 22:21:24 +0200 |
| parents | e5aa6a487f26 |
| children | 6b38f55114c4 |
| files | .hgignore NewTests/NonRegressionPerfs/DownloadedTestFiles/.keep NewTests/NonRegressionPerfs/__init__.py NewTests/NonRegressionPerfs/docker-compose.yml NewTests/NonRegressionPerfs/test_non_regression_perfs.py NewTests/README |
| diffstat | 5 files changed, 223 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Tue Jun 02 21:57:40 2026 +0200 +++ b/.hgignore Tue Jun 02 22:21:24 2026 +0200 @@ -10,6 +10,7 @@ *~ NewTests/storages/** NewTests/configurations/*.json +NewTests/NonRegressionPerfs/DownloadedTestFiles/*.dcm .env/ Tests/*.crt Tests/*.key
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/NewTests/NonRegressionPerfs/DownloadedTestFiles/.keep Tue Jun 02 22:21:24 2026 +0200 @@ -0,0 +1,1 @@ +dummy file to create the directory \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/NewTests/NonRegressionPerfs/docker-compose.yml Tue Jun 02 22:21:24 2026 +0200 @@ -0,0 +1,56 @@ +services: + + orthanc-ref: + image: orthancteam/orthanc:26.1.0 + depends_on: [orthanc-ref-db] + restart: unless-stopped + ports: ["8142:8042"] + volumes: ["storage-orthanc-ref:/var/lib/orthanc/db"] + environment: + VERBOSE_ENABLED: "true" + ORTHANC__POSTGRESQL__HOST: "orthanc-ref-db" + ORTHANC_JSON: | + { + "Name": "Orthanc-ref", + "AuthenticationEnabled": false, + "OverwriteInstances": true + } + + orthanc-new: + image: ${ORTHANC_IMAGE_UNDER_TESTS:-orthancteam/orthanc:latest} + depends_on: [orthanc-new-db] + restart: unless-stopped + ports: ["8143:8042"] + volumes: ["storage-orthanc-new:/var/lib/orthanc/db"] + environment: + VERBOSE_ENABLED: "true" + ORTHANC__POSTGRESQL__HOST: "orthanc-new-db" + ORTHANC_JSON: | + { + "Name": "Orthanc-under-tests", + "AuthenticationEnabled": false, + "OverwriteInstances": true + } + + + orthanc-ref-db: + image: postgres:18 + restart: unless-stopped + volumes: + - "storage-db-ref:/var/lib/postgresql/18/docker" + environment: + POSTGRES_HOST_AUTH_METHOD: "trust" + + orthanc-new-db: + image: postgres:18 + restart: unless-stopped + volumes: + - "storage-db-new:/var/lib/postgresql/18/docker" + environment: + POSTGRES_HOST_AUTH_METHOD: "trust" + +volumes: + storage-orthanc-ref: + storage-orthanc-new: + storage-db-ref: + storage-db-new: \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/NewTests/NonRegressionPerfs/test_non_regression_perfs.py Tue Jun 02 22:21:24 2026 +0200 @@ -0,0 +1,154 @@ +import subprocess +import time +import unittest +from orthanc_api_client import OrthancApiClient +from orthanc_tools import OrthancTestDbPopulator +from helpers import Helpers, wait_container_healthy +from contextlib import contextmanager +from typing import Callable, Tuple +from urllib.parse import urlparse +import shutil + +import pathlib +import os +import requests +here = pathlib.Path(__file__).parent.resolve() + + +test_configs = { + "ref": { + "orthanc-url": "http://localhost:8142" + }, + "new": { + "orthanc-url": "http://localhost:8143" + } +} + +test_results = {} + +# Download a file localy (only the first time) and return its local_path and content +def download_test_file(url: str) -> Tuple[str, bytes]: + local_path = here / "DownloadedTestFiles" / urlparse(url).path.split('/')[-1] + if not os.path.exists(local_path): + http_stream = requests.get(url, stream=True) + with open(local_path, 'wb') as file_stream: + shutil.copyfileobj(http_stream.raw, file_stream) + + with open(local_path, 'rb') as file_stream: + content = file_stream.read() + + return local_path, content + + +class TestNonRegressionPerfs(unittest.TestCase): + + @classmethod + def cleanup(cls): + os.chdir(here) + print("Cleaning old compose") + subprocess.run(["docker", "compose", "down", "-v", "--remove-orphans"], check=True) + + + @classmethod + def setUpClass(cls): + os.chdir(here) + subprocesss_env = os.environ.copy() + subprocesss_env["ORTHANC_IMAGE_UNDER_TESTS"] = Helpers.orthanc_under_tests_docker_image + + # print("Pullling containers") + # subprocess.run(["docker", "compose", "pull"], env=subprocesss_env, check=True) + + print("Launching containers") + subprocess.run(["docker", "compose", "up", "-d"], env=subprocesss_env, check=True) + + o_ref = OrthancApiClient(test_configs["ref"]["orthanc-url"]) + o_new = OrthancApiClient(test_configs["new"]["orthanc-url"]) + + o_ref.wait_started() + o_new.wait_started() + + + @classmethod + def tearDownClass(cls): + cls.cleanup() + + + def measure(self, test_name: str, perform_test: Callable[[OrthancApiClient], None], reapeat_count: int = 1, tolerance_pct = 0.25) -> None: + global test_configs + global test_results + + test_results[test_name] = {} + + for (test_config_name, test_config) in test_configs.items(): + o = OrthancApiClient(orthanc_root_url=test_config["orthanc-url"]) + + start_time = time.perf_counter() + for i in range(0, reapeat_count): + perform_test(o) + + end_time = time.perf_counter() + test_results[test_name][test_config_name] = (end_time - start_time) * 1000 + + ref_time = test_results[test_name]["ref"] + new_time = test_results[test_name]["new"] + + delta_perf = new_time / ref_time - 1 + + if (delta_perf) > tolerance_pct: + delta_text = f"+ {(delta_perf*100):>8.1f} % (FAILED)" + failed = True + elif (delta_perf) > 0: + delta_text = f"+ {(delta_perf*100):>8.1f} %" + failed = False + else: + delta_text = f"- {abs((delta_perf*100)):>8.1f} %" + failed = False + + print(f"{test_name:<50} | {ref_time:>20.3f} | {new_time:>20.3f} | {delta_text:>20}") + test_results[test_name]["success"] = not failed + + + def test_non_regression(self): + print("Launching tests") + + print("") + print(f"{'TEST NAME':<50} | {'REF ORTHANC [ms]':>20} | {'NEW ORTHANC [ms]':>20} | {'DELTA [PCT]':>20}") + print(f"{'-'*119}") + + self.measure(test_name="populate with 5 workers", + perform_test=lambda o: OrthancTestDbPopulator(o, studies_count=5, series_count=3, instances_count=120, random_seed=65, worker_threads_count=5).execute(), + reapeat_count=1) + + self.measure(test_name="studies statistics", + perform_test=lambda o: [o.studies.get_json_statistics(i) for i in o.studies.get_all_ids()], + reapeat_count=5) + + reg_of_path, reg_of_content = download_test_file("https://public-files.orthanc.team/test-files/429_MB_REG_OF.dcm") + reg_ow_path, reg_ow_content = download_test_file("https://public-files.orthanc.team/test-files/429_MB_REG_OW.dcm") + + self.measure(test_name="upload large Reg file with OW VectorGridData", + perform_test=lambda o: o.upload(reg_ow_content), + reapeat_count=1) + + self.measure(test_name="upload large Reg file with OF VectorGridData", + perform_test=lambda o: o.upload(reg_of_content), + reapeat_count=1) + + self.measure(test_name="upload same file", + perform_test=lambda o: o.upload_file(here / "../../Database/Knee/T1/IM-0001-0001.dcm"), + reapeat_count=50) + + self.measure(test_name="upload same file SEG", + perform_test=lambda o: o.upload_file(here / "../../Database/DicomSeg.dcm"), + reapeat_count=50) + + print("Stopping containers") + subprocess.run(["docker", "compose", "down"], check=True) + time.sleep(2) + + regressionsCount = 0 + for (test_name, test_result) in test_results.items(): + if not test_result["success"]: + regressionsCount += 1 + + self.assertEqual(0, regressionsCount) # check the regressions in the report above \ No newline at end of file
--- a/NewTests/README Tue Jun 02 21:57:40 2026 +0200 +++ b/NewTests/README Tue Jun 02 22:21:24 2026 +0200 @@ -283,3 +283,14 @@ python3 NewTests/main.py --pattern=InterruptedDownloads.test_interrupted_downloads.TestInterruptedDownloads.* \ --orthanc_under_tests_docker_image=orthancteam/orthanc-pre-release:master-unstable \ --orthanc_under_tests_http_port=8043 + + +NonRegressionPerfs: +------------------ + +with Docker: + +python3 NewTests/main.py --pattern=NonRegressionPerfs.test_non_regression_perfs.TestNonRegressionPerfs.* \ + --orthanc_under_tests_docker_image=orthancteam/orthanc-pre-release:master-unstable \ + --orthanc_under_tests_http_port=8043 +
