Mercurial > hg > orthanc-tests
annotate NewTests/Concurrency/test_transfer.py @ 622:75dbc81d0e26
fix concurrency transfer tests
author | Alain Mazy <am@osimis.io> |
---|---|
date | Wed, 07 Feb 2024 09:32:39 +0100 |
parents | a5882a40ccb6 |
children | 76c9923050b5 |
rev | line source |
---|---|
601
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
1 import subprocess |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
2 import time |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
3 import unittest |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
4 from orthanc_api_client import OrthancApiClient, ResourceType |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
5 from orthanc_tools import OrthancTestDbPopulator |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
6 from helpers import Helpers, wait_container_healthy |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
7 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
8 import pathlib |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
9 import os |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
10 here = pathlib.Path(__file__).parent.resolve() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
11 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
12 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
13 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
14 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
15 class TestConcurrencyTransfers(unittest.TestCase): |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
16 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
17 @classmethod |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
18 def cleanup(cls): |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
19 os.chdir(here) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
20 print("Cleaning old compose") |
616
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
21 subprocess.run(["docker", "compose", "-f", "docker-compose-transfers-concurrency.yml", "down", "-v", "--remove-orphans"], |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
22 env= { |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
23 "ORTHANC_IMAGE_UNDER_TESTS": Helpers.orthanc_under_tests_docker_image |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
24 }, |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
25 check=True) |
601
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
26 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
27 @classmethod |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
28 def compose_up(cls): |
616
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
29 # print("Pullling containers") |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
30 # subprocess.run(["docker", "compose", "-f", "docker-compose-transfers-concurrency.yml", "pull"], |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
31 # env= { |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
32 # "ORTHANC_IMAGE_UNDER_TESTS": Helpers.orthanc_under_tests_docker_image, |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
33 # "PATH": os.environ.get('PATH') |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
34 # }, |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
35 # check=True) |
601
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
36 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
37 print("Compose up") |
616
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
38 subprocess.run(["docker", "compose", "-f", "docker-compose-transfers-concurrency.yml", "up", "-d"], |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
39 env= { |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
40 "ORTHANC_IMAGE_UNDER_TESTS": Helpers.orthanc_under_tests_docker_image |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
41 }, |
a5882a40ccb6
rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents:
602
diff
changeset
|
42 check=True) |
601
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
43 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
44 @classmethod |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
45 def setUpClass(cls): |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
46 cls.cleanup() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
47 cls.compose_up() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
48 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
49 @classmethod |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
50 def tearDownClass(cls): |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
51 #cls.cleanup() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
52 pass |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
53 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
54 def clean_start(self): |
602 | 55 oa = OrthancApiClient("http://localhost:8062") |
56 ob = OrthancApiClient("http://localhost:8063") | |
601
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
57 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
58 oa.wait_started() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
59 ob.wait_started() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
60 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
61 oa.delete_all_content() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
62 ob.delete_all_content() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
63 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
64 return oa, ob |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
65 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
66 def test_push(self): |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
67 oa, ob = self.clean_start() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
68 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
69 populator = OrthancTestDbPopulator(oa, studies_count=5, random_seed=65) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
70 populator.execute() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
71 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
72 all_studies_ids = oa.studies.get_all_ids() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
73 instances_count = oa.get_statistics().instances_count |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
74 disk_size = oa.get_statistics().total_disk_size |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
75 repeat_count = 2 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
76 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
77 for compression in [True, False]: |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
78 start_time = time.time() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
79 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
80 for i in range(0, repeat_count): |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
81 oa.transfers.send(target_peer='b', |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
82 resources_ids=all_studies_ids, |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
83 resource_type=ResourceType.STUDY, |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
84 compress=compression) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
85 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
86 self.assertEqual(instances_count, ob.get_statistics().instances_count) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
87 self.assertEqual(disk_size, ob.get_statistics().total_disk_size) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
88 ob.delete_all_content() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
89 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
90 elapsed = time.time() - start_time |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
91 print(f"TIMING test_push (compression={compression}) with {instances_count} instances for a total of {disk_size/(1024*1024)} MB (repeat {repeat_count}x): {elapsed:.3f} s") |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
92 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
93 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
94 def test_pull(self): |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
95 oa, ob = self.clean_start() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
96 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
97 populator = OrthancTestDbPopulator(ob, studies_count=5, random_seed=65) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
98 populator.execute() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
99 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
100 all_studies_ids = ob.studies.get_all_ids() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
101 instances_count = ob.get_statistics().instances_count |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
102 disk_size = ob.get_statistics().total_disk_size |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
103 repeat_count = 2 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
104 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
105 for compression in [True, False]: |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
106 start_time = time.time() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
107 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
108 for i in range(0, repeat_count): |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
109 remote_job = ob.transfers.send_async(target_peer='a', |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
110 resources_ids=all_studies_ids, |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
111 resource_type=ResourceType.STUDY, |
622 | 112 compress=compression) |
601
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
113 job = oa.jobs.get(orthanc_id=remote_job.remote_job_id) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
114 job.wait_completed(polling_interval=0.1) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
115 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
116 self.assertEqual(instances_count, oa.get_statistics().instances_count) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
117 self.assertEqual(disk_size, oa.get_statistics().total_disk_size) |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
118 oa.delete_all_content() |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
119 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
120 |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
121 elapsed = time.time() - start_time |
3e15e950c462
new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff
changeset
|
122 print(f"TIMING test_pull (compression={compression}) with {instances_count} instances for a total of {disk_size/(1024*1024)} MB (repeat {repeat_count}x): {elapsed:.3f} s") |