Mercurial > hg > orthanc-tests

annotate NewTests/Concurrency/test_transfer.py @ 622:75dbc81d0e26

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix concurrency transfer tests
author Alain Mazy <am@osimis.io>
date Wed, 07 Feb 2024 09:32:39 +0100
parents a5882a40ccb6
children 76c9923050b5
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
601
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
1 import subprocess
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
2 import time
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
3 import unittest
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
4 from orthanc_api_client import OrthancApiClient, ResourceType
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
5 from orthanc_tools import OrthancTestDbPopulator
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
6 from helpers import Helpers, wait_container_healthy
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
7
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
8 import pathlib
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
9 import os
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
10 here = pathlib.Path(__file__).parent.resolve()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
11
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
12
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
13
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
14
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
15 class TestConcurrencyTransfers(unittest.TestCase):
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
16
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
17 @classmethod
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
18 def cleanup(cls):
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
19 os.chdir(here)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
20 print("Cleaning old compose")
616
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
21 subprocess.run(["docker", "compose", "-f", "docker-compose-transfers-concurrency.yml", "down", "-v", "--remove-orphans"],
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
22 env= {
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
23 "ORTHANC_IMAGE_UNDER_TESTS": Helpers.orthanc_under_tests_docker_image
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
24 },
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
25 check=True)
601
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
26
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
27 @classmethod
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
28 def compose_up(cls):
616
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
29 # print("Pullling containers")
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
30 # subprocess.run(["docker", "compose", "-f", "docker-compose-transfers-concurrency.yml", "pull"],
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
31 # env= {
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
32 # "ORTHANC_IMAGE_UNDER_TESTS": Helpers.orthanc_under_tests_docker_image,
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
33 # "PATH": os.environ.get('PATH')
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
34 # },
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
35 # check=True)
601
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
36
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
37 print("Compose up")
616
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
38 subprocess.run(["docker", "compose", "-f", "docker-compose-transfers-concurrency.yml", "up", "-d"],
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
39 env= {
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
40 "ORTHANC_IMAGE_UNDER_TESTS": Helpers.orthanc_under_tests_docker_image
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
41 },
a5882a40ccb6 rename osimis/orthanc to orthancteam/orthanc
Alain Mazy <am@osimis.io>
parents: 602
diff changeset
42 check=True)
601
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
43
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
44 @classmethod
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
45 def setUpClass(cls):
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
46 cls.cleanup()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
47 cls.compose_up()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
48
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
49 @classmethod
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
50 def tearDownClass(cls):
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
51 #cls.cleanup()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
52 pass
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
53
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
54 def clean_start(self):
602
d88b0fc15f08 change ports
Alain Mazy <am@osimis.io>
parents: 601
diff changeset
55 oa = OrthancApiClient("http://localhost:8062")
d88b0fc15f08 change ports
Alain Mazy <am@osimis.io>
parents: 601
diff changeset
56 ob = OrthancApiClient("http://localhost:8063")
601
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
57
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
58 oa.wait_started()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
59 ob.wait_started()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
60
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
61 oa.delete_all_content()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
62 ob.delete_all_content()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
63
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
64 return oa, ob
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
65
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
66 def test_push(self):
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
67 oa, ob = self.clean_start()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
68
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
69 populator = OrthancTestDbPopulator(oa, studies_count=5, random_seed=65)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
70 populator.execute()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
71
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
72 all_studies_ids = oa.studies.get_all_ids()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
73 instances_count = oa.get_statistics().instances_count
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
74 disk_size = oa.get_statistics().total_disk_size
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
75 repeat_count = 2
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
76
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
77 for compression in [True, False]:
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
78 start_time = time.time()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
79
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
80 for i in range(0, repeat_count):
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
81 oa.transfers.send(target_peer='b',
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
82 resources_ids=all_studies_ids,
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
83 resource_type=ResourceType.STUDY,
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
84 compress=compression)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
85
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
86 self.assertEqual(instances_count, ob.get_statistics().instances_count)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
87 self.assertEqual(disk_size, ob.get_statistics().total_disk_size)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
88 ob.delete_all_content()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
89
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
90 elapsed = time.time() - start_time
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
91 print(f"TIMING test_push (compression={compression}) with {instances_count} instances for a total of {disk_size/(1024*1024)} MB (repeat {repeat_count}x): {elapsed:.3f} s")
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
92
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
93
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
94 def test_pull(self):
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
95 oa, ob = self.clean_start()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
96
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
97 populator = OrthancTestDbPopulator(ob, studies_count=5, random_seed=65)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
98 populator.execute()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
99
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
100 all_studies_ids = ob.studies.get_all_ids()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
101 instances_count = ob.get_statistics().instances_count
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
102 disk_size = ob.get_statistics().total_disk_size
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
103 repeat_count = 2
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
104
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
105 for compression in [True, False]:
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
106 start_time = time.time()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
107
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
108 for i in range(0, repeat_count):
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
109 remote_job = ob.transfers.send_async(target_peer='a',
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
110 resources_ids=all_studies_ids,
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
111 resource_type=ResourceType.STUDY,
622
75dbc81d0e26 fix concurrency transfer tests
Alain Mazy <am@osimis.io>
parents: 616
diff changeset
112 compress=compression)
601
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
113 job = oa.jobs.get(orthanc_id=remote_job.remote_job_id)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
114 job.wait_completed(polling_interval=0.1)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
115
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
116 self.assertEqual(instances_count, oa.get_statistics().instances_count)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
117 self.assertEqual(disk_size, oa.get_statistics().total_disk_size)
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
118 oa.delete_all_content()
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
119
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
120
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
121 elapsed = time.time() - start_time
3e15e950c462 new transfer tests in the concurrency section
Alain Mazy <am@osimis.io>
parents:
diff changeset
122 print(f"TIMING test_pull (compression={compression}) with {instances_count} instances for a total of {disk_size/(1024*1024)} MB (repeat {repeat_count}x): {elapsed:.3f} s")