annotate NewTests/MaxStorage/test_max_storage_pg.py @ 624:543e372d2265

added a PG max storage test
author Alain Mazy <am@osimis.io>
date Wed, 14 Feb 2024 11:25:40 +0100
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
624
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
1 import unittest
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
2 import time
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
3 import os
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
4 import threading
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
5 from helpers import OrthancTestCase, Helpers
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
6
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
7 from orthanc_api_client import OrthancApiClient, ChangeType
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
8 from orthanc_api_client.exceptions import HttpError
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
9 from orthanc_api_client import helpers as OrthancHelpers
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
10
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
11 from orthanc_tools import OrthancTestDbPopulator
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
12
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
13 import pathlib
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
14 import subprocess
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
15 import glob
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
16 here = pathlib.Path(__file__).parent.resolve()
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
17
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
18
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
19 class TestMaxStoragePG(OrthancTestCase):
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
20
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
21 @classmethod
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
22 def terminate(cls):
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
23
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
24 if Helpers.is_docker():
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
25 subprocess.run(["docker", "rm", "-f", "pg-server"])
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
26 else:
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
27 cls.pg_service_process.terminate()
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
28
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
29
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
30 @classmethod
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
31 def prepare(cls):
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
32 test_name = "MaxStoragePG"
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
33 cls._storage_name = "max-storage-pg"
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
34 network_name = "max-storage-pg"
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
35
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
36 print(f'-------------- preparing {test_name} tests')
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
37
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
38 cls.clear_storage(storage_name=cls._storage_name)
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
39
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
40 pg_hostname = "localhost"
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
41 if Helpers.is_docker():
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
42 pg_hostname = "pg-server"
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
43 cls.create_docker_network(network_name)
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
44
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
45 config = {
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
46 "PostgreSQL" : {
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
47 "EnableStorage": False,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
48 "EnableIndex": True,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
49 "Host": pg_hostname,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
50 "Port": 5432,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
51 "Database": "postgres",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
52 "Username": "postgres",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
53 "Password": "postgres",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
54 "IndexConnectionsCount": 10,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
55 "MaximumConnectionRetries" : 2000,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
56 "ConnectionRetryInterval" : 5,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
57 "TransactionMode": "ReadCommitted",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
58 #"TransactionMode": "Serializable",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
59 "EnableVerboseLogs": True
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
60 },
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
61 "AuthenticationEnabled": False,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
62 "OverwriteInstances": True,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
63 "MaximumStorageSize": 1,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
64 "MaximumStorageMode": "Recycle"
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
65 # "MaximumPatientCount": 1,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
66 # "MaximumStorageMode": "Reject"
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
67 }
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
68
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
69 config_path = cls.generate_configuration(
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
70 config_name=f"{test_name}",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
71 storage_name=cls._storage_name,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
72 config=config,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
73 plugins=Helpers.plugins
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
74 )
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
75
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
76 # launch the docker PG server
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
77 print('--------------- launching PostgreSQL server ------------------')
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
78
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
79 pg_cmd = [
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
80 "docker", "run", "--rm",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
81 "-p", "5432:5432",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
82 "--name", "pg-server",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
83 "--env", "POSTGRES_HOST_AUTH_METHOD=trust"
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
84 ]
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
85
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
86 if Helpers.is_docker():
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
87 pg_cmd.extend(["--network", network_name])
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
88 pg_cmd.append("postgres:15")
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
89
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
90 cls.pg_service_process = subprocess.Popen(pg_cmd)
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
91 time.sleep(5)
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
92
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
93 if Helpers.break_after_preparation:
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
94 print(f"++++ It is now time to start your Orthanc under tests with configuration file '{config_path}' +++++")
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
95 input("Press Enter to continue")
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
96 else:
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
97 cls.launch_orthanc_under_tests(
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
98 config_name=f"{test_name}",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
99 storage_name=cls._storage_name,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
100 config=config,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
101 plugins=Helpers.plugins,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
102 docker_network=network_name
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
103 )
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
104
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
105 cls.o = OrthancApiClient(cls.o._root_url)
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
106 cls.o.wait_started()
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
107 cls.o.delete_all_content()
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
108
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
109
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
110 def test_upload(self):
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
111 self.o.delete_all_content()
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
112 self.clear_storage(storage_name=self._storage_name)
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
113
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
114 uploaded_instances_ids = []
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
115 counter = 0
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
116 # upload 10 images of 500x500, since the MaximumStorageSize is 1MB, only 2 of them should remain in the storage
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
117 for i in range(0, 10):
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
118 counter += 1
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
119 dicom_file = OrthancHelpers.generate_test_dicom_file(width=500, height=500,
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
120 tags = {
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
121 "PatientID" : f"{i}",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
122 "StudyInstanceUID" : f"{i}",
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
123 "SeriesInstanceUID" : f"{i}.{counter%10}"
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
124 })
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
125 try:
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
126 uploaded_instances_ids.extend(self.o.upload(dicom_file))
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
127 except HttpError as er:
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
128 if er.http_status_code == 507:
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
129 pass # ignore
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
130
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
131 # some instances have been discarded
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
132 self.assertLess(len(self.o.instances.get_all_ids()), 10)
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
133 self.assertLess(len(self.o.patients.get_all_ids()), 10)
543e372d2265 added a PG max storage test
Alain Mazy <am@osimis.io>
parents:
diff changeset
134 self.assertLess(self.o.get_statistics().total_disk_size, 1*1024*1024)