# HG changeset patch # User am@osimis.io # Date 1534432412 -7200 # Node ID f1a75985caa8969572781c45351e7be51ad045e5 # Parent e0996602b306bcf87b936666fbf13118e7d8be5e first Db test framework - work in progress diff -r e0996602b306 -r f1a75985caa8 .hgignore --- a/.hgignore Tue Aug 14 10:16:25 2018 +0200 +++ b/.hgignore Thu Aug 16 17:13:32 2018 +0200 @@ -1,4 +1,8 @@ syntax: glob *.pyc OrthancStorage/ -IntegrationTestsConfiguration.json \ No newline at end of file +IntegrationTestsConfiguration.json +PerfsDb/.env/ +PerfsDb/Storages/ +PerfsDb/ConfigFiles/ +.vscode/ \ No newline at end of file diff -r e0996602b306 -r f1a75985caa8 PerfsDb/ConfigFileBuilder.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/ConfigFileBuilder.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,57 @@ +import typing +import json + +from DbType import DbType + +class ConfigFileBuilder: + + @staticmethod + def generate( + outputPath: str, + plugins: typing.List[str], + storagePath: str, + dbType: DbType, + dbSize: str, + port: int + ): + + config = {} + config["Plugins"] = plugins + config["StorageDirectory"] = storagePath + + dbConfig = {} + dbConfig["EnableIndex"] = True + dbConfig["Host"] = "127.0.0.1" + dbConfig["Lock"] = False + dbConfig["Port"] = port + + if dbType == DbType.MySQL: + dbConfig["EnableStorage"] = False + dbConfig["Database"] = "orthanc" + dbConfig["Username"] = "orthanc" + dbConfig["Password"] = "orthanc" + + config["MySQL"] = dbConfig + + elif dbType == DbType.PG9 or dbType == DbType.PG10: + dbConfig["EnableStorage"] = False + dbConfig["Database"] = "orthanc" + dbConfig["Username"] = "orthanc" + dbConfig["Password"] = "orthanc" + + config["PostgreSQL"] = dbConfig + + elif dbType == DbType.MSSQL: + dbConfig["ConnectionString"] = "Driver={ODBC Driver 13 for SQL Server};Server=tcp:index," + port + ";Database=master;Uid=sa;Pwd=MyStrOngPa55word!;Encrypt=yes;TrustServerCertificate=yes;Connection Timeout=30" + dbConfig["LicenseString"] = "1abaamBcReVXv6EtE_X___demo-orthanc%osimis.io___HHHnqVHYvEkR3jGs2Y3EvpbxZgTt7yaCniJa2Bz7hFWTMa" # note: this is a trial license expiring on 2018-09-30, replace with your license code + + config["MSSQL"] = dbConfig + + elif DbType == DbType.Sqlite: + config["IndexDirectory"] = storagePath + + else: + raise NotImplementedError + + with open(outputPath, "w") as configFile: + json.dump(config, fp=configFile, indent=4) \ No newline at end of file diff -r e0996602b306 -r f1a75985caa8 PerfsDb/DbPopulator.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/DbPopulator.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,76 @@ +import typing +from orthancRestApi import OrthancClient + +from DbSize import DbSize + +class DbPopulator: + + def __init__(self, orthanc: OrthancClient, dbSize: DbSize): + self._orthanc = orthanc + self._dbSize = dbSize + self._sourceInstanceId = None + + def populate(self): + self._sourceInstanceId = self._orthanc.uploadDicomFile("../Database/DummyCT.dcm") + + if self._dbSize == DbSize.Small: + patientCount = 3 + smallStudiesPerPatient = 2 + largeStudiesPerPatient = 1 + else: + patientCount = 100 + smallStudiesPerPatient = 4 + largeStudiesPerPatient = 8 + + # data that are the same in small and large DBs (and that can be used in tests for comparing the same things !!) + + # used in TestFindStudyByPatientId5Results + self.createStudy(studyIndex=99994, patientIndex=99998, seriesCount=1, instancesPerSeries=1) + self.createStudy(studyIndex=99995, patientIndex=99998, seriesCount=1, instancesPerSeries=1) + self.createStudy(studyIndex=99996, patientIndex=99998, seriesCount=1, instancesPerSeries=1) + self.createStudy(studyIndex=99997, patientIndex=99998, seriesCount=1, instancesPerSeries=1) + self.createStudy(studyIndex=99998, patientIndex=99998, seriesCount=1, instancesPerSeries=1) + + # used in TestFindStudyByStudyDescription1Result + # used in TestFindStudyByPatientId1Result + self.createStudy(studyIndex=99999, patientIndex=99999, seriesCount=1, instancesPerSeries=1) + + # data to make the DB "large" or "small" + for patientIndex in range(0, patientCount): + studyIndex=0 + print("Generating data for patient " + str(patientIndex)) + for i in range(0, smallStudiesPerPatient): + print("Generating small study " + str(i)) + self.createStudy(studyIndex=studyIndex, patientIndex=patientIndex, seriesCount=2, instancesPerSeries=2) + studyIndex+=1 + for i in range(0, largeStudiesPerPatient): + print("Generating large study " + str(i)) + self.createStudy(studyIndex=studyIndex, patientIndex=patientIndex, seriesCount=4, instancesPerSeries=500) + studyIndex+=1 + + + + print("Generation completed") + + def createStudy(self, studyIndex: int, patientIndex: int, seriesCount: int, instancesPerSeries: int): + for seriesIndex in range(0, seriesCount): + for instanceIndex in range(0, instancesPerSeries): + dicomFile = self.createDicomFile(patientIndex=patientIndex, studyIndex=studyIndex, seriesIndex=seriesIndex, instanceIndex=instanceIndex) + self._orthanc.uploadDicom(dicomFile) + + def createDicomFile(self, patientIndex: int, studyIndex: int, seriesIndex: int, instanceIndex: int) -> object: + return self._orthanc.instances.modify( + instanceId=self._sourceInstanceId, + replaceTags={ + "PatientName": "Patient-" + str(patientIndex), + "PatientID": str(patientIndex), + "StudyDescription": str(patientIndex) + "-" + str(studyIndex), + "SeriesDescription": str(patientIndex) + "-" + str(studyIndex) + "-" + str(seriesIndex), + "SOPInstanceUID": str(patientIndex) + "." + str(studyIndex) + "." + str(seriesIndex) + "." + str(instanceIndex), + "StudyInstanceUID": str(patientIndex) + "." + str(studyIndex), + "SeriesInstanceUID": str(patientIndex) + "." + str(studyIndex) + "." + str(seriesIndex), + "SeriesNumber": str(seriesIndex), + "InstanceNumber": str(instanceIndex) + }, + deleteOriginal=False + ) \ No newline at end of file diff -r e0996602b306 -r f1a75985caa8 PerfsDb/DbServer.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/DbServer.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,146 @@ +import typing +import subprocess +import time + +from DbType import DbType + +class DbServer: + + class DockerDefinition: + + def __init__(self, image: str, internalPort: int, envVars: typing.Dict[str, str], storagePath: str, command: typing.List[str]=None): + self.image = image + self.internalPort = internalPort + self.envVars = envVars + self.storagePath = storagePath + self.command = command + + def __init__(self, dbType: DbType, port: int): + + self.port = port + self.dbType = dbType + + self._containerId = None + self._label = None + + def setLabel(self, label: str): + self._label = label + + def launch(self): + dockerDefinition = self.getDockerDefinition() + + # check if the container is already running + ret = subprocess.call([ + "docker", + "top", + self._label + ]) + if ret == 0: + print("DbServer is already running") + return + + # create a volume (if it already exists, it wont be modified) + subprocess.check_call([ + "docker", + "volume", + "create", + "--name=" + self._label + ]) + + dockerRunCommand = [ + "docker", + "run", + "-d", + "--name=" + self._label, + "-p", str(self.port) + ":" + str(dockerDefinition.internalPort), + "--volume=" + self._label + ":" + dockerDefinition.storagePath + ] + + if len(dockerDefinition.envVars) > 0: + for k,v in dockerDefinition.envVars.items(): + dockerRunCommand.extend(["--env", k + "=" + v]) + + dockerRunCommand.append( + dockerDefinition.image + ) + + if dockerDefinition.command is not None: + dockerRunCommand.extend( + dockerDefinition.command + ) + + print("Launching DbServer") + subprocess.check_call(dockerRunCommand) + + print("Waiting for DbServer to be ready") + + # wait until its port is open + retryCounter = 0 + connected = False + while not connected and retryCounter < 30: + time.sleep(1) + connected = subprocess.call(["nc", "-z", "localhost", str(self.port)]) == 0 + if retryCounter >= 30: + print("DbServer still not ready after 30 sec") + raise TimeoutError + + def stop(self): + subprocess.check_call([ + "docker", + "stop", + self._label + ]) + subprocess.check_call([ + "docker", + "rm", + self._label + ]) + + def clear(self): + # remove the volume + self.stop() + subprocess.check_call([ + "docker", + "volume", + "rm", + self._label + ]) + + + def getDockerDefinition(self): + if self.dbType == DbType.MySQL: + return DbServer.DockerDefinition( + image="mysql:8.0", + internalPort=3306, + envVars={ + "MYSQL_PASSWORD": "orthanc", + "MYSQL_USER": "orthanc", + "MYSQL_DATABASE": "orthanc", + "MYSQL_ROOT_PASSWORD": "foo-root" + }, + storagePath="/var/lib/mysql", + command=["mysqld", "--default-authentication-plugin=mysql_native_password", "--log-bin-trust-function-creators=1"] + ) + elif self.dbType == DbType.MSSQL: + return DbServer.DockerDefinition( + image="microsoft/mssql-server-linux", + internalPort=1433, + envVars={ + "ACCEPT_EULA": "Y", + "SA_PASSWORD": "MyStrOngPa55word!" + }, + storagePath="/var/opt/mssql/data" + ) + elif self.dbType == DbType.PG9 or self.dbType == DbType.PG10: + if self.dbType == DbType.PG9: + image = "postgres:9" + elif self.dbType == DbType.PG10: + image = "postgres:10" + return DbServer.DockerDefinition( + image=image, + internalPort=5432, + envVars={ + }, + storagePath="/var/lib/postgresql/data" + ) + diff -r e0996602b306 -r f1a75985caa8 PerfsDb/DbSize.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/DbSize.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,5 @@ +from enum import Enum + +class DbSize(Enum): + Small = 1 + Large = 2 diff -r e0996602b306 -r f1a75985caa8 PerfsDb/DbType.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/DbType.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,8 @@ +from enum import Enum + +class DbType(Enum): + Sqlite = 1 + PG9 = 2 + MySQL = 3 + MSSQL = 4 + PG10 = 5 diff -r e0996602b306 -r f1a75985caa8 PerfsDb/README.md --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/README.md Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,69 @@ +Performance Db tests +==================== + +Introduction +------------ + +This project performs benchmark tests of Orthanc with its various DB servers. + +It can be used to: + +- compare DB servers performance +- compare performance between small and big DB +- test the effectiveness of some code refactoring + +In a first step, the project creates a set of DB servers and populates them. +Then, it will perform a set of basic operations on Orthanc with each of these servers +and measure the time required for each operation. + +Timings are measured at the API level. Since we want to measure mainly the DB performance, +we'll mainly use very small DICOM files (without pixels data). + +Prerequisites +------------- + +- install python3, pip3 and pipenv + +```bash +sudo apt-get install -y python3 python3-pip python3-venv +``` + +- [install Docker-CE](https://docs.docker.com/install/linux/docker-ce/ubuntu/#set-up-the-repository) +- [install docker-compose](https://docs.docker.com/compose/install/) +- have access to docker without typing `sudo`. This is done by typing: `sudo groupadd docker` and `sudo usermod -aG docker $USER` +- have Orthanc and its DB plugins natively installed or compiled on your host system + +Once all prerequisites are installed, you should always execute all commands from a python virtual-env. To initialize the virtual env the first time: + +```bash +python3 -m venv .env +source .env/bin/activate +pip install -r requirements.txt +``` + +To enter the virtual-env the next times: + +```bash +source .env/bin/activate +``` + +Initializing a DB before tests +----------------- + +```bash +python run.py --orthanc-path=/home/amazy/builds/orthanc-build-release/ --plugins-path=/home/amazy/builds/mysql-release/ --init --mysql-small +``` + +Clearing a DB +----------------- + +```bash +python run.py --orthanc-path=/home/amazy/builds/orthanc-build-release/ --plugins-path=/home/amazy/builds/mysql-release/ --clear --mysql-small +``` + +Runing tests on two DBs +----------------- + +```bash +python run.py --orthanc-path=/home/amazy/builds/orthanc-build-release/ --plugins-path=/home/amazy/builds/mysql-release/ --run --mysql-large --mysql-small +``` diff -r e0996602b306 -r f1a75985caa8 PerfsDb/Test.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/Test.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,59 @@ +import time +import statistics +from orthancRestApi import OrthancClient + +from TestResult import TestResult + +class Test: + + def __init__(self, name: str): + self.name = name + self._orthanc = None + self.repeatCount = 10 + + def setOrthancClient(self, orthanc: OrthancClient): + self._orthanc = orthanc + + def setRepeatCount(self, repeatCount: int): + self.repeatCount = repeatCount + + def prepare(self): + """ + Code to execute before the execution of a test; i.e: upload a file (not including in timings) + """ + pass + + def test(self): + """ + Code whose execution time will be measured + """ + pass + + def cleanup(self): + """ + Code to execute after the execution of a test; i.e: remove an instance (not including in timings) + """ + pass + + def run(self) -> TestResult: + result = TestResult(self.name) + + for i in range(0, self.repeatCount): + self.prepare() + startTime = time.time() + self.test() + endTime = time.time() + self.cleanup() + + result.add((endTime - startTime) * 1000) + + result.compute() + return result + + def __str__(self): + return "{name:<40}: {avg:>8.2f} ms {min:>8.2f} ms {max:>8.2f} ms".format( + name=self.name, + avg = self.averageTimeInMs, + min=self.minTimeInMs, + max=self.maxTimeInMs + ) \ No newline at end of file diff -r e0996602b306 -r f1a75985caa8 PerfsDb/TestConfig.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/TestConfig.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,108 @@ +import typing +import subprocess +import os +from orthancRestApi import OrthancClient + +from DbSize import DbSize +from DbType import DbType +from ConfigFileBuilder import ConfigFileBuilder +from DbServer import DbServer +from DbPopulator import DbPopulator +from Tests import * +from TestResult import TestResult + +class TestConfig: + + def __init__(self, + label: str, + dbSize: DbSize, + dbType: DbType=None, + dbServer: DbServer=None + ): + + self._dbSize = dbSize + self._dbServer = dbServer + self._label = label + self._port = None + self._name = "unknown" + self._orthancProcess = None + self._repeatCount = 10 + + if dbServer is not None: + self._dbType = dbServer.dbType + self._dbServer.setLabel(self._label) + self._port = dbServer.port + else: + self._dbType = dbType + + def setName(self, name: str): + self._name = name + + def setRepeatCount(self, repeatCount: int): + self._repeatCount = repeatCount + + def launchDbServer(self): + if self._dbServer is not None: + self._dbServer.launch() + + def launchOrthanc(self, orthancPath): + orthanc = OrthancClient("http://127.0.0.1:8042") + + print("Checking if Orthanc is already running") + if orthanc.isAlive(): + print("Orthanc is already running") + return + + print("Launching Orthanc") + self._orthancProcess = subprocess.Popen([ + os.path.join(orthancPath, "Orthanc"), + os.path.join("ConfigFiles", self._name + ".json"), + ]) + + print("Waiting for Orthanc to start") + orthanc.waitStarted(timeout=30) + print("Orthanc has started") + + def stopOrthanc(self): + if self._orthancProcess is not None: + self._orthancProcess.terminate() + self._orthancProcess.wait() + + def initializeDb(self): + dbPopulator = DbPopulator(orthanc=OrthancClient("http://127.0.0.1:8042"), dbSize=self._dbSize) + dbPopulator.populate() + + def runTests(self) -> typing.List[TestResult]: + allTests = [ + TestFindStudyByStudyDescription1Result(), + TestFindStudyByPatientId1Result(), + TestFindStudyByStudyDescription0Results(), + TestFindStudyByPatientId0Results(), + TestFindStudyByPatientId5Results(), + TestUploadFile() + ] + + results = [] + for test in allTests: + test.setOrthancClient(OrthancClient("http://127.0.0.1:8042")) + test.setRepeatCount(self._repeatCount) + result = test.run() + print(str(result)) + + results.append(result) + return results + + def clearDb(self): + if self._dbServer is not None: + self._dbServer.clear() + + def generateOrthancConfigurationFile(self, pluginsPath: str): + + ConfigFileBuilder.generate( + outputPath="ConfigFiles/{name}.json".format(name=self._name), + plugins=[pluginsPath], + storagePath="Storages/{name}".format(name=self._name), + dbType=self._dbType, + dbSize=self._dbSize, + port=self._port + ) diff -r e0996602b306 -r f1a75985caa8 PerfsDb/TestResult.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/TestResult.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,36 @@ +import time +import statistics +from orthancRestApi import OrthancClient + +class TestResult: + + def __init__(self, name: str): + self.minTimeInMs = 0 + self.maxTimeInMs = 0 + self.averageTimeInMs = 0 + self.name = name + self._durations = [] + + def add(self, durationInMs: float): + self._durations.append(durationInMs) + + def compute(self): + + mean = statistics.mean(self._durations) + stdDev = statistics.stdev(self._durations) + + # remove outliers + cleanedDurations = [x for x in self._durations if (x > mean - 2*stdDev) and (x < mean + 2*stdDev)] + + self.averageTimeInMs = statistics.mean(cleanedDurations) + self.minTimeInMs = min(cleanedDurations) + self.maxTimeInMs = max(cleanedDurations) + + + def __str__(self): + return "{name:<40}: {avg:>8.2f} ms {min:>8.2f} ms {max:>8.2f} ms".format( + name=self.name, + avg = self.averageTimeInMs, + min=self.minTimeInMs, + max=self.maxTimeInMs + ) \ No newline at end of file diff -r e0996602b306 -r f1a75985caa8 PerfsDb/Tests/FindStudy.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/Tests/FindStudy.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,51 @@ +from Test import Test + +class TestFindStudyByStudyDescription1Result(Test): + + def __init__(self, name:str = "FindStudyByStudyDescription1Result"): + super().__init__(name) + + def test(self): + self._orthanc.studies.query( + query = {"StudyDescription": "99999-99999"} + ) + +class TestFindStudyByStudyDescription0Results(Test): + + def __init__(self, name:str = "FindStudyByStudyDescription0Results"): + super().__init__(name) + + def test(self): + self._orthanc.studies.query( + query = {"StudyDescription": "X"} + ) + +class TestFindStudyByPatientId1Result(Test): + + def __init__(self, name:str = "FindStudyByPatientId1Result"): + super().__init__(name) + + def test(self): + self._orthanc.studies.query( + query = {"PatientID": "99999"} + ) + +class TestFindStudyByPatientId0Results(Test): + + def __init__(self, name:str = "FindStudyByPatientId0Results"): + super().__init__(name) + + def test(self): + self._orthanc.studies.query( + query = {"PatientID": "X"} + ) + +class TestFindStudyByPatientId5Results(Test): + + def __init__(self, name:str = "FindStudyByPatientId5Results"): + super().__init__(name) + + def test(self): + self._orthanc.studies.query( + query = {"PatientID": "99998"} + ) \ No newline at end of file diff -r e0996602b306 -r f1a75985caa8 PerfsDb/Tests/UploadFile.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/Tests/UploadFile.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,21 @@ +from Test import Test + +class TestUploadFile(Test): + + def __init__(self, name:str = "UploadFile", filePath:str = "../Database/DummyCT.dcm"): + super().__init__(name) + self._instanceId = None + self._filePath = filePath + self._dicomFileContent = None + + def prepare(self): + # get the instance Id and dicom file content + if self._instanceId is None: + self._instanceId = self._orthanc.uploadDicomFile(self._filePath) + self._dicomFileContent = self._orthanc.instances.getDicom(self._instanceId) + + # make sure the file is not in Orthanc before the upload + self._orthanc.instances.delete(self._instanceId) + + def test(self): + self._orthanc.uploadDicom(self._dicomFileContent) \ No newline at end of file diff -r e0996602b306 -r f1a75985caa8 PerfsDb/Tests/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/Tests/__init__.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,2 @@ +from .UploadFile import TestUploadFile +from .FindStudy import * \ No newline at end of file diff -r e0996602b306 -r f1a75985caa8 PerfsDb/requirements.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/requirements.txt Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,1 @@ +http://orthanc.osimis.io/pythonToolbox/OsimisToolbox-22.0.0.tar.gz \ No newline at end of file diff -r e0996602b306 -r f1a75985caa8 PerfsDb/run.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PerfsDb/run.py Thu Aug 16 17:13:32 2018 +0200 @@ -0,0 +1,111 @@ +import argparse +from ConfigFileBuilder import ConfigFileBuilder +from TestConfig import TestConfig +from DbServer import DbServer +from DbType import DbType +from DbSize import DbSize + +testConfigs = { + "mysql-small" : TestConfig(label= "mysql-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.MySQL, port=2000)), + "mysql-large" : TestConfig(label= "mysql-large", dbSize=DbSize.Large, dbServer=DbServer(dbType=DbType.MySQL, port=2001)), + "sqlite-small": TestConfig(label= "sqlite-small", dbSize=DbSize.Small, dbType=DbType.Sqlite), + "pg9-small": TestConfig(label= "pg9-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG9, port=2002)), +} + +selectedTestConfigs = [] + +parser = argparse.ArgumentParser(description = "Initializes/Runs/Clears PerfsDb setup.") + +# create a cli option for each config +for testConfigName in testConfigs.keys(): + parser.add_argument("--" + testConfigName, action = "store_true") + +parser.add_argument("--init", help = "initializes DBs", action = "store_true") +parser.add_argument("--run", help = "runs tests", action = "store_true") +parser.add_argument("--clear", help = "clear DBs", action = "store_true") + +parser.add_argument("--orthanc-path", help = "path to the folder containing Orthanc executable", default=".") +parser.add_argument("--plugins-path", help = "path to the folder containing Orthanc executable", default=".") +parser.add_argument("--repeat", help = "number of times to repeat each test to average timings", type=int, default=50) + +args = parser.parse_args() + +for testConfigName in testConfigs.keys(): + if args.__dict__[testConfigName.replace("-", "_")]: + selectedTestConfigs.append(testConfigName) + +# if no test config specified, take them all +if len(selectedTestConfigs) == 0: + selectedTestConfigs = testConfigs.keys() + +# if no action specified, it means only run +if not (args.init | args.run | args.clear): + args.init = False + args.run = True + args.clear = False + +print("***** Orthanc *******") +print("path :", args.orthanc_path) + + +# libOrthancMySQLIndex.so +# libOrthancMySQLStorage.so +# libOrthancPostgreSQLIndex.so +# libOrthancPostgreSQLStorage.so +# libOrthancMSSQLIndex.so + +results = {} + +for configName in selectedTestConfigs: + testConfig = testConfigs[configName] + testConfig.setName(configName) + testConfig.setRepeatCount(args.repeat) + + print("======= " + configName + " ========") + + if args.clear: + print("** Clearing Db") + testConfig.clearDb() + + if args.init or args.run: + print("** Generating config files") + testConfig.generateOrthancConfigurationFile(args.plugins_path) + + print("** Launching DbServer") + testConfig.launchDbServer() + + print("** Launching Orthanc") + testConfig.launchOrthanc(args.orthanc_path) + + if args.init: + testConfig.initializeDb() + + if args.run: + print("** Runnnig tests") + results[configName] = testConfig.runTests() + print("** Stoping Orthanc") + testConfig.stopOrthanc() + +print("++++++++++++++ results summary +++++++++++++++") +testNames = set() +resultsByTestName = {} +for configName, configResult in results.items(): + for result in configResult: + testNames.add(result.name) + if not result.name in resultsByTestName: + resultsByTestName[result.name] = {} + resultsByTestName[result.name][configName] = result + +headerLine = "{empty:<40}|".format(empty="") +for configName in selectedTestConfigs: + headerLine += "{configName:^15}|".format(configName=configName) + +print(headerLine) + +for testName in sorted(testNames): + resultLine = "{name:<40}|".format(name=testName) + for configName in selectedTestConfigs: + resultLine += "{avg:>11.2f} ms |".format(avg = resultsByTestName[testName][configName].averageTimeInMs) + print(resultLine) + +print("** Done")