Mercurial > hg > orthanc-tests

comparison PerfsDb/Run.py @ 160:6995d5d12d88

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
filtering tests
author am@osimis.io
date Fri, 17 Aug 2018 16:17:57 +0200
parents df1f9946571c
children 27b3b0df5f90
comparison
equal deleted inserted replaced
159:616da104a996 160:6995d5d12d88
1 import argparse 1 import argparse
2 import fnmatch
3
2 from ConfigFileBuilder import ConfigFileBuilder 4 from ConfigFileBuilder import ConfigFileBuilder
3 from TestConfig import TestConfig 5 from TestConfig import TestConfig
6 from Tests import *
4 from DbServer import DbServer 7 from DbServer import DbServer
5 from DbType import DbType 8 from DbType import DbType
6 from DbSize import DbSize 9 from DbSize import DbSize
7 10
8 testConfigs = {
9 "mysql-small" : TestConfig(label= "mysql-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.MySQL, port=2000)),
10 "pg9-small": TestConfig(label= "pg9-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG9, port=2001)),
11 "pg10-small": TestConfig(label= "pg10-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG10, port=2002)),
12 "pg11-small": TestConfig(label= "pg11-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG11, port=2003)),
13 "mssql-small" : TestConfig(label= "mssql-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.MSSQL, port=2004)),
14 "sqlite-small": TestConfig(label= "sqlite-small", dbSize=DbSize.Small, dbType=DbType.Sqlite),
15 "sqliteplugin-small": TestConfig(label= "sqliteplugin-small", dbSize=DbSize.Small, dbType=DbType.SqlitePlugin),
16 11
17 "mysql-tiny" : TestConfig(label= "mysql-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.MySQL, port=3000)), 12 allTestConfigs = [
18 "pg9-tiny": TestConfig(label= "pg9-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.PG9, port=3001)), 13 TestConfig(label= "mysql-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.MySQL, port=2000)),
19 "pg10-tiny": TestConfig(label= "pg10-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.PG10, port=3002)), 14 TestConfig(label= "pg9-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG9, port=2001)),
20 "pg11-tiny": TestConfig(label= "pg11-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.PG11, port=3003)), 15 TestConfig(label= "pg10-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG10, port=2002)),
21 "mssql-tiny" : TestConfig(label= "mssql-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.MSSQL, port=3004)), 16 TestConfig(label= "pg11-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG11, port=2003)),
22 "sqlite-tiny": TestConfig(label= "sqlite-tiny", dbSize=DbSize.Tiny, dbType=DbType.Sqlite), 17 TestConfig(label= "mssql-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.MSSQL, port=2004)),
23 "sqliteplugin-tiny": TestConfig(label= "sqliteplugin-tiny", dbSize=DbSize.Tiny, dbType=DbType.SqlitePlugin), 18 TestConfig(label= "sqlite-small", dbSize=DbSize.Small, dbType=DbType.Sqlite),
19 TestConfig(label= "sqliteplugin-small", dbSize=DbSize.Small, dbType=DbType.SqlitePlugin),
24 20
25 "mysql-medium" : TestConfig(label= "mysql-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.MySQL, port=4000)), 21 TestConfig(label= "mysql-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.MySQL, port=3000)),
26 "pg9-medium": TestConfig(label= "pg9-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.PG9, port=4001)), 22 TestConfig(label= "pg9-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.PG9, port=3001)),
27 "pg10-medium": TestConfig(label= "pg10-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.PG10, port=4002)), 23 TestConfig(label= "pg10-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.PG10, port=3002)),
28 "pg11-medium": TestConfig(label= "pg11-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.PG11, port=4003)), 24 TestConfig(label= "pg11-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.PG11, port=3003)),
29 "mssql-medium" : TestConfig(label= "mssql-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.MSSQL, port=4004)), 25 TestConfig(label= "mssql-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.MSSQL, port=3004)),
30 "sqlite-medium": TestConfig(label= "sqlite-medium", dbSize=DbSize.Medium, dbType=DbType.Sqlite), 26 TestConfig(label= "sqlite-tiny", dbSize=DbSize.Tiny, dbType=DbType.Sqlite),
31 "sqliteplugin-medium": TestConfig(label= "sqliteplugin-medium", dbSize=DbSize.Medium, dbType=DbType.SqlitePlugin), 27 TestConfig(label= "sqliteplugin-tiny", dbSize=DbSize.Tiny, dbType=DbType.SqlitePlugin),
32 } 28
29 TestConfig(label= "mysql-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.MySQL, port=4000)),
30 TestConfig(label= "pg9-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.PG9, port=4001)),
31 TestConfig(label= "pg10-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.PG10, port=4002)),
32 TestConfig(label= "pg11-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.PG11, port=4003)),
33 TestConfig(label= "mssql-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.MSSQL, port=4004)),
34 TestConfig(label= "sqlite-medium", dbSize=DbSize.Medium, dbType=DbType.Sqlite),
35 TestConfig(label= "sqliteplugin-medium", dbSize=DbSize.Medium, dbType=DbType.SqlitePlugin),
36 ]
37
38 allTests = [
39 TestStatistics(),
40 TestFindStudyByStudyDescription1Result(),
41 TestFindStudyByPatientId1Result(),
42 TestFindStudyByStudyDescription0Results(),
43 TestFindStudyByPatientId0Results(),
44 TestFindStudyByPatientId5Results(),
45 TestUploadFile(),
46 ]
33 47
34 selectedTestConfigs = [] 48 selectedTestConfigs = []
49 selectedTests = []
35 50
36 parser = argparse.ArgumentParser(description = "Initializes/Runs/Clears PerfsDb setup.") 51 parser = argparse.ArgumentParser(description = "Initializes/Runs/Clears PerfsDb setup.")
37 52
38 # create a cli option for each config 53 # create a cli option for each config
39 for testConfigName in testConfigs.keys(): 54 for testConfig in allTestConfigs:
40 parser.add_argument("--" + testConfigName, action = "store_true") 55 parser.add_argument("--" + testConfig.label, action = "store_true")
41 56
42 parser.add_argument("--init", help = "initializes DBs", action = "store_true") 57 parser.add_argument("--init", help = "initializes DBs", action = "store_true")
43 parser.add_argument("--run", help = "runs tests", action = "store_true") 58 parser.add_argument("--run", help = "runs tests", action = "store_true")
44 parser.add_argument("--clear", help = "clear DBs", action = "store_true") 59 parser.add_argument("--clear", help = "clear DBs", action = "store_true")
45 60
46 parser.add_argument("--orthanc-path", help = "path to the folder containing Orthanc executable", default=".") 61 parser.add_argument("--orthanc-path", help = "path to the folder containing Orthanc executable", default=".")
47 parser.add_argument("--plugins-path", help = "path to the folder containing Orthanc executable", default=".") 62 parser.add_argument("--plugins-path", help = "path to the folder containing Orthanc executable", default=".")
48 parser.add_argument("--repeat", help = "number of times to repeat each test to average timings", type=int, default=50) 63 parser.add_argument("--repeat", help = "number of times to repeat each test to average timings", type=int, default=50)
64 parser.add_argument("--test-filter", help = "filter tests by names (wildcards are allowed)", default="*")
49 65
50 args = parser.parse_args() 66 args = parser.parse_args()
51 67
52 for testConfigName in testConfigs.keys(): 68 for testConfig in allTestConfigs:
53 if args.__dict__[testConfigName.replace("-", "_")]: 69 if args.__dict__[testConfig.label.replace("-", "_")]:
54 selectedTestConfigs.append(testConfigName) 70 selectedTestConfigs.append(testConfig)
55 71
56 # if no test config specified, take them all 72 # if no test config specified, take them all
57 if len(selectedTestConfigs) == 0: 73 if len(selectedTestConfigs) == 0:
58 selectedTestConfigs = testConfigs.keys() 74 selectedTestConfigs = allTestConfigs
59 75
60 selectedTestConfigs = sorted(selectedTestConfigs) 76 selectedTestConfigs.sort(key=lambda x: x.label)
77
78 # filter tests
79 for test in allTests:
80 if fnmatch.fnmatch(test.name, args.test_filter):
81 selectedTests.append(test)
82
83 selectedTests.sort(key=lambda x: x.name)
61 84
62 # if no action specified, it means only run 85 # if no action specified, it means only run
63 if not (args.init | args.run | args.clear): 86 if not (args.init | args.run | args.clear):
64 args.init = False 87 args.init = False
65 args.run = True 88 args.run = True
68 print("***** Orthanc *******") 91 print("***** Orthanc *******")
69 print("path :", args.orthanc_path) 92 print("path :", args.orthanc_path)
70 93
71 results = {} 94 results = {}
72 95
73 for configName in selectedTestConfigs: 96 for testConfig in selectedTestConfigs:
74 testConfig = testConfigs[configName]
75 testConfig.setName(configName)
76 testConfig.setRepeatCount(args.repeat) 97 testConfig.setRepeatCount(args.repeat)
77 98
78 print("======= " + configName + " ========") 99 print("======= " + testConfig.label + " ========")
79 100
80 if args.clear: 101 if args.clear:
81 print("** Clearing Db") 102 print("** Clearing Db")
82 testConfig.clearDb() 103 testConfig.clearDb()
83 104
97 if args.init: 118 if args.init:
98 testConfig.initializeDb() 119 testConfig.initializeDb()
99 120
100 if args.run: 121 if args.run:
101 print("** Runnnig tests") 122 print("** Runnnig tests")
102 results[configName] = testConfig.runTests() 123 results[testConfig.label] = testConfig.runTests(selectedTests)
103 124
104 print("** Stopping Orthanc") 125 print("** Stopping Orthanc")
105 testConfig.stopOrthanc() 126 testConfig.stopOrthanc()
106 127
107 print("++++++++++++++ results summary +++++++++++++++") 128 print("++++++++++++++ results summary +++++++++++++++")
113 if not result.name in resultsByTestName: 134 if not result.name in resultsByTestName:
114 resultsByTestName[result.name] = {} 135 resultsByTestName[result.name] = {}
115 resultsByTestName[result.name][configName] = result 136 resultsByTestName[result.name][configName] = result
116 137
117 headerLine = "{empty:<40}|".format(empty="") 138 headerLine = "{empty:<40}|".format(empty="")
118 for configName in selectedTestConfigs: 139 for testConfig in selectedTestConfigs:
119 headerLine += "{configName:^15}|".format(configName=configName) 140 headerLine += "{configName:^15}|".format(configName=testConfig.label)
120 141
121 print(headerLine) 142 print(headerLine)
122 143
123 for testName in sorted(testNames): 144 for testName in sorted(testNames):
124 resultLine = "{name:<40}|".format(name=testName) 145 resultLine = "{name:<40}|".format(name=testName)
125 for configName in selectedTestConfigs: 146 for testConfig in selectedTestConfigs:
126 resultLine += "{avg:>11.2f} ms |".format(avg = resultsByTestName[testName][configName].averageTimeInMs) 147 resultLine += "{avg:>11.2f} ms |".format(avg = resultsByTestName[testName][testConfig.label].averageTimeInMs)
127 print(resultLine) 148 print(resultLine)
128 149
129 print("** Done") 150 print("** Done")