Mercurial > hg > orthanc-tests
annotate PerfsDb/Run.py @ 164:ff939d07989f
saving results
author | am@osimis.io |
---|---|
date | Mon, 20 Aug 2018 15:50:47 +0200 |
parents | dbc42a03ee75 |
children | 1ff0d830bb87 |
rev | line source |
---|---|
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
1 import argparse |
160 | 2 import fnmatch |
164 | 3 import csv |
4 import datetime | |
5 import os | |
160 | 6 |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
7 from ConfigFileBuilder import ConfigFileBuilder |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
8 from TestConfig import TestConfig |
160 | 9 from Tests import * |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
10 from DbServer import DbServer |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
11 from DbType import DbType |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
12 from DbSize import DbSize |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
13 |
160 | 14 |
15 allTestConfigs = [ | |
16 TestConfig(label= "mysql-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.MySQL, port=2000)), | |
17 TestConfig(label= "pg9-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG9, port=2001)), | |
18 TestConfig(label= "pg10-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG10, port=2002)), | |
19 TestConfig(label= "pg11-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.PG11, port=2003)), | |
20 TestConfig(label= "mssql-small", dbSize=DbSize.Small, dbServer=DbServer(dbType=DbType.MSSQL, port=2004)), | |
21 TestConfig(label= "sqlite-small", dbSize=DbSize.Small, dbType=DbType.Sqlite), | |
22 TestConfig(label= "sqliteplugin-small", dbSize=DbSize.Small, dbType=DbType.SqlitePlugin), | |
23 | |
24 TestConfig(label= "mysql-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.MySQL, port=3000)), | |
25 TestConfig(label= "pg9-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.PG9, port=3001)), | |
26 TestConfig(label= "pg10-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.PG10, port=3002)), | |
27 TestConfig(label= "pg11-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.PG11, port=3003)), | |
28 TestConfig(label= "mssql-tiny", dbSize=DbSize.Tiny, dbServer=DbServer(dbType=DbType.MSSQL, port=3004)), | |
29 TestConfig(label= "sqlite-tiny", dbSize=DbSize.Tiny, dbType=DbType.Sqlite), | |
30 TestConfig(label= "sqliteplugin-tiny", dbSize=DbSize.Tiny, dbType=DbType.SqlitePlugin), | |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
31 |
160 | 32 TestConfig(label= "mysql-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.MySQL, port=4000)), |
33 TestConfig(label= "pg9-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.PG9, port=4001)), | |
34 TestConfig(label= "pg10-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.PG10, port=4002)), | |
35 TestConfig(label= "pg11-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.PG11, port=4003)), | |
36 TestConfig(label= "mssql-medium", dbSize=DbSize.Medium, dbServer=DbServer(dbType=DbType.MSSQL, port=4004)), | |
37 TestConfig(label= "sqlite-medium", dbSize=DbSize.Medium, dbType=DbType.Sqlite), | |
38 TestConfig(label= "sqliteplugin-medium", dbSize=DbSize.Medium, dbType=DbType.SqlitePlugin), | |
161 | 39 |
40 TestConfig(label= "mysql-large", dbSize=DbSize.Large, dbServer=DbServer(dbType=DbType.MySQL, port=5000)), | |
41 TestConfig(label= "pg9-large", dbSize=DbSize.Large, dbServer=DbServer(dbType=DbType.PG9, port=5001)), | |
42 TestConfig(label= "pg10-large", dbSize=DbSize.Large, dbServer=DbServer(dbType=DbType.PG10, port=5002)), | |
43 TestConfig(label= "pg11-large", dbSize=DbSize.Large, dbServer=DbServer(dbType=DbType.PG11, port=5003)), | |
44 TestConfig(label= "mssql-large", dbSize=DbSize.Large, dbServer=DbServer(dbType=DbType.MSSQL, port=5004)), | |
45 TestConfig(label= "sqlite-large", dbSize=DbSize.Large, dbType=DbType.Sqlite), | |
46 TestConfig(label= "sqliteplugin-large", dbSize=DbSize.Large, dbType=DbType.SqlitePlugin), | |
160 | 47 ] |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
48 |
160 | 49 allTests = [ |
50 TestStatistics(), | |
51 TestFindStudyByStudyDescription1Result(), | |
52 TestFindStudyByPatientId1Result(), | |
53 TestFindStudyByStudyDescription0Results(), | |
54 TestFindStudyByPatientId0Results(), | |
55 TestFindStudyByPatientId5Results(), | |
161 | 56 TestUploadNextPatientFile(), |
57 TestUploadFirstPatientFile(), | |
162 | 58 TestUploadLargeFile10MB(), |
59 TestToolsFindStudyByStudyInstanceUID(), | |
60 TestToolsFindPatientByPatientID() | |
160 | 61 ] |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
62 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
63 selectedTestConfigs = [] |
160 | 64 selectedTests = [] |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
65 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
66 parser = argparse.ArgumentParser(description = "Initializes/Runs/Clears PerfsDb setup.") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
67 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
68 # create a cli option for each config |
160 | 69 for testConfig in allTestConfigs: |
70 parser.add_argument("--" + testConfig.label, action = "store_true") | |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
71 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
72 parser.add_argument("--init", help = "initializes DBs", action = "store_true") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
73 parser.add_argument("--run", help = "runs tests", action = "store_true") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
74 parser.add_argument("--clear", help = "clear DBs", action = "store_true") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
75 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
76 parser.add_argument("--orthanc-path", help = "path to the folder containing Orthanc executable", default=".") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
77 parser.add_argument("--plugins-path", help = "path to the folder containing Orthanc executable", default=".") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
78 parser.add_argument("--repeat", help = "number of times to repeat each test to average timings", type=int, default=50) |
160 | 79 parser.add_argument("--test-filter", help = "filter tests by names (wildcards are allowed)", default="*") |
161 | 80 parser.add_argument("--verbose", help = "start Orthanc in verbose mode", action = "store_true") |
81 parser.add_argument("--trace", help = "start Orthanc in trace mode", action = "store_true") | |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
82 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
83 args = parser.parse_args() |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
84 |
160 | 85 for testConfig in allTestConfigs: |
86 if args.__dict__[testConfig.label.replace("-", "_")]: | |
87 selectedTestConfigs.append(testConfig) | |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
88 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
89 # if no test config specified, take them all |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
90 if len(selectedTestConfigs) == 0: |
160 | 91 selectedTestConfigs = allTestConfigs |
92 | |
93 selectedTestConfigs.sort(key=lambda x: x.label) | |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
94 |
160 | 95 # filter tests |
96 for test in allTests: | |
97 if fnmatch.fnmatch(test.name, args.test_filter): | |
98 selectedTests.append(test) | |
99 | |
100 selectedTests.sort(key=lambda x: x.name) | |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
101 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
102 # if no action specified, it means only run |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
103 if not (args.init | args.run | args.clear): |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
104 args.init = False |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
105 args.run = True |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
106 args.clear = False |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
107 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
108 print("***** Orthanc *******") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
109 print("path :", args.orthanc_path) |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
110 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
111 results = {} |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
112 |
160 | 113 for testConfig in selectedTestConfigs: |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
114 testConfig.setRepeatCount(args.repeat) |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
115 |
160 | 116 print("======= " + testConfig.label + " ========") |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
117 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
118 if args.clear: |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
119 print("** Clearing Db") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
120 testConfig.clearDb() |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
121 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
122 if args.init or args.run: |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
123 print("** Generating config files") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
124 testConfig.generateOrthancConfigurationFile(args.plugins_path) |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
125 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
126 print("** Launching DbServer") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
127 testConfig.launchDbServer() |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
128 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
129 print("** Launching Orthanc") |
161 | 130 orthancWasAlreadyRunning = not testConfig.launchOrthanc(args.orthanc_path, verboseEnabled=args.verbose, traceEnabled=args.trace) |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
131 if orthancWasAlreadyRunning and len(selectedTestConfigs) > 1: |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
132 print("Error: Can't execute multiple configuration on already running Orthanc. Exit Orthanc and let this script start Orthanc instances") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
133 exit(-1) |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
134 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
135 if args.init: |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
136 testConfig.initializeDb() |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
137 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
138 if args.run: |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
139 print("** Runnnig tests") |
160 | 140 results[testConfig.label] = testConfig.runTests(selectedTests) |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
141 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
142 print("** Stopping Orthanc") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
143 testConfig.stopOrthanc() |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
144 |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
145 print("++++++++++++++ results summary +++++++++++++++") |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
146 testNames = set() |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
147 resultsByTestName = {} |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
148 for configName, configResult in results.items(): |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
149 for result in configResult: |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
150 testNames.add(result.name) |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
151 if not result.name in resultsByTestName: |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
152 resultsByTestName[result.name] = {} |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
153 resultsByTestName[result.name][configName] = result |
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
154 |
164 | 155 resultFileName = os.path.join(os.path.dirname(os.path.abspath(__file__)), "Results/run-" + datetime.datetime.now().strftime("%Y.%m.%d.%H.%M.%S")) |
156 | |
157 with open(resultFileName, 'w', newline='') as resultFile: | |
158 resultWriter = csv.writer(resultFile) | |
159 resultHeaderRow = [""] | |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
160 |
164 | 161 headerLine = "{empty:<40}|".format(empty="") |
162 | |
163 for testConfig in selectedTestConfigs: | |
164 headerLine += "{configName:^15}|".format(configName=testConfig.label) | |
165 resultHeaderRow.append(configName) | |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
166 |
164 | 167 print(headerLine) |
168 resultWriter.writerow(resultHeaderRow) | |
158
df1f9946571c
perfs db continued: tests working with tiny DBs on all setup but the sqliteplugin
am@osimis.io
parents:
diff
changeset
|
169 |
164 | 170 for testName in sorted(testNames): |
171 resultLine = "{name:<40}|".format(name=testName) | |
172 resultRow=[testName] | |
173 | |
174 for testConfig in selectedTestConfigs: | |
175 resultLine += "{avg:>11.2f} ms |".format(avg = resultsByTestName[testName][testConfig.label].averageTimeInMs) | |
176 resultRow.append(resultsByTestName[testName][testConfig.label].averageTimeInMs) | |
177 | |
178 print(resultLine) | |
179 resultWriter.writerow(resultRow) | |
180 | |
181 print("** Done; results saved in " + resultFileName) |