Mercurial > hg > orthanc-stone
comparison Resources/CodeGeneration/stonegentool.py @ 496:8b6ceae45ba0 bgo-commands-codegen
Finished (untested) C++, html, typescript, tsc & browserify production.
author | bgo-osimis |
---|---|
date | Sat, 23 Feb 2019 15:04:29 +0100 |
parents | fc17251477d6 |
children | ce49eae4c887 |
comparison
equal
deleted
inserted
replaced
495:6405435480ae | 496:8b6ceae45ba0 |
---|---|
2 import yaml | 2 import yaml |
3 import re | 3 import re |
4 import os | 4 import os |
5 import sys | 5 import sys |
6 from jinja2 import Template | 6 from jinja2 import Template |
7 from typing import ( | |
8 Any, | |
9 Dict, | |
10 Generator, | |
11 Iterable, | |
12 Iterator, | |
13 List, | |
14 Match, | |
15 Optional, | |
16 Tuple, | |
17 Union, | |
18 cast, | |
19 ) | |
20 from io import StringIO | 7 from io import StringIO |
21 import time | 8 import time |
22 | 9 |
23 """ | 10 """ |
24 1 2 3 4 5 6 7 | 11 1 2 3 4 5 6 7 |
69 self.tsEnums = StringIO() | 56 self.tsEnums = StringIO() |
70 self.tsStructs = StringIO() | 57 self.tsStructs = StringIO() |
71 self.tsDispatcher = StringIO() | 58 self.tsDispatcher = StringIO() |
72 self.tsHandler = StringIO() | 59 self.tsHandler = StringIO() |
73 | 60 |
74 def FlattenToFiles(self, outputDir: str): | 61 def FlattenToFiles(self, outputDir): |
75 raise NotImplementedError() | 62 raise NotImplementedError() |
76 | 63 |
77 | 64 |
78 class JsonHelpers: | 65 class JsonHelpers: |
79 """A set of utilities to perform JSON operations""" | 66 """A set of utilities to perform JSON operations""" |
104 fileContent = fp.read() | 91 fileContent = fp.read() |
105 fileContent = JsonHelpers.removeCommentsFromJsonContent(fileContent) | 92 fileContent = JsonHelpers.removeCommentsFromJsonContent(fileContent) |
106 return json.loads(fileContent) | 93 return json.loads(fileContent) |
107 | 94 |
108 | 95 |
109 def LoadSchemaFromJson(filePath: str): | 96 def LoadSchemaFromJson(filePath): |
110 return JsonHelpers.loadJsonWithComments(filePath) | 97 return JsonHelpers.loadJsonWithComments(filePath) |
111 | 98 |
112 def CanonToCpp(canonicalTypename: str) -> str: | 99 def CanonToCpp(canonicalTypename): |
113 # C++: prefix map vector and string with std::map, std::vector and | 100 # C++: prefix map vector and string with std::map, std::vector and |
114 # std::string | 101 # std::string |
115 # replace int32 by int32_t | 102 # replace int32 by int32_t |
116 # replace float32 by float | 103 # replace float32 by float |
117 # replace float64 by double | 104 # replace float64 by double |
118 retVal: str = canonicalTypename | 105 retVal = canonicalTypename |
119 retVal = retVal.replace("map", "std::map") | 106 retVal = retVal.replace("map", "std::map") |
120 retVal = retVal.replace("vector", "std::vector") | 107 retVal = retVal.replace("vector", "std::vector") |
121 retVal = retVal.replace("string", "std::string") | 108 retVal = retVal.replace("string", "std::string") |
122 retVal = retVal.replace("int32", "int32_t") | 109 retVal = retVal.replace("int32", "int32_t") |
123 retVal = retVal.replace("float32", "float") | 110 retVal = retVal.replace("float32", "float") |
124 retVal = retVal.replace("float64", "double") | 111 retVal = retVal.replace("float64", "double") |
125 return retVal | 112 return retVal |
126 | 113 |
127 def CanonToTs(canonicalTypename: str) -> str: | 114 def CanonToTs(canonicalTypename): |
128 # TS: replace vector with Array and map with Map | 115 # TS: replace vector with Array and map with Map |
129 # string remains string | 116 # string remains string |
130 # replace int32 by number | 117 # replace int32 by number |
131 # replace float32 by number | 118 # replace float32 by number |
132 # replace float64 by number | 119 # replace float64 by number |
133 retVal: str = canonicalTypename | 120 retVal = canonicalTypename |
134 retVal = retVal.replace("map", "Map") | 121 retVal = retVal.replace("map", "Map") |
135 retVal = retVal.replace("vector", "Array") | 122 retVal = retVal.replace("vector", "Array") |
136 retVal = retVal.replace("int32", "number") | 123 retVal = retVal.replace("int32", "number") |
137 retVal = retVal.replace("float32", "number") | 124 retVal = retVal.replace("float32", "number") |
138 retVal = retVal.replace("float64", "number") | 125 retVal = retVal.replace("float64", "number") |
139 retVal = retVal.replace("bool", "boolean") | 126 retVal = retVal.replace("bool", "boolean") |
140 return retVal | 127 return retVal |
141 | 128 |
142 def NeedsTsConstruction(enums: Dict, tsType: str): | 129 def NeedsTsConstruction(enums, tsType): |
143 if tsType == 'boolean': | 130 if tsType == 'boolean': |
144 return False | 131 return False |
145 elif tsType == 'number': | 132 elif tsType == 'number': |
146 return False | 133 return False |
147 elif tsType == 'string': | 134 elif tsType == 'string': |
174 templateFile = open(templateFileName, "r") | 161 templateFile = open(templateFileName, "r") |
175 templateFileContents = templateFile.read() | 162 templateFileContents = templateFile.read() |
176 return MakeTemplate(templateFileContents) | 163 return MakeTemplate(templateFileContents) |
177 templateFile.close() | 164 templateFile.close() |
178 | 165 |
179 def EatToken(sentence: str) -> Tuple[str, str]: | 166 def EatToken(sentence): |
180 """splits "A,B,C" into "A" and "B,C" where A, B and C are type names | 167 """splits "A,B,C" into "A" and "B,C" where A, B and C are type names |
181 (including templates) like "int32", "TotoTutu", or | 168 (including templates) like "int32", "TotoTutu", or |
182 "map<map<int32,vector<string>>,map<string,int32>>" """ | 169 "map<map<int32,vector<string>>,map<string,int32>>" """ |
183 | 170 |
184 if sentence.count("<") != sentence.count(">"): | 171 if sentence.count("<") != sentence.count(">"): |
185 raise Exception( | 172 raise Exception( |
186 f"Error in the partial template type list {sentence}." | 173 "Error in the partial template type list " + str(sentence) + "." |
187 + " The number of < and > do not match!" | 174 + " The number of < and > do not match!" |
188 ) | 175 ) |
189 | 176 |
190 # the template level we're currently in | 177 # the template level we're currently in |
191 templateLevel = 0 | 178 templateLevel = 0 |
197 elif sentence[i] == ">": | 184 elif sentence[i] == ">": |
198 templateLevel -= 1 | 185 templateLevel -= 1 |
199 return (sentence, "") | 186 return (sentence, "") |
200 | 187 |
201 | 188 |
202 def SplitListOfTypes(typename: str) -> List[str]: | 189 def SplitListOfTypes(typename): |
203 """Splits something like | 190 """Splits something like |
204 vector<string>,int32,map<string,map<string,int32>> | 191 vector<string>,int32,map<string,map<string,int32>> |
205 in: | 192 in: |
206 - vector<string> | 193 - vector<string> |
207 - int32 | 194 - int32 |
208 map<string,map<string,int32>> | 195 map<string,map<string,int32>> |
209 | 196 |
210 This is not possible with a regex so | 197 This is not possible with a regex so |
211 """ | 198 """ |
212 stillStuffToEat: bool = True | 199 stillStuffToEat = True |
213 tokenList = [] | 200 tokenList = [] |
214 restOfString = typename | 201 restOfString = typename |
215 while stillStuffToEat: | 202 while stillStuffToEat: |
216 firstToken, restOfString = EatToken(restOfString) | 203 firstToken, restOfString = EatToken(restOfString) |
217 tokenList.append(firstToken) | 204 tokenList.append(firstToken) |
222 | 209 |
223 templateRegex = \ | 210 templateRegex = \ |
224 re.compile(r"([a-zA-Z0-9_]*[a-zA-Z0-9_]*)<([a-zA-Z0-9_,:<>]+)>") | 211 re.compile(r"([a-zA-Z0-9_]*[a-zA-Z0-9_]*)<([a-zA-Z0-9_,:<>]+)>") |
225 | 212 |
226 | 213 |
227 def ParseTemplateType(typename) -> Tuple[bool, str, List[str]]: | 214 def ParseTemplateType(typename): |
228 """ If the type is a template like "SOMETHING<SOME<THING,EL<SE>>>", | 215 """ If the type is a template like "SOMETHING<SOME<THING,EL<SE>>>", |
229 then it returns (true,"SOMETHING","SOME<THING,EL<SE>>") | 216 then it returns (true,"SOMETHING","SOME<THING,EL<SE>>") |
230 otherwise it returns (false,"","")""" | 217 otherwise it returns (false,"","")""" |
231 | 218 |
232 # let's remove all whitespace from the type | 219 # let's remove all whitespace from the type |
235 typename = "".join(typename.split()) | 222 typename = "".join(typename.split()) |
236 matches = templateRegex.match(typename) | 223 matches = templateRegex.match(typename) |
237 if matches == None: | 224 if matches == None: |
238 return (False, "", []) | 225 return (False, "", []) |
239 else: | 226 else: |
240 m = cast(Match[str], matches) | 227 m = matches |
241 assert len(m.groups()) == 2 | 228 assert len(m.groups()) == 2 |
242 # we need to split with the commas that are outside of the | 229 # we need to split with the commas that are outside of the |
243 # defined types. Simply splitting at commas won't work | 230 # defined types. Simply splitting at commas won't work |
244 listOfDependentTypes = SplitListOfTypes(m.group(2)) | 231 listOfDependentTypes = SplitListOfTypes(m.group(2)) |
245 return (True, m.group(1), listOfDependentTypes) | 232 return (True, m.group(1), listOfDependentTypes) |
246 | 233 |
247 | 234 |
248 def ComputeOrderFromTypeTree( | 235 def ComputeOrderFromTypeTree( |
249 ancestors: List[str], | 236 ancestors, |
250 genOrder: List[str], | 237 genOrder, |
251 shortTypename: str, schema: Dict[str, Dict]) -> None: | 238 shortTypename, schema): |
252 | 239 |
253 if shortTypename in ancestors: | 240 if shortTypename in ancestors: |
254 raise Exception( | 241 raise Exception( |
255 f"Cyclic dependency chain found: the last of {ancestors} " | 242 "Cyclic dependency chain found: the last of " + str(ancestors) + |
256 + f"depends on {shortTypename} that is already in the list." | 243 + " depends on " + str(shortTypename) + " that is already in the list." |
257 ) | 244 ) |
258 | 245 |
259 if not (shortTypename in genOrder): | 246 if not (shortTypename in genOrder): |
260 (isTemplate, _, dependentTypenames) = ParseTemplateType(shortTypename) | 247 (isTemplate, _, dependentTypenames) = ParseTemplateType(shortTypename) |
261 if isTemplate: | 248 if isTemplate: |
271 else: | 258 else: |
272 # If it is not template, we are only interested if it is a | 259 # If it is not template, we are only interested if it is a |
273 # dependency that we must take into account in the dep graph, | 260 # dependency that we must take into account in the dep graph, |
274 # i.e., a struct. | 261 # i.e., a struct. |
275 if IsShortStructType(shortTypename, schema): | 262 if IsShortStructType(shortTypename, schema): |
276 struct:Dict = schema[GetLongTypename(shortTypename, schema)] | 263 struct = schema[GetLongTypename(shortTypename, schema)] |
277 # The keys in the struct dict are the member names | 264 # The keys in the struct dict are the member names |
278 # The values in the struct dict are the member types | 265 # The values in the struct dict are the member types |
279 for field in struct.keys(): | 266 for field in struct.keys(): |
280 # we fill the chain of dependent types (starting here) | 267 # we fill the chain of dependent types (starting here) |
281 ancestors.append(shortTypename) | 268 ancestors.append(shortTypename) |
292 | 279 |
293 # +-----------------------+ | 280 # +-----------------------+ |
294 # | Utility functions | | 281 # | Utility functions | |
295 # +-----------------------+ | 282 # +-----------------------+ |
296 | 283 |
297 def IsShortStructType(typename: str, schema: Dict[str, Dict]) -> bool: | 284 def IsShortStructType(typename, schema): |
298 fullStructName = "struct " + typename | 285 fullStructName = "struct " + typename |
299 return (fullStructName in schema) | 286 return (fullStructName in schema) |
300 | 287 |
301 def GetLongTypename(shortTypename: str, schema: Dict): | 288 def GetLongTypename(shortTypename, schema): |
302 if shortTypename.startswith("enum "): | 289 if shortTypename.startswith("enum "): |
303 raise RuntimeError('shortTypename.startswith("enum "):') | 290 raise RuntimeError('shortTypename.startswith("enum "):') |
304 enumName: str = "enum " + shortTypename | 291 enumName = "enum " + shortTypename |
305 isEnum = enumName in schema | 292 isEnum = enumName in schema |
306 | 293 |
307 if shortTypename.startswith("struct "): | 294 if shortTypename.startswith("struct "): |
308 raise RuntimeError('shortTypename.startswith("struct "):') | 295 raise RuntimeError('shortTypename.startswith("struct "):') |
309 structName: str = "struct " + shortTypename | 296 structName = "struct " + shortTypename |
310 isStruct = ("struct " + shortTypename) in schema | 297 isStruct = ("struct " + shortTypename) in schema |
311 | 298 |
312 if isEnum and isStruct: | 299 if isEnum and isStruct: |
313 raise RuntimeError('Enums and structs cannot have the same name') | 300 raise RuntimeError('Enums and structs cannot have the same name') |
314 | 301 |
315 if isEnum: | 302 if isEnum: |
316 return enumName | 303 return enumName |
317 if isStruct: | 304 if isStruct: |
318 return structName | 305 return structName |
319 | 306 |
320 def IsTypename(fullName: str) -> bool: | 307 def IsTypename(fullName): |
321 return (fullName.startswith("enum ") or fullName.startswith("struct ")) | 308 return (fullName.startswith("enum ") or fullName.startswith("struct ")) |
322 | 309 |
323 def IsEnumType(fullName: str) -> bool: | 310 def IsEnumType(fullName): |
324 return fullName.startswith("enum ") | 311 return fullName.startswith("enum ") |
325 | 312 |
326 def IsStructType(fullName: str) -> bool: | 313 def IsStructType(fullName): |
327 return fullName.startswith("struct ") | 314 return fullName.startswith("struct ") |
328 | 315 |
329 def GetShortTypename(fullTypename: str) -> str: | 316 def GetShortTypename(fullTypename): |
330 if fullTypename.startswith("struct "): | 317 if fullTypename.startswith("struct "): |
331 return fullTypename[7:] | 318 return fullTypename[7:] |
332 elif fullTypename.startswith("enum"): | 319 elif fullTypename.startswith("enum"): |
333 return fullTypename[5:] | 320 return fullTypename[5:] |
334 else: | 321 else: |
335 raise RuntimeError \ | 322 raise RuntimeError \ |
336 ('fullTypename should start with either "struct " or "enum "') | 323 ('fullTypename should start with either "struct " or "enum "') |
337 | 324 |
338 def CheckSchemaSchema(schema: Dict) -> None: | 325 def CheckSchemaSchema(schema): |
339 if not "rootName" in schema: | 326 if not "rootName" in schema: |
340 raise Exception("schema lacks the 'rootName' key") | 327 raise Exception("schema lacks the 'rootName' key") |
341 for name in schema.keys(): | 328 for name in schema.keys(): |
342 if (not IsEnumType(name)) and (not IsStructType(name)) and \ | 329 if (not IsEnumType(name)) and (not IsStructType(name)) and \ |
343 (name != 'rootName'): | 330 (name != 'rootName'): |
344 raise RuntimeError \ | 331 raise RuntimeError \ |
345 (f'Type "{name}" should start with "enum " or "struct "') | 332 ('Type "' + str(name) + '" should start with "enum " or "struct "') |
346 | 333 |
347 # TODO: check enum fields are unique (in whole namespace) | 334 # TODO: check enum fields are unique (in whole namespace) |
348 # TODO: check struct fields are unique (in each struct) | 335 # TODO: check struct fields are unique (in each struct) |
349 # TODO: check that in the source schema, there are spaces after each colon | 336 # TODO: check that in the source schema, there are spaces after each colon |
350 | 337 |
351 # +-----------------------+ | 338 # +-----------------------+ |
352 # | Main processing logic | | 339 # | Main processing logic | |
353 # +-----------------------+ | 340 # +-----------------------+ |
354 | 341 |
355 def ComputeRequiredDeclarationOrder(schema: dict) -> List[str]: | 342 def ComputeRequiredDeclarationOrder(schema): |
356 # sanity check | 343 # sanity check |
357 CheckSchemaSchema(schema) | 344 CheckSchemaSchema(schema) |
358 | 345 |
359 # we traverse the type dependency graph and we fill a queue with | 346 # we traverse the type dependency graph and we fill a queue with |
360 # the required struct types, in a bottom-up fashion, to compute | 347 # the required struct types, in a bottom-up fashion, to compute |
362 # The genOrder list contains the struct full names in the order | 349 # The genOrder list contains the struct full names in the order |
363 # where they must be defined. | 350 # where they must be defined. |
364 # We do not care about the enums here... They do not depend upon | 351 # We do not care about the enums here... They do not depend upon |
365 # anything and we'll handle them, in their original declaration | 352 # anything and we'll handle them, in their original declaration |
366 # order, at the start | 353 # order, at the start |
367 genOrder: List = [] | 354 genOrder = [] |
368 for fullName in schema.keys(): | 355 for fullName in schema.keys(): |
369 if IsStructType(fullName): | 356 if IsStructType(fullName): |
370 realName: str = GetShortTypename(fullName) | 357 realName = GetShortTypename(fullName) |
371 ancestors: List[str] = [] | 358 ancestors = [] |
372 ComputeOrderFromTypeTree(ancestors, genOrder, realName, schema) | 359 ComputeOrderFromTypeTree(ancestors, genOrder, realName, schema) |
373 return genOrder | 360 return genOrder |
374 | 361 |
375 def ProcessSchema(schema: dict, genOrder: List[str]) -> Dict: | 362 def ProcessSchema(schema, genOrder): |
376 # sanity check | 363 # sanity check |
377 CheckSchemaSchema(schema) | 364 CheckSchemaSchema(schema) |
378 | 365 |
379 # let's doctor the schema to clean it up a bit | 366 # let's doctor the schema to clean it up a bit |
380 # order DOES NOT matter for enums, even though it's a list | 367 # order DOES NOT matter for enums, even though it's a list |
381 enums: List[Dict] = [] | 368 enums = [] |
382 for fullName in schema.keys(): | 369 for fullName in schema.keys(): |
383 if IsEnumType(fullName): | 370 if IsEnumType(fullName): |
384 # convert "enum Toto" to "Toto" | 371 # convert "enum Toto" to "Toto" |
385 typename:str = GetShortTypename(fullName) | 372 typename = GetShortTypename(fullName) |
386 enum = {} | 373 enum = {} |
387 enum['name'] = typename | 374 enum['name'] = typename |
388 assert(type(schema[fullName]) == list) | 375 assert(type(schema[fullName]) == list) |
389 enum['fields'] = schema[fullName] # must be a list | 376 enum['fields'] = schema[fullName] # must be a list |
390 enums.append(enum) | 377 enums.append(enum) |
407 # "someOtherMember22":"vector<Message1>" | 394 # "someOtherMember22":"vector<Message1>" |
408 # } | 395 # } |
409 # } | 396 # } |
410 # ] | 397 # ] |
411 | 398 |
412 structs: List[Dict] = [] | 399 structs = [] |
413 for i in range(len(genOrder)): | 400 for i in range(len(genOrder)): |
414 # this is already the short name | 401 # this is already the short name |
415 typename = genOrder[i] | 402 typename = genOrder[i] |
416 fieldDict = schema["struct " + typename] | 403 fieldDict = schema["struct " + typename] |
417 struct = {} | 404 struct = {} |
454 schema = yaml.load(schemaText) | 441 schema = yaml.load(schemaText) |
455 return schema | 442 return schema |
456 | 443 |
457 def GetTemplatingDictFromSchemaFilename(fn): | 444 def GetTemplatingDictFromSchemaFilename(fn): |
458 obj = LoadSchema(fn) | 445 obj = LoadSchema(fn) |
459 genOrder: str = ComputeRequiredDeclarationOrder(obj) | 446 genOrder = ComputeRequiredDeclarationOrder(obj) |
460 templatingDict = ProcessSchema(obj, genOrder) | 447 templatingDict = ProcessSchema(obj, genOrder) |
461 return templatingDict | 448 return templatingDict |
462 | 449 |
463 # +-----------------------+ | 450 # +-----------------------+ |
464 # | ENTRY POINT | | 451 # | ENTRY POINT | |
495 | 482 |
496 args = parser.parse_args() | 483 args = parser.parse_args() |
497 schemaFile = args.input_schema | 484 schemaFile = args.input_schema |
498 outDir = args.out_dir | 485 outDir = args.out_dir |
499 | 486 |
500 tdico: Dict = GetTemplatingDictFromSchemaFilename(schemaFile) | 487 tdico = GetTemplatingDictFromSchemaFilename(schemaFile) |
501 | 488 |
502 tsTemplateFile = \ | 489 tsTemplateFile = \ |
503 os.path.join(os.path.dirname(__file__), 'template.in.ts') | 490 os.path.join(os.path.dirname(__file__), 'template.in.ts') |
504 template = MakeTemplateFromFile(tsTemplateFile) | 491 template = MakeTemplateFromFile(tsTemplateFile) |
505 renderedTsCode: str = template.render(**tdico) | 492 renderedTsCode = template.render(**tdico) |
506 outputTsFile = os.path.join( \ | 493 outputTsFile = os.path.join( \ |
507 outDir,f"{tdico['rootName']}_generated.ts") | 494 outDir,str(tdico['rootName']) + "_generated.ts") |
508 with open(outputTsFile,"wt",encoding='utf8') as outFile: | 495 with open(outputTsFile,"wt",encoding='utf8') as outFile: |
509 outFile.write(renderedTsCode) | 496 outFile.write(renderedTsCode) |
510 | 497 |
511 cppTemplateFile = \ | 498 cppTemplateFile = \ |
512 os.path.join(os.path.dirname(__file__), 'template.in.h') | 499 os.path.join(os.path.dirname(__file__), 'template.in.h') |
513 template = MakeTemplateFromFile(cppTemplateFile) | 500 template = MakeTemplateFromFile(cppTemplateFile) |
514 renderedCppCode: str = template.render(**tdico) | 501 renderedCppCode = template.render(**tdico) |
515 outputCppFile = os.path.join( \ | 502 outputCppFile = os.path.join( \ |
516 outDir,f"{tdico['rootName']}_generated.hpp") | 503 outDir, str(tdico['rootName']) + "_generated.hpp") |
517 with open(outputCppFile,"wt",encoding='utf8') as outFile: | 504 with open(outputCppFile,"wt",encoding='utf8') as outFile: |
518 outFile.write(renderedCppCode) | 505 outFile.write(renderedCppCode) |
519 | |
520 | 506 |
521 # def GenEnumDecl(genc: GenCode, fullName: str, schema: Dict) -> None: | 507 # def GenEnumDecl(genc: GenCode, fullName: str, schema: Dict) -> None: |
522 # """Writes the enumerations in genc""" | 508 # """Writes the enumerations in genc""" |
523 # enumDict:Dict=schema[fullName] | 509 # enumDict:Dict=schema[fullName] |
524 # # jinja2 template | 510 # # jinja2 template |