Mercurial > hg > orthanc-stone
comparison Resources/CodeGeneration/stonegentool.py @ 520:7a16fb9a4ba5 am-touch-events
merge bgo-commands-codegen
author | Alain Mazy <alain@mazy.be> |
---|---|
date | Tue, 12 Mar 2019 14:50:14 +0100 |
parents | 17106b29ed6d |
children | 75664eeacae5 |
comparison
equal
deleted
inserted
replaced
503:77e0eb83ff63 | 520:7a16fb9a4ba5 |
---|---|
1 import json | |
2 import yaml | |
3 import re | |
4 import os | |
5 import sys | |
6 from jinja2 import Template | |
7 from io import StringIO | |
8 import time | |
9 import datetime | |
10 | |
11 """ | |
12 1 2 3 4 5 6 7 | |
13 12345678901234567890123456789012345678901234567890123456789012345678901234567890 | |
14 """ | |
15 | |
16 # see https://stackoverflow.com/a/2504457/2927708 | |
17 def trim(docstring): | |
18 if not docstring: | |
19 return '' | |
20 # Convert tabs to spaces (following the normal Python rules) | |
21 # and split into a list of lines: | |
22 lines = docstring.expandtabs().splitlines() | |
23 # Determine minimum indentation (first line doesn't count): | |
24 indent = sys.maxsize | |
25 for line in lines[1:]: | |
26 stripped = line.lstrip() | |
27 if stripped: | |
28 indent = min(indent, len(line) - len(stripped)) | |
29 # Remove indentation (first line is special): | |
30 trimmed = [lines[0].strip()] | |
31 if indent < sys.maxsize: | |
32 for line in lines[1:]: | |
33 trimmed.append(line[indent:].rstrip()) | |
34 # Strip off trailing and leading blank lines: | |
35 while trimmed and not trimmed[-1]: | |
36 trimmed.pop() | |
37 while trimmed and not trimmed[0]: | |
38 trimmed.pop(0) | |
39 # Return a single string: | |
40 return '\n'.join(trimmed) | |
41 | |
42 class JsonHelpers: | |
43 """A set of utilities to perform JSON operations""" | |
44 | |
45 @staticmethod | |
46 def removeCommentsFromJsonContent(string): | |
47 """ | |
48 Remove comments from a JSON file | |
49 | |
50 Comments are not allowed in JSON but, i.e., Orthanc configuration files | |
51 contains C++ like comments that we need to remove before python can | |
52 parse the file | |
53 """ | |
54 # remove all occurrence streamed comments (/*COMMENT */) from string | |
55 string = re.sub(re.compile("/\*.*?\*/", re.DOTALL), "", string) | |
56 | |
57 # remove all occurrence singleline comments (//COMMENT\n ) from string | |
58 string = re.sub(re.compile("//.*?\n"), "", string) | |
59 | |
60 return string | |
61 | |
62 @staticmethod | |
63 def loadJsonWithComments(path): | |
64 """ | |
65 Reads a JSON file that may contain C++ like comments | |
66 """ | |
67 with open(path, "r") as fp: | |
68 fileContent = fp.read() | |
69 fileContent = JsonHelpers.removeCommentsFromJsonContent(fileContent) | |
70 return json.loads(fileContent) | |
71 | |
72 | |
73 def LoadSchemaFromJson(filePath): | |
74 return JsonHelpers.loadJsonWithComments(filePath) | |
75 | |
76 def CanonToCpp(canonicalTypename): | |
77 # C++: prefix map vector and string with std::map, std::vector and | |
78 # std::string | |
79 # replace int32 by int32_t | |
80 # replace float32 by float | |
81 # replace float64 by double | |
82 retVal = canonicalTypename | |
83 retVal = retVal.replace("map", "std::map") | |
84 retVal = retVal.replace("vector", "std::vector") | |
85 retVal = retVal.replace("string", "std::string") | |
86 retVal = retVal.replace("int32", "int32_t") | |
87 retVal = retVal.replace("float32", "float") | |
88 retVal = retVal.replace("float64", "double") | |
89 retVal = retVal.replace("json", "Json::Value") | |
90 return retVal | |
91 | |
92 def CanonToTs(canonicalTypename): | |
93 # TS: replace vector with Array and map with Map | |
94 # string remains string | |
95 # replace int32 by number | |
96 # replace float32 by number | |
97 # replace float64 by number | |
98 retVal = canonicalTypename | |
99 retVal = retVal.replace("map", "Map") | |
100 retVal = retVal.replace("vector", "Array") | |
101 retVal = retVal.replace("int32", "number") | |
102 retVal = retVal.replace("float32", "number") | |
103 retVal = retVal.replace("float64", "number") | |
104 retVal = retVal.replace("bool", "boolean") | |
105 retVal = retVal.replace("json", "Object") | |
106 return retVal | |
107 | |
108 def NeedsTsConstruction(enums, tsType): | |
109 if tsType == 'boolean': | |
110 return False | |
111 elif tsType == 'number': | |
112 return False | |
113 elif tsType == 'string': | |
114 return False | |
115 else: | |
116 enumNames = [] | |
117 for enum in enums: | |
118 enumNames.append(enum['name']) | |
119 if tsType in enumNames: | |
120 return False | |
121 return True | |
122 | |
123 def NeedsCppConstruction(canonTypename): | |
124 return False | |
125 | |
126 def RegisterTemplateFunction(template,func): | |
127 """Makes a function callable by a jinja2 template""" | |
128 template.globals[func.__name__] = func | |
129 return func | |
130 | |
131 def MakeTemplate(templateStr): | |
132 template = Template(templateStr) | |
133 RegisterTemplateFunction(template,CanonToCpp) | |
134 RegisterTemplateFunction(template,CanonToTs) | |
135 RegisterTemplateFunction(template,NeedsTsConstruction) | |
136 RegisterTemplateFunction(template,NeedsCppConstruction) | |
137 return template | |
138 | |
139 def MakeTemplateFromFile(templateFileName): | |
140 templateFile = open(templateFileName, "r") | |
141 templateFileContents = templateFile.read() | |
142 return MakeTemplate(templateFileContents) | |
143 templateFile.close() | |
144 | |
145 def EatToken(sentence): | |
146 """splits "A,B,C" into "A" and "B,C" where A, B and C are type names | |
147 (including templates) like "int32", "TotoTutu", or | |
148 "map<map<int32,vector<string>>,map<string,int32>>" """ | |
149 | |
150 if sentence.count("<") != sentence.count(">"): | |
151 raise Exception( | |
152 "Error in the partial template type list " + str(sentence) + "." | |
153 + " The number of < and > do not match!" | |
154 ) | |
155 | |
156 # the template level we're currently in | |
157 templateLevel = 0 | |
158 for i in range(len(sentence)): | |
159 if (sentence[i] == ",") and (templateLevel == 0): | |
160 return (sentence[0:i], sentence[i + 1 :]) | |
161 elif sentence[i] == "<": | |
162 templateLevel += 1 | |
163 elif sentence[i] == ">": | |
164 templateLevel -= 1 | |
165 return (sentence, "") | |
166 | |
167 | |
168 def SplitListOfTypes(typename): | |
169 """Splits something like | |
170 vector<string>,int32,map<string,map<string,int32>> | |
171 in: | |
172 - vector<string> | |
173 - int32 | |
174 map<string,map<string,int32>> | |
175 | |
176 This is not possible with a regex so | |
177 """ | |
178 stillStuffToEat = True | |
179 tokenList = [] | |
180 restOfString = typename | |
181 while stillStuffToEat: | |
182 firstToken, restOfString = EatToken(restOfString) | |
183 tokenList.append(firstToken) | |
184 if restOfString == "": | |
185 stillStuffToEat = False | |
186 return tokenList | |
187 | |
188 | |
189 templateRegex = \ | |
190 re.compile(r"([a-zA-Z0-9_]*[a-zA-Z0-9_]*)<([a-zA-Z0-9_,:<>]+)>") | |
191 | |
192 | |
193 def ParseTemplateType(typename): | |
194 """ If the type is a template like "SOMETHING<SOME<THING,EL<SE>>>", | |
195 then it returns (true,"SOMETHING","SOME<THING,EL<SE>>") | |
196 otherwise it returns (false,"","")""" | |
197 | |
198 # let's remove all whitespace from the type | |
199 # split without argument uses any whitespace string as separator | |
200 # (space, tab, newline, return or formfeed) | |
201 typename = "".join(typename.split()) | |
202 matches = templateRegex.match(typename) | |
203 if matches == None: | |
204 return (False, "", []) | |
205 else: | |
206 m = matches | |
207 assert len(m.groups()) == 2 | |
208 # we need to split with the commas that are outside of the | |
209 # defined types. Simply splitting at commas won't work | |
210 listOfDependentTypes = SplitListOfTypes(m.group(2)) | |
211 return (True, m.group(1), listOfDependentTypes) | |
212 | |
213 def GetStructFields(struct): | |
214 """This filters out the special metadata key from the struct fields""" | |
215 return [k for k in struct.keys() if k != '__handler'] | |
216 | |
217 def ComputeOrderFromTypeTree( | |
218 ancestors, | |
219 genOrder, | |
220 shortTypename, schema): | |
221 | |
222 if shortTypename in ancestors: | |
223 raise Exception( | |
224 "Cyclic dependency chain found: the last of " + str(ancestors) + | |
225 + " depends on " + str(shortTypename) + " that is already in the list." | |
226 ) | |
227 | |
228 if not (shortTypename in genOrder): | |
229 (isTemplate, _, dependentTypenames) = ParseTemplateType(shortTypename) | |
230 if isTemplate: | |
231 # if it is a template, it HAS dependent types... They can be | |
232 # anything (primitive, collection, enum, structs..). | |
233 # Let's process them! | |
234 for dependentTypename in dependentTypenames: | |
235 # childAncestors = ancestors.copy() NO TEMPLATE ANCESTOR!!! | |
236 # childAncestors.append(typename) | |
237 ComputeOrderFromTypeTree( | |
238 ancestors, genOrder, dependentTypename, schema | |
239 ) | |
240 else: | |
241 # If it is not template, we are only interested if it is a | |
242 # dependency that we must take into account in the dep graph, | |
243 # i.e., a struct. | |
244 if IsShortStructType(shortTypename, schema): | |
245 struct = schema[GetLongTypename(shortTypename, schema)] | |
246 # The keys in the struct dict are the member names | |
247 # The values in the struct dict are the member types | |
248 if struct: | |
249 # we reach this if struct is not None AND not empty | |
250 for field in GetStructFields(struct): | |
251 # we fill the chain of dependent types (starting here) | |
252 ancestors.append(shortTypename) | |
253 ComputeOrderFromTypeTree( | |
254 ancestors, genOrder, struct[field], schema) | |
255 # don't forget to restore it! | |
256 ancestors.pop() | |
257 | |
258 # now we're pretty sure our dependencies have been processed, | |
259 # we can start marking our code for generation (it might | |
260 # already have been done if someone referenced us earlier) | |
261 if not shortTypename in genOrder: | |
262 genOrder.append(shortTypename) | |
263 | |
264 # +-----------------------+ | |
265 # | Utility functions | | |
266 # +-----------------------+ | |
267 | |
268 def IsShortStructType(typename, schema): | |
269 fullStructName = "struct " + typename | |
270 return (fullStructName in schema) | |
271 | |
272 def GetLongTypename(shortTypename, schema): | |
273 if shortTypename.startswith("enum "): | |
274 raise RuntimeError('shortTypename.startswith("enum "):') | |
275 enumName = "enum " + shortTypename | |
276 isEnum = enumName in schema | |
277 | |
278 if shortTypename.startswith("struct "): | |
279 raise RuntimeError('shortTypename.startswith("struct "):') | |
280 structName = "struct " + shortTypename | |
281 isStruct = ("struct " + shortTypename) in schema | |
282 | |
283 if isEnum and isStruct: | |
284 raise RuntimeError('Enums and structs cannot have the same name') | |
285 | |
286 if isEnum: | |
287 return enumName | |
288 if isStruct: | |
289 return structName | |
290 | |
291 def IsTypename(fullName): | |
292 return (fullName.startswith("enum ") or fullName.startswith("struct ")) | |
293 | |
294 def IsEnumType(fullName): | |
295 return fullName.startswith("enum ") | |
296 | |
297 def IsStructType(fullName): | |
298 return fullName.startswith("struct ") | |
299 | |
300 def GetShortTypename(fullTypename): | |
301 if fullTypename.startswith("struct "): | |
302 return fullTypename[7:] | |
303 elif fullTypename.startswith("enum"): | |
304 return fullTypename[5:] | |
305 else: | |
306 raise RuntimeError \ | |
307 ('fullTypename should start with either "struct " or "enum "') | |
308 | |
309 def CheckSchemaSchema(schema): | |
310 if not "rootName" in schema: | |
311 raise Exception("schema lacks the 'rootName' key") | |
312 for name in schema.keys(): | |
313 if (not IsEnumType(name)) and (not IsStructType(name)) and \ | |
314 (name != 'rootName'): | |
315 raise RuntimeError \ | |
316 ('Type "' + str(name) + '" should start with "enum " or "struct "') | |
317 | |
318 # TODO: check enum fields are unique (in whole namespace) | |
319 # TODO: check struct fields are unique (in each struct) | |
320 # TODO: check that in the source schema, there are spaces after each colon | |
321 | |
322 nonTypeKeys = ['rootName'] | |
323 def GetTypesInSchema(schema): | |
324 """Returns the top schema keys that are actual type names""" | |
325 typeList = [k for k in schema if k not in nonTypeKeys] | |
326 return typeList | |
327 | |
328 # +-----------------------+ | |
329 # | Main processing logic | | |
330 # +-----------------------+ | |
331 | |
332 def ComputeRequiredDeclarationOrder(schema): | |
333 # sanity check | |
334 CheckSchemaSchema(schema) | |
335 | |
336 # we traverse the type dependency graph and we fill a queue with | |
337 # the required struct types, in a bottom-up fashion, to compute | |
338 # the declaration order | |
339 # The genOrder list contains the struct full names in the order | |
340 # where they must be defined. | |
341 # We do not care about the enums here... They do not depend upon | |
342 # anything and we'll handle them, in their original declaration | |
343 # order, at the start | |
344 genOrder = [] | |
345 for fullName in GetTypesInSchema(schema): | |
346 if IsStructType(fullName): | |
347 realName = GetShortTypename(fullName) | |
348 ancestors = [] | |
349 ComputeOrderFromTypeTree(ancestors, genOrder, realName, schema) | |
350 return genOrder | |
351 | |
352 def GetStructFields(fieldDict): | |
353 """Returns the regular (non __handler) struct fields""" | |
354 # the following happens for empty structs | |
355 if fieldDict == None: | |
356 return fieldDict | |
357 ret = {} | |
358 for k,v in fieldDict.items(): | |
359 if k != "__handler": | |
360 ret[k] = v | |
361 if k.startswith("__") and k != "__handler": | |
362 raise RuntimeError("Fields starting with __ (double underscore) are reserved names!") | |
363 return ret | |
364 | |
365 def GetStructMetadata(fieldDict): | |
366 """Returns the __handler struct fields (there are default values that | |
367 can be overridden by entries in the schema | |
368 Not tested because it's a fail-safe: if something is broken in this, | |
369 dependent projects will not build.""" | |
370 metadataDict = {} | |
371 metadataDict['handleInCpp'] = False | |
372 metadataDict['handleInTypescript'] = False | |
373 | |
374 if fieldDict != None: | |
375 for k,v in fieldDict.items(): | |
376 if k.startswith("__") and k != "__handler": | |
377 raise RuntimeError("Fields starting with __ (double underscore) are reserved names") | |
378 if k == "__handler": | |
379 if type(v) == list: | |
380 for i in v: | |
381 if i == "cpp": | |
382 metadataDict['handleInCpp'] = True | |
383 elif i == "ts": | |
384 metadataDict['handleInTypescript'] = True | |
385 else: | |
386 raise RuntimeError("Error in schema. Allowed values for __handler are \"cpp\" or \"ts\"") | |
387 elif type(v) == str: | |
388 if v == "cpp": | |
389 metadataDict['handleInCpp'] = True | |
390 elif v == "ts": | |
391 metadataDict['handleInTypescript'] = True | |
392 else: | |
393 raise RuntimeError("Error in schema. Allowed values for __handler are \"cpp\" or \"ts\" (or a list of both)") | |
394 else: | |
395 raise RuntimeError("Error in schema. Allowed values for __handler are \"cpp\" or \"ts\" (or a list of both)") | |
396 return metadataDict | |
397 | |
398 def ProcessSchema(schema, genOrder): | |
399 # sanity check | |
400 CheckSchemaSchema(schema) | |
401 | |
402 # let's doctor the schema to clean it up a bit | |
403 # order DOES NOT matter for enums, even though it's a list | |
404 enums = [] | |
405 for fullName in schema.keys(): | |
406 if IsEnumType(fullName): | |
407 # convert "enum Toto" to "Toto" | |
408 typename = GetShortTypename(fullName) | |
409 enum = {} | |
410 enum['name'] = typename | |
411 assert(type(schema[fullName]) == list) | |
412 enum['fields'] = schema[fullName] # must be a list | |
413 enums.append(enum) | |
414 | |
415 # now that the order has been established, we actually store\ | |
416 # the structs in the correct order | |
417 # the structs are like: | |
418 # example = [ | |
419 # { | |
420 # "name": "Message1", | |
421 # "fields": { | |
422 # "someMember":"int32", | |
423 # "someOtherMember":"vector<string>" | |
424 # } | |
425 # }, | |
426 # { | |
427 # "name": "Message2", | |
428 # "fields": { | |
429 # "someMember":"int32", | |
430 # "someOtherMember22":"vector<Message1>" | |
431 # } | |
432 # } | |
433 # ] | |
434 | |
435 structs = [] | |
436 for i in range(len(genOrder)): | |
437 # this is already the short name | |
438 typename = genOrder[i] | |
439 fieldDict = schema["struct " + typename] | |
440 struct = {} | |
441 struct['name'] = typename | |
442 struct['fields'] = GetStructFields(fieldDict) | |
443 struct['__meta__'] = GetStructMetadata(fieldDict) | |
444 structs.append(struct) | |
445 | |
446 templatingDict = {} | |
447 templatingDict['enums'] = enums | |
448 templatingDict['structs'] = structs | |
449 templatingDict['rootName'] = schema['rootName'] | |
450 | |
451 return templatingDict | |
452 | |
453 # +-----------------------+ | |
454 # | Write to files | | |
455 # +-----------------------+ | |
456 | |
457 # def WriteStreamsToFiles(rootName: str, genc: Dict[str, StringIO]) \ | |
458 # -> None: | |
459 # pass | |
460 | |
461 def LoadSchema(fn): | |
462 # latin-1 is a trick, when we do NOT care about NON-ascii chars but | |
463 # we wish to avoid using a decoding error handler | |
464 # (see http://python-notes.curiousefficiency.org/en/latest/python3/text_file_processing.html#files-in-an-ascii-compatible-encoding-best-effort-is-acceptable) | |
465 # TL;DR: all 256 values are mapped to characters in latin-1 so the file | |
466 # contents never cause an error. | |
467 with open(fn, 'r', encoding='latin-1') as f: | |
468 schemaText = f.read() | |
469 assert(type(schemaText) == str) | |
470 # ensure there is a space after each colon. Otherwise, dicts could be | |
471 # erroneously recognized as an array of strings containing ':' | |
472 for i in range(len(schemaText)-1): | |
473 ch = schemaText[i] | |
474 nextCh = schemaText[i+1] | |
475 if ch == ':': | |
476 if not (nextCh == ' ' or nextCh == '\n'): | |
477 lineNumber = schemaText.count("\n",0,i) + 1 | |
478 raise RuntimeError("Error at line " + str(lineNumber) + " in the schema: colons must be followed by a space or a newline!") | |
479 schema = yaml.load(schemaText) | |
480 return schema | |
481 | |
482 def GetTemplatingDictFromSchemaFilename(fn): | |
483 obj = LoadSchema(fn) | |
484 genOrder = ComputeRequiredDeclarationOrder(obj) | |
485 templatingDict = ProcessSchema(obj, genOrder) | |
486 currentDT = datetime.datetime.now() | |
487 templatingDict['currentDatetime'] = str(currentDT) | |
488 return templatingDict | |
489 | |
490 # +-----------------------+ | |
491 # | ENTRY POINT | | |
492 # +-----------------------+ | |
493 def Process(schemaFile, outDir): | |
494 tdico = GetTemplatingDictFromSchemaFilename(schemaFile) | |
495 | |
496 tsTemplateFile = \ | |
497 os.path.join(os.path.dirname(__file__), 'template.in.ts.j2') | |
498 template = MakeTemplateFromFile(tsTemplateFile) | |
499 renderedTsCode = template.render(**tdico) | |
500 outputTsFile = os.path.join( \ | |
501 outDir,str(tdico['rootName']) + "_generated.ts") | |
502 with open(outputTsFile,"wt",encoding='utf8') as outFile: | |
503 outFile.write(renderedTsCode) | |
504 | |
505 cppTemplateFile = \ | |
506 os.path.join(os.path.dirname(__file__), 'template.in.h.j2') | |
507 template = MakeTemplateFromFile(cppTemplateFile) | |
508 renderedCppCode = template.render(**tdico) | |
509 outputCppFile = os.path.join( \ | |
510 outDir, str(tdico['rootName']) + "_generated.hpp") | |
511 with open(outputCppFile,"wt",encoding='utf8') as outFile: | |
512 outFile.write(renderedCppCode) | |
513 | |
514 if __name__ == "__main__": | |
515 import argparse | |
516 | |
517 parser = argparse.ArgumentParser( | |
518 usage="""stonegentool.py [-h] [-o OUT_DIR] [-v] input_schema | |
519 EXAMPLE: python stonegentool.py -o "generated_files/" """ | |
520 + """ "mainSchema.yaml,App Specific Commands.json" """ | |
521 ) | |
522 parser.add_argument("input_schema", type=str, \ | |
523 help="path to the schema file") | |
524 parser.add_argument( | |
525 "-o", | |
526 "--out_dir", | |
527 type=str, | |
528 default=".", | |
529 help="""path of the directory where the files | |
530 will be generated. Default is current | |
531 working folder""", | |
532 ) | |
533 parser.add_argument( | |
534 "-v", | |
535 "--verbosity", | |
536 action="count", | |
537 default=0, | |
538 help="""increase output verbosity (0 == errors | |
539 only, 1 == some verbosity, 2 == nerd | |
540 mode""", | |
541 ) | |
542 | |
543 args = parser.parse_args() | |
544 schemaFile = args.input_schema | |
545 outDir = args.out_dir | |
546 Process(schemaFile, outDir) |