Mercurial > hg > orthanc-stone
annotate Resources/CodeGeneration/stonegentool.py @ 493:6fbf2eae7c88 bgo-commands-codegen
All unit tests pass for generation, including handler and dispatcher
author | bgo-osimis |
---|---|
date | Fri, 22 Feb 2019 10:48:43 +0100 |
parents | 8e7e151ef472 |
children | fc17251477d6 |
rev | line source |
---|---|
471 | 1 import json |
491 | 2 import yaml |
471 | 3 import re |
473 | 4 import sys |
491 | 5 from jinja2 import Template |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
6 from typing import ( |
489 | 7 Any, |
8 Dict, | |
9 Generator, | |
10 Iterable, | |
11 Iterator, | |
12 List, | |
13 Match, | |
14 Optional, | |
15 Tuple, | |
16 Union, | |
17 cast, | |
18 ) | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
19 from io import StringIO |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
20 import time |
469 | 21 |
472 | 22 """ |
23 1 2 3 4 5 6 7 | |
24 12345678901234567890123456789012345678901234567890123456789012345678901234567890 | |
25 """ | |
26 | |
491 | 27 # see https://stackoverflow.com/a/2504457/2927708 |
28 def trim(docstring): | |
29 if not docstring: | |
30 return '' | |
31 # Convert tabs to spaces (following the normal Python rules) | |
32 # and split into a list of lines: | |
33 lines = docstring.expandtabs().splitlines() | |
34 # Determine minimum indentation (first line doesn't count): | |
35 indent = sys.maxsize | |
36 for line in lines[1:]: | |
37 stripped = line.lstrip() | |
38 if stripped: | |
39 indent = min(indent, len(line) - len(stripped)) | |
40 # Remove indentation (first line is special): | |
41 trimmed = [lines[0].strip()] | |
42 if indent < sys.maxsize: | |
43 for line in lines[1:]: | |
44 trimmed.append(line[indent:].rstrip()) | |
45 # Strip off trailing and leading blank lines: | |
46 while trimmed and not trimmed[-1]: | |
47 trimmed.pop() | |
48 while trimmed and not trimmed[0]: | |
49 trimmed.pop(0) | |
50 # Return a single string: | |
51 return '\n'.join(trimmed) | |
52 | |
53 | |
54 class GenCode: | |
489 | 55 def __init__(self): |
56 | |
57 # file-wide preamble (#include directives, comment...) | |
58 self.cppPreamble = StringIO() | |
59 | |
60 self.cppEnums = StringIO() | |
61 self.cppStructs = StringIO() | |
62 self.cppDispatcher = StringIO() | |
63 self.cppHandler = StringIO() | |
473 | 64 |
489 | 65 # file-wide preamble (module directives, comment...) |
66 self.tsPreamble = StringIO() | |
67 | |
68 self.tsEnums = StringIO() | |
69 self.tsStructs = StringIO() | |
70 self.tsDispatcher = StringIO() | |
71 self.tsHandler = StringIO() | |
486
8e40355a172b
Unit tests OK for preambles, enums and structs in both TS and C++
bgo-osimis
parents:
485
diff
changeset
|
72 |
489 | 73 def FlattenToFiles(self, outputDir: str): |
74 raise NotImplementedError() | |
75 | |
76 | |
473 | 77 class JsonHelpers: |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
78 """A set of utilities to perform JSON operations""" |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
79 |
473 | 80 @staticmethod |
81 def removeCommentsFromJsonContent(string): | |
82 """ | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
83 Remove comments from a JSON file |
473 | 84 |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
85 Comments are not allowed in JSON but, i.e., Orthanc configuration files |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
86 contains C++ like comments that we need to remove before python can |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
87 parse the file |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
88 """ |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
89 # remove all occurrence streamed comments (/*COMMENT */) from string |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
90 string = re.sub(re.compile("/\*.*?\*/", re.DOTALL), "", string) |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
91 |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
92 # remove all occurrence singleline comments (//COMMENT\n ) from string |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
93 string = re.sub(re.compile("//.*?\n"), "", string) |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
94 |
473 | 95 return string |
96 | |
97 @staticmethod | |
98 def loadJsonWithComments(path): | |
99 """ | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
100 Reads a JSON file that may contain C++ like comments |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
101 """ |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
102 with open(path, "r") as fp: |
473 | 103 fileContent = fp.read() |
104 fileContent = JsonHelpers.removeCommentsFromJsonContent(fileContent) | |
105 return json.loads(fileContent) | |
106 | |
107 | |
490 | 108 def LoadSchemaFromJson(filePath: str): |
473 | 109 return JsonHelpers.loadJsonWithComments(filePath) |
469 | 110 |
493
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
111 def CanonToCpp(canonicalTypename: str) -> str: |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
112 # C++: prefix map vector and string with std::map, std::vector and |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
113 # std::string |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
114 # replace int32 by int32_t |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
115 # replace float32 by float |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
116 # replace float64 by double |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
117 retVal: str = canonicalTypename |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
118 retVal = retVal.replace("map", "std::map") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
119 retVal = retVal.replace("vector", "std::vector") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
120 retVal = retVal.replace("int32", "int32_t") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
121 retVal = retVal.replace("float32", "float") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
122 retVal = retVal.replace("float64", "double") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
123 return retVal |
470 | 124 |
493
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
125 def CanonToTs(canonicalTypename: str) -> str: |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
126 # TS: replace vector with Array and map with Map |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
127 # string remains string |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
128 # replace int32 by number |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
129 # replace float32 by number |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
130 # replace float64 by number |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
131 retVal: str = canonicalTypename |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
132 retVal = retVal.replace("map", "Map") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
133 retVal = retVal.replace("vector", "Array") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
134 retVal = retVal.replace("int32", "number") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
135 retVal = retVal.replace("float32", "number") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
136 retVal = retVal.replace("float64", "number") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
137 retVal = retVal.replace("bool", "boolean") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
138 return retVal |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
139 |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
140 def NeedsConstruction(canonTypename): |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
141 return True |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
142 |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
143 def RegisterTemplateFunction(template,func): |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
144 """Makes a function callable by a jinja2 template""" |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
145 template.globals[func.__name__] = func |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
146 return func |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
147 |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
148 def MakeTemplate(templateStr): |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
149 template = Template(templateStr) |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
150 RegisterTemplateFunction(template,CanonToCpp) |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
151 RegisterTemplateFunction(template,CanonToTs) |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
152 RegisterTemplateFunction(template,NeedsConstruction) |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
153 return template |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
154 |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
155 def MakeTemplateFromFile(templateFileName): |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
156 templateFile = open(templateFileName, "r") |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
157 templateFileContents = templateFile.read() |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
158 return MakeTemplate(templateFileContents) |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
159 templateFile.close() |
469 | 160 |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
161 def EatToken(sentence: str) -> Tuple[str, str]: |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
162 """splits "A,B,C" into "A" and "B,C" where A, B and C are type names |
471 | 163 (including templates) like "int32", "TotoTutu", or |
164 "map<map<int32,vector<string>>,map<string,int32>>" """ | |
472 | 165 |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
166 if sentence.count("<") != sentence.count(">"): |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
167 raise Exception( |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
168 f"Error in the partial template type list {sentence}." |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
169 + " The number of < and > do not match!" |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
170 ) |
471 | 171 |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
172 # the template level we're currently in |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
173 templateLevel = 0 |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
174 for i in range(len(sentence)): |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
175 if (sentence[i] == ",") and (templateLevel == 0): |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
176 return (sentence[0:i], sentence[i + 1 :]) |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
177 elif sentence[i] == "<": |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
178 templateLevel += 1 |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
179 elif sentence[i] == ">": |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
180 templateLevel -= 1 |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
181 return (sentence, "") |
471 | 182 |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
183 |
491 | 184 def SplitListOfTypes(typename: str) -> List[str]: |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
185 """Splits something like |
471 | 186 vector<string>,int32,map<string,map<string,int32>> |
187 in: | |
188 - vector<string> | |
189 - int32 | |
190 map<string,map<string,int32>> | |
191 | |
192 This is not possible with a regex so | |
193 """ | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
194 stillStuffToEat: bool = True |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
195 tokenList = [] |
491 | 196 restOfString = typename |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
197 while stillStuffToEat: |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
198 firstToken, restOfString = EatToken(restOfString) |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
199 tokenList.append(firstToken) |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
200 if restOfString == "": |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
201 stillStuffToEat = False |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
202 return tokenList |
471 | 203 |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
204 |
490 | 205 templateRegex = \ |
206 re.compile(r"([a-zA-Z0-9_]*[a-zA-Z0-9_]*)<([a-zA-Z0-9_,:<>]+)>") | |
472 | 207 |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
208 |
491 | 209 def ParseTemplateType(typename) -> Tuple[bool, str, List[str]]: |
210 """ If the type is a template like "SOMETHING<SOME<THING,EL<SE>>>", | |
211 then it returns (true,"SOMETHING","SOME<THING,EL<SE>>") | |
470 | 212 otherwise it returns (false,"","")""" |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
213 |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
214 # let's remove all whitespace from the type |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
215 # split without argument uses any whitespace string as separator |
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
216 # (space, tab, newline, return or formfeed) |
491 | 217 typename = "".join(typename.split()) |
218 matches = templateRegex.match(typename) | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
219 if matches == None: |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
220 return (False, "", []) |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
221 else: |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
222 m = cast(Match[str], matches) |
489 | 223 assert len(m.groups()) == 2 |
224 # we need to split with the commas that are outside of the | |
225 # defined types. Simply splitting at commas won't work | |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
226 listOfDependentTypes = SplitListOfTypes(m.group(2)) |
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
227 return (True, m.group(1), listOfDependentTypes) |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
228 |
491 | 229 |
230 def ComputeOrderFromTypeTree( | |
231 ancestors: List[str], | |
232 genOrder: List[str], | |
233 shortTypename: str, schema: Dict[str, Dict]) -> None: | |
234 | |
235 if shortTypename in ancestors: | |
489 | 236 raise Exception( |
237 f"Cyclic dependency chain found: the last of {ancestors} " | |
491 | 238 + f"depends on {shortTypename} that is already in the list." |
489 | 239 ) |
240 | |
491 | 241 if not (shortTypename in genOrder): |
242 (isTemplate, _, dependentTypenames) = ParseTemplateType(shortTypename) | |
489 | 243 if isTemplate: |
491 | 244 # if it is a template, it HAS dependent types... They can be |
245 # anything (primitive, collection, enum, structs..). | |
246 # Let's process them! | |
247 for dependentTypename in dependentTypenames: | |
489 | 248 # childAncestors = ancestors.copy() NO TEMPLATE ANCESTOR!!! |
491 | 249 # childAncestors.append(typename) |
250 ComputeOrderFromTypeTree( | |
251 ancestors, genOrder, dependentTypename, schema | |
489 | 252 ) |
253 else: | |
491 | 254 # If it is not template, we are only interested if it is a |
255 # dependency that we must take into account in the dep graph, | |
256 # i.e., a struct. | |
257 if IsShortStructType(shortTypename, schema): | |
258 struct:Dict = schema[GetLongTypename(shortTypename, schema)] | |
259 # The keys in the struct dict are the member names | |
260 # The values in the struct dict are the member types | |
261 for field in struct.keys(): | |
262 # we fill the chain of dependent types (starting here) | |
263 ancestors.append(shortTypename) | |
264 ComputeOrderFromTypeTree( | |
265 ancestors, genOrder, struct[field], schema) | |
266 # don't forget to restore it! | |
267 ancestors.pop() | |
268 | |
269 # now we're pretty sure our dependencies have been processed, | |
270 # we can start marking our code for generation (it might | |
271 # already have been done if someone referenced us earlier) | |
272 if not shortTypename in genOrder: | |
273 genOrder.append(shortTypename) | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
274 |
491 | 275 # +-----------------------+ |
276 # | Utility functions | | |
277 # +-----------------------+ | |
278 | |
279 def IsShortStructType(typename: str, schema: Dict[str, Dict]) -> bool: | |
280 fullStructName = "struct " + typename | |
281 return (fullStructName in schema) | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
282 |
491 | 283 def GetLongTypename(shortTypename: str, schema: Dict): |
284 if shortTypename.startswith("enum "): | |
285 raise RuntimeError('shortTypename.startswith("enum "):') | |
286 enumName: str = "enum " + shortTypename | |
287 isEnum = enumName in schema | |
490 | 288 |
491 | 289 if shortTypename.startswith("struct "): |
290 raise RuntimeError('shortTypename.startswith("struct "):') | |
291 structName: str = "struct " + shortTypename | |
292 isStruct = ("struct " + shortTypename) in schema | |
474 | 293 |
491 | 294 if isEnum and isStruct: |
295 raise RuntimeError('Enums and structs cannot have the same name') | |
474 | 296 |
491 | 297 if isEnum: |
298 return enumName | |
299 if isStruct: | |
300 return structName | |
301 | |
302 def IsTypename(fullName: str) -> bool: | |
303 return (fullName.startswith("enum ") or fullName.startswith("struct ")) | |
304 | |
305 def IsEnumType(fullName: str) -> bool: | |
306 return fullName.startswith("enum ") | |
490 | 307 |
491 | 308 def IsStructType(fullName: str) -> bool: |
309 return fullName.startswith("struct ") | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
310 |
491 | 311 def GetShortTypename(fullTypename: str) -> str: |
312 if fullTypename.startswith("struct "): | |
313 return fullTypename[7:] | |
314 elif fullTypename.startswith("enum"): | |
315 return fullTypename[5:] | |
316 else: | |
317 raise RuntimeError \ | |
318 ('fullTypename should start with either "struct " or "enum "') | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
319 |
491 | 320 def CheckSchemaSchema(schema: Dict) -> None: |
321 if not "rootName" in schema: | |
322 raise Exception("schema lacks the 'rootName' key") | |
323 for name in schema.keys(): | |
324 if (not IsEnumType(name)) and (not IsStructType(name)) and \ | |
325 (name != 'rootName'): | |
326 raise RuntimeError \ | |
327 (f'Type "{name}" should start with "enum " or "struct "') | |
328 | |
329 # TODO: check enum fields are unique (in whole namespace) | |
330 # TODO: check struct fields are unique (in each struct) | |
493
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
331 # TODO: check that in the source schema, there are spaces after each colon |
491 | 332 |
333 # +-----------------------+ | |
334 # | Main processing logic | | |
335 # +-----------------------+ | |
336 | |
337 def ComputeRequiredDeclarationOrder(schema: dict) -> List[str]: | |
338 # sanity check | |
339 CheckSchemaSchema(schema) | |
474 | 340 |
491 | 341 # we traverse the type dependency graph and we fill a queue with |
342 # the required struct types, in a bottom-up fashion, to compute | |
343 # the declaration order | |
344 # The genOrder list contains the struct full names in the order | |
345 # where they must be defined. | |
346 # We do not care about the enums here... They do not depend upon | |
347 # anything and we'll handle them, in their original declaration | |
348 # order, at the start | |
349 genOrder: List = [] | |
350 for fullName in schema.keys(): | |
351 if IsStructType(fullName): | |
352 realName: str = GetShortTypename(fullName) | |
353 ancestors: List[str] = [] | |
354 ComputeOrderFromTypeTree(ancestors, genOrder, realName, schema) | |
355 return genOrder | |
489 | 356 |
491 | 357 def ProcessSchema(schema: dict, genOrder: List[str]) -> Dict: |
358 # sanity check | |
359 CheckSchemaSchema(schema) | |
472 | 360 |
491 | 361 # let's doctor the schema to clean it up a bit |
362 # order DOES NOT matter for enums, even though it's a list | |
363 enums: List[Dict] = [] | |
364 for fullName in schema.keys(): | |
365 if IsEnumType(fullName): | |
366 # convert "enum Toto" to "Toto" | |
367 typename:str = GetShortTypename(fullName) | |
368 enum = {} | |
369 enum['name'] = typename | |
370 assert(type(schema[fullName]) == list) | |
371 enum['fields'] = schema[fullName] # must be a list | |
372 enums.append(enum) | |
489 | 373 |
491 | 374 # now that the order has been established, we actually store\ |
375 # the structs in the correct order | |
376 # the structs are like: | |
377 # example = [ | |
378 # { | |
379 # "name": "Message1", | |
380 # "fields": { | |
381 # "someMember":"int32", | |
382 # "someOtherMember":"vector<string>" | |
383 # } | |
384 # }, | |
385 # { | |
386 # "name": "Message2", | |
387 # "fields": { | |
388 # "someMember":"int32", | |
389 # "someOtherMember22":"vector<Message1>" | |
390 # } | |
391 # } | |
392 # ] | |
489 | 393 |
491 | 394 structs: List[Dict] = [] |
395 for i in range(len(genOrder)): | |
396 # this is already the short name | |
397 typename = genOrder[i] | |
398 fieldDict = schema["struct " + typename] | |
399 struct = {} | |
400 struct['name'] = typename | |
401 struct['fields'] = fieldDict | |
402 structs.append(struct) | |
489 | 403 |
491 | 404 templatingDict = {} |
405 templatingDict['enums'] = enums | |
406 templatingDict['structs'] = structs | |
407 templatingDict['rootName'] = schema['rootName'] | |
489 | 408 |
491 | 409 return templatingDict |
489 | 410 |
491 | 411 # +-----------------------+ |
412 # | Write to files | | |
413 # +-----------------------+ | |
474 | 414 |
491 | 415 # def WriteStreamsToFiles(rootName: str, genc: Dict[str, StringIO]) \ |
416 # -> None: | |
417 # pass | |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
418 |
491 | 419 def LoadSchema(fn): |
493
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
420 # latin-1 is a trick, when we do NOT care about NON-ascii chars but |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
421 # we wish to avoid using a decoding error handler |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
422 # (see http://python-notes.curiousefficiency.org/en/latest/python3/text_file_processing.html#files-in-an-ascii-compatible-encoding-best-effort-is-acceptable) |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
423 # TL;DR: all 256 values are mapped to characters in latin-1 so the file |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
424 # contents never cause an error. |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
425 with open(fn, 'r', encoding='latin-1') as f: |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
426 schemaText = f.read() |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
427 assert(type(schemaText) == str) |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
428 # ensure there is a space after each colon. Otherwise, dicts could be |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
429 # erroneously recognized as an array of strings containing ':' |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
430 for i in range(len(schemaText)-1): |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
431 ch = schemaText[i] |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
432 nextCh = schemaText[i+1] |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
433 if ch == ':': |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
434 if not (nextCh == ' ' or nextCh == '\n'): |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
435 assert(False) |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
436 schema = yaml.load(schemaText) |
491 | 437 return schema |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
438 |
491 | 439 def GetTemplatingDictFromSchemaFilename(fn): |
440 obj = LoadSchema(fn) | |
441 genOrder: str = ComputeRequiredDeclarationOrder(obj) | |
442 templatingDict = ProcessSchema(obj, genOrder) | |
443 return templatingDict | |
470 | 444 |
491 | 445 # +-----------------------+ |
446 # | ENTRY POINT | | |
447 # +-----------------------+ | |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
448 |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
449 if __name__ == "__main__": |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
450 import argparse |
468 | 451 |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
452 parser = argparse.ArgumentParser( |
491 | 453 usage="""stonegentool.py [-h] [-o OUT_DIR] [-v] input_schema |
454 EXAMPLE: python stonegentool.py -o "generated_files/" """ | |
455 + """ "mainSchema.yaml,App Specific Commands.json" """ | |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
456 ) |
491 | 457 parser.add_argument("input_schema", type=str, \ |
458 help="path to the schema file") | |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
459 parser.add_argument( |
489 | 460 "-o", |
461 "--out_dir", | |
462 type=str, | |
463 default=".", | |
464 help="""path of the directory where the files | |
465 will be generated. Default is current | |
466 working folder""", | |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
467 ) |
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
468 parser.add_argument( |
489 | 469 "-v", |
470 "--verbosity", | |
471 action="count", | |
472 default=0, | |
473 help="""increase output verbosity (0 == errors | |
474 only, 1 == some verbosity, 2 == nerd | |
475 mode""", | |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
476 ) |
468 | 477 |
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
478 args = parser.parse_args() |
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
479 inputSchemaFilename = args.input_schema |
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
480 outDir = args.out_dir |
468 | 481 |
491 | 482 schema: Dict = LoadSchema(inputSchemaFilename) |
483 genOrder: List[str] = ComputeRequiredDeclarationOrder(schema) | |
484 processedSchema: Dict = ProcessSchema(schema,genOrder) | |
485 | |
486 | |
487 # def GenEnumDecl(genc: GenCode, fullName: str, schema: Dict) -> None: | |
488 # """Writes the enumerations in genc""" | |
489 # enumDict:Dict=schema[fullName] | |
490 # # jinja2 template | |
491 # j2cppEnum = Template(trim( | |
492 # """ {{fullName}} | |
493 # { | |
494 # {% for key in enumDict.keys()%} | |
495 # {{key}}, | |
496 # {%endfor%} | |
497 # }; | |
498 # """)) | |
499 # j2cppEnumR = j2cppEnum.render(locals()) | |
500 # genc.cppEnums.write(j2cppEnumR) | |
501 | |
502 # j2tsEnum = Template(trim( | |
503 # """ export {{fullName}} | |
504 # { | |
505 # {% for key in enumDict.keys()%} | |
506 # {{key}}, | |
507 # {%endfor%} | |
508 # }; | |
509 # """)) | |
510 # j2cppEnumR = j2cppEnum.render(locals()) | |
511 # genc.tsEnums.write(j2cppEnumR) | |
512 | |
513 | |
514 | |
515 # def GetSerializationCode(typename: str,valueName: str, tempName: str) | |
516 # if IsPrimitiveType(typename) or IsTemplateCollection(typename): | |
517 # # no need to write code for the primitive types or collections. | |
518 # # It is handled in C++ by the template functions and in TS by | |
519 # # the JSON.stringify code. | |
520 # elif IsStructType(typename): | |
521 # pass | |
522 | |
523 # def GenStructTypeDeclAndSerialize(genc: GenCode, type, schema) -> None: | |
524 # ###### | |
525 # # CPP | |
526 # ###### | |
527 # sampleCpp = """ struct Message1 | |
528 # { | |
529 # int32_t a; | |
530 # std::string b; | |
531 # EnumMonth0 c; | |
532 # bool d; | |
533 # }; | |
534 | |
535 # Json::Value StoneSerialize(const Message1& value) | |
536 # { | |
537 # Json::Value result(Json::objectValue); | |
538 # result["a"] = StoneSerialize(value.a); | |
539 # result["b"] = StoneSerialize(value.b); | |
540 # result["c"] = StoneSerialize(value.c); | |
541 # result["d"] = StoneSerialize(value.d); | |
542 # return result; | |
543 # } | |
544 # """ | |
545 | |
546 | |
547 # ###### | |
548 # # TS | |
549 # ###### | |
550 # sampleTs = """ | |
551 # { | |
552 # export class Message1 { | |
553 # a: number; | |
554 # b: string; | |
555 # c: EnumMonth0; | |
556 # d: boolean; | |
557 # public StoneSerialize(): string { | |
558 # let container: object = {}; | |
559 # container['type'] = 'Message1'; | |
560 # container['value'] = this; | |
561 # return JSON.stringify(container); | |
562 # } | |
563 # }; | |
564 # } | |
565 # """ | |
566 | |
567 | |
568 | |
569 | |
570 # tsText: StringIO = StringIO() | |
571 # cppText: StringIO = StringIO() | |
572 | |
573 # tsText.write("class %s\n" % typeDict["name"]) | |
574 # tsText.write("{\n") | |
575 | |
576 # cppText.write("struct %s\n" % typeDict["name"]) | |
577 # cppText.write("{\n") | |
578 | |
579 # """ | |
580 | |
581 # GenerateSerializationCode(typename,valueName) | |
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
582 |
491 | 583 # primitives: |
584 # ----------- | |
585 # int | |
586 # jsonValue val(objectInt); | |
587 # val.setValue("$name") | |
588 # parent.add(("$name",$name) | |
589 # double | |
590 # ... | |
591 # string | |
592 # ... | |
593 | |
594 # collections: | |
595 # ----------- | |
596 # dict { } | |
597 | |
598 # serializeValue() | |
599 # """ | |
600 | |
601 # for i in range(len(typeDict["fields"])): | |
602 # field = typeDict["fields"][i] | |
603 # name = field["name"] | |
604 # tsType = GetTypeScriptTypenameFromCanonical(field["type"]) | |
605 # tsText.write(" public %s %s;\n" % (tsType, name)) | |
606 # cppType = GetCppTypenameFromCanonical(field["type"]) | |
607 # cppText.write(" %s %s;\n" % (cppType, name)) | |
608 | |
609 # tsText.write("};\n\n") | |
610 # cppText.write("};\n\n") | |
611 | |
612 # genc.tsStructs.write(tsText.getvalue()) | |
613 # genc.cppStructs.write(cppText.getvalue()) | |
614 | |
615 | |
616 # def GenerateCodeFromTsTemplate(genc) | |
617 | |
618 | |
619 # +-----------------------+ | |
620 # | CODE GENERATION | | |
621 # +-----------------------+ | |
622 | |
623 # def GenPreambles(rootName: str, genc: GenCode) -> None: | |
624 # cppPreambleT = Template(trim( | |
625 # """// autogenerated by stonegentool on {{time.ctime()}} | |
626 # // for module {{rootName}} | |
627 # #include <cstdint> | |
628 # #include <string> | |
629 # #include <vector> | |
630 # #include <map> | |
631 # namespace {{rootName}} | |
632 # { | |
633 # Json::Value StoneSerialize(int32_t value) | |
634 # { | |
635 # Json::Value result(value); | |
636 # return result; | |
637 # } | |
638 # Json::Value StoneSerialize(double value) | |
639 # { | |
640 # Json::Value result(value); | |
641 # return result; | |
642 # } | |
643 # Json::Value StoneSerialize(bool value) | |
644 # { | |
645 # Json::Value result(value); | |
646 # return result; | |
647 # } | |
648 # Json::Value StoneSerialize(const std::string& value) | |
649 # { | |
650 # // the following is better than | |
651 # Json::Value result(value.data(),value.data()+value.size()); | |
652 # return result; | |
653 # } | |
654 # template<typename T> | |
655 # Json::Value StoneSerialize(const std::map<std::string,T>& value) | |
656 # { | |
657 # Json::Value result(Json::objectValue); | |
658 | |
659 # for (std::map<std::string, T>::const_iterator it = value.cbegin(); | |
660 # it != value.cend(); ++it) | |
661 # { | |
662 # // it->first it->second | |
663 # result[it->first] = StoneSerialize(it->second); | |
664 # } | |
665 # return result; | |
666 # } | |
667 # template<typename T> | |
668 # Json::Value StoneSerialize(const std::vector<T>& value) | |
669 # { | |
670 # Json::Value result(Json::arrayValue); | |
671 # for (size_t i = 0; i < value.size(); ++i) | |
672 # { | |
673 # result.append(StoneSerialize(value[i])); | |
674 # } | |
675 # return result; | |
676 # } | |
677 # """ | |
678 # cppPreambleR = cppPreambleT.render(locals()) | |
679 # genc.cppPreamble.write(cppPreambleR) | |
680 | |
681 # tsPreambleT = Template(trim( | |
682 # """// autogenerated by stonegentool on {{time.ctime()}} | |
683 # // for module {{rootName}} | |
684 | |
685 # namespace {{rootName}} | |
686 # { | |
687 # """ | |
688 # tsPreambleR = tsPreambleT.render(locals()) | |
689 # genc.tsPreamble.write(tsPreambleR) | |
690 | |
691 # def ComputeOrder_ProcessStruct( \ | |
692 # genOrder: List[str], name:str, schema: Dict[str, str]) -> None: | |
693 # # let's generate the code according to the | |
694 # struct = schema[name] | |
695 | |
696 # if not IsStructType(name): | |
697 # raise Exception(f'{typename} should start with "struct "') | |
493
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
698 |
6fbf2eae7c88
All unit tests pass for generation, including handler and dispatcher
bgo-osimis
parents:
491
diff
changeset
|
699 |