Mercurial > hg > orthanc-stone
annotate Resources/CodeGeneration/stonegentool.py @ 491:8e7e151ef472 bgo-commands-codegen
Unit tests pass for enum generation
| author | bgo-osimis |
|---|---|
| date | Wed, 20 Feb 2019 20:51:30 +0100 |
| parents | 6470248790db |
| children | 6fbf2eae7c88 |
| rev | line source |
|---|---|
| 471 | 1 import json |
| 491 | 2 import yaml |
| 471 | 3 import re |
| 473 | 4 import sys |
| 491 | 5 from jinja2 import Template |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
6 from typing import ( |
| 489 | 7 Any, |
| 8 Dict, | |
| 9 Generator, | |
| 10 Iterable, | |
| 11 Iterator, | |
| 12 List, | |
| 13 Match, | |
| 14 Optional, | |
| 15 Tuple, | |
| 16 Union, | |
| 17 cast, | |
| 18 ) | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
19 from io import StringIO |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
20 import time |
| 469 | 21 |
| 472 | 22 """ |
| 23 1 2 3 4 5 6 7 | |
| 24 12345678901234567890123456789012345678901234567890123456789012345678901234567890 | |
| 25 """ | |
| 26 | |
| 491 | 27 # see https://stackoverflow.com/a/2504457/2927708 |
| 28 def trim(docstring): | |
| 29 if not docstring: | |
| 30 return '' | |
| 31 # Convert tabs to spaces (following the normal Python rules) | |
| 32 # and split into a list of lines: | |
| 33 lines = docstring.expandtabs().splitlines() | |
| 34 # Determine minimum indentation (first line doesn't count): | |
| 35 indent = sys.maxsize | |
| 36 for line in lines[1:]: | |
| 37 stripped = line.lstrip() | |
| 38 if stripped: | |
| 39 indent = min(indent, len(line) - len(stripped)) | |
| 40 # Remove indentation (first line is special): | |
| 41 trimmed = [lines[0].strip()] | |
| 42 if indent < sys.maxsize: | |
| 43 for line in lines[1:]: | |
| 44 trimmed.append(line[indent:].rstrip()) | |
| 45 # Strip off trailing and leading blank lines: | |
| 46 while trimmed and not trimmed[-1]: | |
| 47 trimmed.pop() | |
| 48 while trimmed and not trimmed[0]: | |
| 49 trimmed.pop(0) | |
| 50 # Return a single string: | |
| 51 return '\n'.join(trimmed) | |
| 52 | |
| 53 | |
| 54 class GenCode: | |
| 489 | 55 def __init__(self): |
| 56 | |
| 57 # file-wide preamble (#include directives, comment...) | |
| 58 self.cppPreamble = StringIO() | |
| 59 | |
| 60 self.cppEnums = StringIO() | |
| 61 self.cppStructs = StringIO() | |
| 62 self.cppDispatcher = StringIO() | |
| 63 self.cppHandler = StringIO() | |
| 473 | 64 |
| 489 | 65 # file-wide preamble (module directives, comment...) |
| 66 self.tsPreamble = StringIO() | |
| 67 | |
| 68 self.tsEnums = StringIO() | |
| 69 self.tsStructs = StringIO() | |
| 70 self.tsDispatcher = StringIO() | |
| 71 self.tsHandler = StringIO() | |
|
486
8e40355a172b
Unit tests OK for preambles, enums and structs in both TS and C++
bgo-osimis
parents:
485
diff
changeset
|
72 |
| 489 | 73 def FlattenToFiles(self, outputDir: str): |
| 74 raise NotImplementedError() | |
| 75 | |
| 76 | |
| 473 | 77 class JsonHelpers: |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
78 """A set of utilities to perform JSON operations""" |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
79 |
| 473 | 80 @staticmethod |
| 81 def removeCommentsFromJsonContent(string): | |
| 82 """ | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
83 Remove comments from a JSON file |
| 473 | 84 |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
85 Comments are not allowed in JSON but, i.e., Orthanc configuration files |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
86 contains C++ like comments that we need to remove before python can |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
87 parse the file |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
88 """ |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
89 # remove all occurrence streamed comments (/*COMMENT */) from string |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
90 string = re.sub(re.compile("/\*.*?\*/", re.DOTALL), "", string) |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
91 |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
92 # remove all occurrence singleline comments (//COMMENT\n ) from string |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
93 string = re.sub(re.compile("//.*?\n"), "", string) |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
94 |
| 473 | 95 return string |
| 96 | |
| 97 @staticmethod | |
| 98 def loadJsonWithComments(path): | |
| 99 """ | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
100 Reads a JSON file that may contain C++ like comments |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
101 """ |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
102 with open(path, "r") as fp: |
| 473 | 103 fileContent = fp.read() |
| 104 fileContent = JsonHelpers.removeCommentsFromJsonContent(fileContent) | |
| 105 return json.loads(fileContent) | |
| 106 | |
| 107 | |
| 490 | 108 def LoadSchemaFromJson(filePath: str): |
| 473 | 109 return JsonHelpers.loadJsonWithComments(filePath) |
| 469 | 110 |
| 491 | 111 def GetCppTypenameFromCanonical(canonicalTypename: str) -> str: |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
112 # C++: prefix map vector and string with std::map, std::vector and |
| 472 | 113 # std::string |
| 469 | 114 # replace int32 by int32_t |
| 115 # replace float32 by float | |
| 116 # replace float64 by double | |
| 491 | 117 retVal: str = canonicalTypename |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
118 retVal = retVal.replace("map", "std::map") |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
119 retVal = retVal.replace("vector", "std::vector") |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
120 retVal = retVal.replace("int32", "int32_t") |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
121 retVal = retVal.replace("float32", "float") |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
122 retVal = retVal.replace("float64", "double") |
| 469 | 123 return retVal |
| 470 | 124 |
| 491 | 125 def GetTypeScriptTypenameFromCanonical(canonicalTypename: str) -> str: |
| 469 | 126 # TS: replace vector with Array and map with Map |
| 127 # string remains string | |
| 128 # replace int32 by number | |
| 129 # replace float32 by number | |
| 130 # replace float64 by number | |
| 491 | 131 retVal: str = canonicalTypename |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
132 retVal = retVal.replace("map", "Map") |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
133 retVal = retVal.replace("vector", "Array") |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
134 retVal = retVal.replace("int32", "number") |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
135 retVal = retVal.replace("float32", "number") |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
136 retVal = retVal.replace("float64", "number") |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
137 retVal = retVal.replace("bool", "boolean") |
| 469 | 138 return retVal |
| 139 | |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
140 def EatToken(sentence: str) -> Tuple[str, str]: |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
141 """splits "A,B,C" into "A" and "B,C" where A, B and C are type names |
| 471 | 142 (including templates) like "int32", "TotoTutu", or |
| 143 "map<map<int32,vector<string>>,map<string,int32>>" """ | |
| 472 | 144 |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
145 if sentence.count("<") != sentence.count(">"): |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
146 raise Exception( |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
147 f"Error in the partial template type list {sentence}." |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
148 + " The number of < and > do not match!" |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
149 ) |
| 471 | 150 |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
151 # the template level we're currently in |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
152 templateLevel = 0 |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
153 for i in range(len(sentence)): |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
154 if (sentence[i] == ",") and (templateLevel == 0): |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
155 return (sentence[0:i], sentence[i + 1 :]) |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
156 elif sentence[i] == "<": |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
157 templateLevel += 1 |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
158 elif sentence[i] == ">": |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
159 templateLevel -= 1 |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
160 return (sentence, "") |
| 471 | 161 |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
162 |
| 491 | 163 def SplitListOfTypes(typename: str) -> List[str]: |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
164 """Splits something like |
| 471 | 165 vector<string>,int32,map<string,map<string,int32>> |
| 166 in: | |
| 167 - vector<string> | |
| 168 - int32 | |
| 169 map<string,map<string,int32>> | |
| 170 | |
| 171 This is not possible with a regex so | |
| 172 """ | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
173 stillStuffToEat: bool = True |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
174 tokenList = [] |
| 491 | 175 restOfString = typename |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
176 while stillStuffToEat: |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
177 firstToken, restOfString = EatToken(restOfString) |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
178 tokenList.append(firstToken) |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
179 if restOfString == "": |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
180 stillStuffToEat = False |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
181 return tokenList |
| 471 | 182 |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
183 |
| 490 | 184 templateRegex = \ |
| 185 re.compile(r"([a-zA-Z0-9_]*[a-zA-Z0-9_]*)<([a-zA-Z0-9_,:<>]+)>") | |
| 472 | 186 |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
187 |
| 491 | 188 def ParseTemplateType(typename) -> Tuple[bool, str, List[str]]: |
| 189 """ If the type is a template like "SOMETHING<SOME<THING,EL<SE>>>", | |
| 190 then it returns (true,"SOMETHING","SOME<THING,EL<SE>>") | |
| 470 | 191 otherwise it returns (false,"","")""" |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
192 |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
193 # let's remove all whitespace from the type |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
194 # split without argument uses any whitespace string as separator |
|
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
195 # (space, tab, newline, return or formfeed) |
| 491 | 196 typename = "".join(typename.split()) |
| 197 matches = templateRegex.match(typename) | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
198 if matches == None: |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
199 return (False, "", []) |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
200 else: |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
201 m = cast(Match[str], matches) |
| 489 | 202 assert len(m.groups()) == 2 |
| 203 # we need to split with the commas that are outside of the | |
| 204 # defined types. Simply splitting at commas won't work | |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
205 listOfDependentTypes = SplitListOfTypes(m.group(2)) |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
206 return (True, m.group(1), listOfDependentTypes) |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
207 |
| 491 | 208 |
| 209 def ComputeOrderFromTypeTree( | |
| 210 ancestors: List[str], | |
| 211 genOrder: List[str], | |
| 212 shortTypename: str, schema: Dict[str, Dict]) -> None: | |
| 213 | |
| 214 if shortTypename in ancestors: | |
| 489 | 215 raise Exception( |
| 216 f"Cyclic dependency chain found: the last of {ancestors} " | |
| 491 | 217 + f"depends on {shortTypename} that is already in the list." |
| 489 | 218 ) |
| 219 | |
| 491 | 220 if not (shortTypename in genOrder): |
| 221 (isTemplate, _, dependentTypenames) = ParseTemplateType(shortTypename) | |
| 489 | 222 if isTemplate: |
| 491 | 223 # if it is a template, it HAS dependent types... They can be |
| 224 # anything (primitive, collection, enum, structs..). | |
| 225 # Let's process them! | |
| 226 for dependentTypename in dependentTypenames: | |
| 489 | 227 # childAncestors = ancestors.copy() NO TEMPLATE ANCESTOR!!! |
| 491 | 228 # childAncestors.append(typename) |
| 229 ComputeOrderFromTypeTree( | |
| 230 ancestors, genOrder, dependentTypename, schema | |
| 489 | 231 ) |
| 232 else: | |
| 491 | 233 # If it is not template, we are only interested if it is a |
| 234 # dependency that we must take into account in the dep graph, | |
| 235 # i.e., a struct. | |
| 236 if IsShortStructType(shortTypename, schema): | |
| 237 struct:Dict = schema[GetLongTypename(shortTypename, schema)] | |
| 238 # The keys in the struct dict are the member names | |
| 239 # The values in the struct dict are the member types | |
| 240 for field in struct.keys(): | |
| 241 # we fill the chain of dependent types (starting here) | |
| 242 ancestors.append(shortTypename) | |
| 243 ComputeOrderFromTypeTree( | |
| 244 ancestors, genOrder, struct[field], schema) | |
| 245 # don't forget to restore it! | |
| 246 ancestors.pop() | |
| 247 | |
| 248 # now we're pretty sure our dependencies have been processed, | |
| 249 # we can start marking our code for generation (it might | |
| 250 # already have been done if someone referenced us earlier) | |
| 251 if not shortTypename in genOrder: | |
| 252 genOrder.append(shortTypename) | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
253 |
| 491 | 254 # +-----------------------+ |
| 255 # | Utility functions | | |
| 256 # +-----------------------+ | |
| 257 | |
| 258 def IsShortStructType(typename: str, schema: Dict[str, Dict]) -> bool: | |
| 259 fullStructName = "struct " + typename | |
| 260 return (fullStructName in schema) | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
261 |
| 491 | 262 def GetLongTypename(shortTypename: str, schema: Dict): |
| 263 if shortTypename.startswith("enum "): | |
| 264 raise RuntimeError('shortTypename.startswith("enum "):') | |
| 265 enumName: str = "enum " + shortTypename | |
| 266 isEnum = enumName in schema | |
| 490 | 267 |
| 491 | 268 if shortTypename.startswith("struct "): |
| 269 raise RuntimeError('shortTypename.startswith("struct "):') | |
| 270 structName: str = "struct " + shortTypename | |
| 271 isStruct = ("struct " + shortTypename) in schema | |
| 474 | 272 |
| 491 | 273 if isEnum and isStruct: |
| 274 raise RuntimeError('Enums and structs cannot have the same name') | |
| 474 | 275 |
| 491 | 276 if isEnum: |
| 277 return enumName | |
| 278 if isStruct: | |
| 279 return structName | |
| 280 | |
| 281 def IsTypename(fullName: str) -> bool: | |
| 282 return (fullName.startswith("enum ") or fullName.startswith("struct ")) | |
| 283 | |
| 284 def IsEnumType(fullName: str) -> bool: | |
| 285 return fullName.startswith("enum ") | |
| 490 | 286 |
| 491 | 287 def IsStructType(fullName: str) -> bool: |
| 288 return fullName.startswith("struct ") | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
289 |
| 491 | 290 def GetShortTypename(fullTypename: str) -> str: |
| 291 if fullTypename.startswith("struct "): | |
| 292 return fullTypename[7:] | |
| 293 elif fullTypename.startswith("enum"): | |
| 294 return fullTypename[5:] | |
| 295 else: | |
| 296 raise RuntimeError \ | |
| 297 ('fullTypename should start with either "struct " or "enum "') | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
298 |
| 491 | 299 def CheckSchemaSchema(schema: Dict) -> None: |
| 300 if not "rootName" in schema: | |
| 301 raise Exception("schema lacks the 'rootName' key") | |
| 302 for name in schema.keys(): | |
| 303 if (not IsEnumType(name)) and (not IsStructType(name)) and \ | |
| 304 (name != 'rootName'): | |
| 305 raise RuntimeError \ | |
| 306 (f'Type "{name}" should start with "enum " or "struct "') | |
| 307 | |
| 308 # TODO: check enum fields are unique (in whole namespace) | |
| 309 # TODO: check struct fields are unique (in each struct) | |
| 310 | |
| 311 # +-----------------------+ | |
| 312 # | Main processing logic | | |
| 313 # +-----------------------+ | |
| 314 | |
| 315 def ComputeRequiredDeclarationOrder(schema: dict) -> List[str]: | |
| 316 # sanity check | |
| 317 CheckSchemaSchema(schema) | |
| 474 | 318 |
| 491 | 319 # we traverse the type dependency graph and we fill a queue with |
| 320 # the required struct types, in a bottom-up fashion, to compute | |
| 321 # the declaration order | |
| 322 # The genOrder list contains the struct full names in the order | |
| 323 # where they must be defined. | |
| 324 # We do not care about the enums here... They do not depend upon | |
| 325 # anything and we'll handle them, in their original declaration | |
| 326 # order, at the start | |
| 327 genOrder: List = [] | |
| 328 for fullName in schema.keys(): | |
| 329 if IsStructType(fullName): | |
| 330 realName: str = GetShortTypename(fullName) | |
| 331 ancestors: List[str] = [] | |
| 332 ComputeOrderFromTypeTree(ancestors, genOrder, realName, schema) | |
| 333 return genOrder | |
| 489 | 334 |
| 491 | 335 def ProcessSchema(schema: dict, genOrder: List[str]) -> Dict: |
| 336 # sanity check | |
| 337 CheckSchemaSchema(schema) | |
| 472 | 338 |
| 491 | 339 # let's doctor the schema to clean it up a bit |
| 340 # order DOES NOT matter for enums, even though it's a list | |
| 341 enums: List[Dict] = [] | |
| 342 for fullName in schema.keys(): | |
| 343 if IsEnumType(fullName): | |
| 344 # convert "enum Toto" to "Toto" | |
| 345 typename:str = GetShortTypename(fullName) | |
| 346 enum = {} | |
| 347 enum['name'] = typename | |
| 348 assert(type(schema[fullName]) == list) | |
| 349 enum['fields'] = schema[fullName] # must be a list | |
| 350 enums.append(enum) | |
| 489 | 351 |
| 491 | 352 # now that the order has been established, we actually store\ |
| 353 # the structs in the correct order | |
| 354 # the structs are like: | |
| 355 # example = [ | |
| 356 # { | |
| 357 # "name": "Message1", | |
| 358 # "fields": { | |
| 359 # "someMember":"int32", | |
| 360 # "someOtherMember":"vector<string>" | |
| 361 # } | |
| 362 # }, | |
| 363 # { | |
| 364 # "name": "Message2", | |
| 365 # "fields": { | |
| 366 # "someMember":"int32", | |
| 367 # "someOtherMember22":"vector<Message1>" | |
| 368 # } | |
| 369 # } | |
| 370 # ] | |
| 489 | 371 |
| 491 | 372 structs: List[Dict] = [] |
| 373 for i in range(len(genOrder)): | |
| 374 # this is already the short name | |
| 375 typename = genOrder[i] | |
| 376 fieldDict = schema["struct " + typename] | |
| 377 struct = {} | |
| 378 struct['name'] = typename | |
| 379 struct['fields'] = fieldDict | |
| 380 structs.append(struct) | |
| 489 | 381 |
| 491 | 382 templatingDict = {} |
| 383 templatingDict['enums'] = enums | |
| 384 templatingDict['structs'] = structs | |
| 385 templatingDict['rootName'] = schema['rootName'] | |
| 489 | 386 |
| 491 | 387 return templatingDict |
| 489 | 388 |
| 491 | 389 # +-----------------------+ |
| 390 # | Write to files | | |
| 391 # +-----------------------+ | |
| 474 | 392 |
| 491 | 393 # def WriteStreamsToFiles(rootName: str, genc: Dict[str, StringIO]) \ |
| 394 # -> None: | |
| 395 # pass | |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
396 |
| 491 | 397 def LoadSchema(fn): |
| 398 with open(fn, 'rb') as f: | |
| 399 schema = yaml.load(f) | |
| 400 return schema | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
401 |
| 491 | 402 def GetTemplatingDictFromSchemaFilename(fn): |
| 403 obj = LoadSchema(fn) | |
| 404 genOrder: str = ComputeRequiredDeclarationOrder(obj) | |
| 405 templatingDict = ProcessSchema(obj, genOrder) | |
| 406 return templatingDict | |
| 470 | 407 |
| 491 | 408 # +-----------------------+ |
| 409 # | ENTRY POINT | | |
| 410 # +-----------------------+ | |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
411 |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
412 if __name__ == "__main__": |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
413 import argparse |
| 468 | 414 |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
415 parser = argparse.ArgumentParser( |
| 491 | 416 usage="""stonegentool.py [-h] [-o OUT_DIR] [-v] input_schema |
| 417 EXAMPLE: python stonegentool.py -o "generated_files/" """ | |
| 418 + """ "mainSchema.yaml,App Specific Commands.json" """ | |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
419 ) |
| 491 | 420 parser.add_argument("input_schema", type=str, \ |
| 421 help="path to the schema file") | |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
422 parser.add_argument( |
| 489 | 423 "-o", |
| 424 "--out_dir", | |
| 425 type=str, | |
| 426 default=".", | |
| 427 help="""path of the directory where the files | |
| 428 will be generated. Default is current | |
| 429 working folder""", | |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
430 ) |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
431 parser.add_argument( |
| 489 | 432 "-v", |
| 433 "--verbosity", | |
| 434 action="count", | |
| 435 default=0, | |
| 436 help="""increase output verbosity (0 == errors | |
| 437 only, 1 == some verbosity, 2 == nerd | |
| 438 mode""", | |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
439 ) |
| 468 | 440 |
|
485
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
441 args = parser.parse_args() |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
442 inputSchemaFilename = args.input_schema |
|
772516adcbf6
Ongoing work on code generation. Enums and structs OK in ts and cpp
bgo-osimis
parents:
482
diff
changeset
|
443 outDir = args.out_dir |
| 468 | 444 |
| 491 | 445 schema: Dict = LoadSchema(inputSchemaFilename) |
| 446 genOrder: List[str] = ComputeRequiredDeclarationOrder(schema) | |
| 447 processedSchema: Dict = ProcessSchema(schema,genOrder) | |
| 448 | |
| 449 | |
| 450 # def GenEnumDecl(genc: GenCode, fullName: str, schema: Dict) -> None: | |
| 451 # """Writes the enumerations in genc""" | |
| 452 # enumDict:Dict=schema[fullName] | |
| 453 # # jinja2 template | |
| 454 # j2cppEnum = Template(trim( | |
| 455 # """ {{fullName}} | |
| 456 # { | |
| 457 # {% for key in enumDict.keys()%} | |
| 458 # {{key}}, | |
| 459 # {%endfor%} | |
| 460 # }; | |
| 461 # """)) | |
| 462 # j2cppEnumR = j2cppEnum.render(locals()) | |
| 463 # genc.cppEnums.write(j2cppEnumR) | |
| 464 | |
| 465 # j2tsEnum = Template(trim( | |
| 466 # """ export {{fullName}} | |
| 467 # { | |
| 468 # {% for key in enumDict.keys()%} | |
| 469 # {{key}}, | |
| 470 # {%endfor%} | |
| 471 # }; | |
| 472 # """)) | |
| 473 # j2cppEnumR = j2cppEnum.render(locals()) | |
| 474 # genc.tsEnums.write(j2cppEnumR) | |
| 475 | |
| 476 | |
| 477 | |
| 478 # def GetSerializationCode(typename: str,valueName: str, tempName: str) | |
| 479 # if IsPrimitiveType(typename) or IsTemplateCollection(typename): | |
| 480 # # no need to write code for the primitive types or collections. | |
| 481 # # It is handled in C++ by the template functions and in TS by | |
| 482 # # the JSON.stringify code. | |
| 483 # elif IsStructType(typename): | |
| 484 # pass | |
| 485 | |
| 486 # def GenStructTypeDeclAndSerialize(genc: GenCode, type, schema) -> None: | |
| 487 # ###### | |
| 488 # # CPP | |
| 489 # ###### | |
| 490 # sampleCpp = """ struct Message1 | |
| 491 # { | |
| 492 # int32_t a; | |
| 493 # std::string b; | |
| 494 # EnumMonth0 c; | |
| 495 # bool d; | |
| 496 # }; | |
| 497 | |
| 498 # Json::Value StoneSerialize(const Message1& value) | |
| 499 # { | |
| 500 # Json::Value result(Json::objectValue); | |
| 501 # result["a"] = StoneSerialize(value.a); | |
| 502 # result["b"] = StoneSerialize(value.b); | |
| 503 # result["c"] = StoneSerialize(value.c); | |
| 504 # result["d"] = StoneSerialize(value.d); | |
| 505 # return result; | |
| 506 # } | |
| 507 # """ | |
| 508 | |
| 509 | |
| 510 # ###### | |
| 511 # # TS | |
| 512 # ###### | |
| 513 # sampleTs = """ | |
| 514 # { | |
| 515 # export class Message1 { | |
| 516 # a: number; | |
| 517 # b: string; | |
| 518 # c: EnumMonth0; | |
| 519 # d: boolean; | |
| 520 # public StoneSerialize(): string { | |
| 521 # let container: object = {}; | |
| 522 # container['type'] = 'Message1'; | |
| 523 # container['value'] = this; | |
| 524 # return JSON.stringify(container); | |
| 525 # } | |
| 526 # }; | |
| 527 # } | |
| 528 # """ | |
| 529 | |
| 530 | |
| 531 | |
| 532 | |
| 533 # tsText: StringIO = StringIO() | |
| 534 # cppText: StringIO = StringIO() | |
| 535 | |
| 536 # tsText.write("class %s\n" % typeDict["name"]) | |
| 537 # tsText.write("{\n") | |
| 538 | |
| 539 # cppText.write("struct %s\n" % typeDict["name"]) | |
| 540 # cppText.write("{\n") | |
| 541 | |
| 542 # """ | |
| 543 | |
| 544 # GenerateSerializationCode(typename,valueName) | |
|
482
f58fe38c8c04
Ongoing work on codegen: ts and cpp enum and struct writing seem to be OK. No file write yet
bgo-osimis
parents:
474
diff
changeset
|
545 |
| 491 | 546 # primitives: |
| 547 # ----------- | |
| 548 # int | |
| 549 # jsonValue val(objectInt); | |
| 550 # val.setValue("$name") | |
| 551 # parent.add(("$name",$name) | |
| 552 # double | |
| 553 # ... | |
| 554 # string | |
| 555 # ... | |
| 556 | |
| 557 # collections: | |
| 558 # ----------- | |
| 559 # dict { } | |
| 560 | |
| 561 # serializeValue() | |
| 562 # """ | |
| 563 | |
| 564 # for i in range(len(typeDict["fields"])): | |
| 565 # field = typeDict["fields"][i] | |
| 566 # name = field["name"] | |
| 567 # tsType = GetTypeScriptTypenameFromCanonical(field["type"]) | |
| 568 # tsText.write(" public %s %s;\n" % (tsType, name)) | |
| 569 # cppType = GetCppTypenameFromCanonical(field["type"]) | |
| 570 # cppText.write(" %s %s;\n" % (cppType, name)) | |
| 571 | |
| 572 # tsText.write("};\n\n") | |
| 573 # cppText.write("};\n\n") | |
| 574 | |
| 575 # genc.tsStructs.write(tsText.getvalue()) | |
| 576 # genc.cppStructs.write(cppText.getvalue()) | |
| 577 | |
| 578 | |
| 579 # def GenerateCodeFromTsTemplate(genc) | |
| 580 | |
| 581 | |
| 582 # +-----------------------+ | |
| 583 # | CODE GENERATION | | |
| 584 # +-----------------------+ | |
| 585 | |
| 586 # def GenPreambles(rootName: str, genc: GenCode) -> None: | |
| 587 # cppPreambleT = Template(trim( | |
| 588 # """// autogenerated by stonegentool on {{time.ctime()}} | |
| 589 # // for module {{rootName}} | |
| 590 # #include <cstdint> | |
| 591 # #include <string> | |
| 592 # #include <vector> | |
| 593 # #include <map> | |
| 594 # namespace {{rootName}} | |
| 595 # { | |
| 596 # Json::Value StoneSerialize(int32_t value) | |
| 597 # { | |
| 598 # Json::Value result(value); | |
| 599 # return result; | |
| 600 # } | |
| 601 # Json::Value StoneSerialize(double value) | |
| 602 # { | |
| 603 # Json::Value result(value); | |
| 604 # return result; | |
| 605 # } | |
| 606 # Json::Value StoneSerialize(bool value) | |
| 607 # { | |
| 608 # Json::Value result(value); | |
| 609 # return result; | |
| 610 # } | |
| 611 # Json::Value StoneSerialize(const std::string& value) | |
| 612 # { | |
| 613 # // the following is better than | |
| 614 # Json::Value result(value.data(),value.data()+value.size()); | |
| 615 # return result; | |
| 616 # } | |
| 617 # template<typename T> | |
| 618 # Json::Value StoneSerialize(const std::map<std::string,T>& value) | |
| 619 # { | |
| 620 # Json::Value result(Json::objectValue); | |
| 621 | |
| 622 # for (std::map<std::string, T>::const_iterator it = value.cbegin(); | |
| 623 # it != value.cend(); ++it) | |
| 624 # { | |
| 625 # // it->first it->second | |
| 626 # result[it->first] = StoneSerialize(it->second); | |
| 627 # } | |
| 628 # return result; | |
| 629 # } | |
| 630 # template<typename T> | |
| 631 # Json::Value StoneSerialize(const std::vector<T>& value) | |
| 632 # { | |
| 633 # Json::Value result(Json::arrayValue); | |
| 634 # for (size_t i = 0; i < value.size(); ++i) | |
| 635 # { | |
| 636 # result.append(StoneSerialize(value[i])); | |
| 637 # } | |
| 638 # return result; | |
| 639 # } | |
| 640 # """ | |
| 641 # cppPreambleR = cppPreambleT.render(locals()) | |
| 642 # genc.cppPreamble.write(cppPreambleR) | |
| 643 | |
| 644 # tsPreambleT = Template(trim( | |
| 645 # """// autogenerated by stonegentool on {{time.ctime()}} | |
| 646 # // for module {{rootName}} | |
| 647 | |
| 648 # namespace {{rootName}} | |
| 649 # { | |
| 650 # """ | |
| 651 # tsPreambleR = tsPreambleT.render(locals()) | |
| 652 # genc.tsPreamble.write(tsPreambleR) | |
| 653 | |
| 654 # def ComputeOrder_ProcessStruct( \ | |
| 655 # genOrder: List[str], name:str, schema: Dict[str, str]) -> None: | |
| 656 # # let's generate the code according to the | |
| 657 # struct = schema[name] | |
| 658 | |
| 659 # if not IsStructType(name): | |
| 660 # raise Exception(f'{typename} should start with "struct "') |