changeset 473:628941d63b8c bgo-commands-codegen

Ongoing work. Parsing tests work
author bgo-osimis
date Wed, 13 Feb 2019 12:07:00 +0100
parents 3db3289e1c25
children 38997ceb9bc6
files .hgignore Resources/CodeGeneration/stonegentool.py Resources/CodeGeneration/stonegentool_test.py Resources/CodeGeneration/test/test1.json Resources/CodeGeneration/test/test1.jsonc Resources/CodeGeneration/test/test1_bogus_json.jsonc Resources/CodeGeneration/test/test1_bogus_schema.jsonc
diffstat 7 files changed, 269 insertions(+), 53 deletions(-) [+]
line wrap: on
line diff
--- a/.hgignore	Wed Feb 13 06:46:36 2019 +0100
+++ b/.hgignore	Wed Feb 13 12:07:00 2019 +0100
@@ -10,3 +10,4 @@
 Resources/CommandTool/protoc-tests/generated_ts/
 Resources/CommandTool/flatc-tests/basic/build/
 .vscode/
+Resources/CodeGeneration/__pycache__
--- a/Resources/CodeGeneration/stonegentool.py	Wed Feb 13 06:46:36 2019 +0100
+++ b/Resources/CodeGeneration/stonegentool.py	Wed Feb 13 12:07:00 2019 +0100
@@ -1,17 +1,46 @@
-from typing import List,Dict,Set
-import sys
 import json
 import re
+import sys
+from typing import Dict, List, Set
+
 
 """
          1         2         3         4         5         6         7
 12345678901234567890123456789012345678901234567890123456789012345678901234567890
 """
 
-def LoadSchema(file_path : str):
-    with open(file_path, 'r') as fp:
-      obj = json.load(fp)
-    return obj
+import json
+import re
+
+"""A set of utilities to perform JSON operation"""
+
+class JsonHelpers:
+    @staticmethod
+    def removeCommentsFromJsonContent(string):
+        """
+        remove comments from a JSON file
+
+        Comments are not allowed in JSON but, i.e., Orthanc configuration files contains C++ like comments that we need to remove before python can parse the file
+        """
+        string = re.sub(re.compile("/\*.*?\*/", re.DOTALL), "",
+                        string)  # remove all occurance streamed comments (/*COMMENT */) from string
+        string = re.sub(re.compile("//.*?\n"), "",
+                        string)  # remove all occurance singleline comments (//COMMENT\n ) from string
+        return string
+
+    @staticmethod
+    def loadJsonWithComments(path):
+        """
+        reads a JSON file that may contain C++ like comments
+        """
+        with open(path, 'r') as fp:
+            fileContent = fp.read()
+        fileContent = JsonHelpers.removeCommentsFromJsonContent(fileContent)
+        return json.loads(fileContent)
+
+
+def LoadSchema(filePath : str):
+    return JsonHelpers.loadJsonWithComments(filePath)
 
 # class Type:
 #   def __init__(self, canonicalTypeName:str, kind:str):
@@ -63,38 +92,38 @@
 
 def CheckTypeSchema(definedType : Dict) -> None:
   allowedDefinedTypeKinds = ["enum","struct"]
-  if not definedType.has_key('name'):
+  if not 'name' in definedType:
     raise Exception("type lacks the 'name' key")
   name = definedType['name']
-  if not definedType.has_key('kind'):
+  if not 'kind' in definedType:
     raise Exception(f"type {name} lacks the 'kind' key")
   kind = definedType['kind']
   if not (kind in allowedDefinedTypeKinds):
     raise Exception(f"type {name} : kind {kind} is not allowed. " + 
       f"It must be one of {allowedDefinedTypeKinds}")
   
-  if not definedType.has_key('fields'):
+  if not 'fields' in definedType:
     raise Exception("type {name} lacks the 'fields' key")
 
   # generic check on all kinds of types
   fields = definedType['fields']
   for field in fields:
     fieldName = field['name']
-    if not field.has_key('name'):
+    if not 'name' in field:
       raise Exception("field in type {name} lacks the 'name' key")
 
   # fields in struct must have types
   if kind == 'struct':  
     for field in fields:
       fieldName = field['name']
-      if not field.has_key('type'):
+      if not 'type' in field:
         raise Exception(f"field {fieldName} in type {name} "
         + "lacks the 'type' key")
 
 def CheckSchemaSchema(schema : Dict) -> None:
-  if not schema.has_key('root_name'):
+  if not 'root_name' in schema:
     raise Exception("schema lacks the 'root_name' key")
-  if not schema.has_key('types'):
+  if not 'types' in schema:
     raise Exception("schema lacks the 'types' key")
   for definedType in schema['types']:
     CheckTypeSchema(definedType)
@@ -102,7 +131,7 @@
 # def CreateAndCacheTypeObject(allTypes : Dict[str,Type], typeDict : Dict)  -> None:
 #   """This does not set the dependentTypes field"""
 #   typeName : str = typeDict['name']
-#   if allTypes.has_key(typeName):
+#   if typeName in allTypes:
 #     raise Exception(f'Type {typeName} is defined more than once!')
 #   else:
 #     typeObject = Type(typeName, typeDict['kind'])
@@ -119,7 +148,7 @@
 
   # the template level we're currently in
   templateLevel = 0
-  for i in len(sentence):
+  for i in range(len(sentence)):
     if (sentence[i] == ",") and (templateLevel == 0):
       return (sentence[0:i],sentence[i+1:])
     elif (sentence[i] == "<"):
@@ -144,6 +173,8 @@
   while stillStuffToEat:
     firstToken,restOfString = EatToken(restOfString)
     tokenList.append(firstToken)
+    if restOfString == "":
+      stillStuffToEat = False
   return tokenList
 
 templateRegex = \
@@ -167,9 +198,8 @@
     listOfDependentTypes = SplitListOfTypes(matches.group(2))
     return (True,matches.group(1),listOfDependentTypes)
 
-
 # def GetPrimitiveType(typeName : str) -> Type:
-#   if allTypes.has_key(typeName):
+#   if typeName in allTypes:
 #     return allTypes[typeName]
 #   else:
 #     primitiveTypes = ['int32', 'float32', 'float64', 'string']
@@ -202,7 +232,7 @@
         ProcessTypeTree(ancestors, genOrderQueue,
           structTypes, dependentTypeName)
     else:
-      if structTypes.has_key(typeName):
+      if typeName in structTypes:
         ProcessStructType_DepthFirstRecursive(genOrderQueue, structTypes,
         structTypes[typeName])
 
--- a/Resources/CodeGeneration/stonegentool_test.py	Wed Feb 13 06:46:36 2019 +0100
+++ b/Resources/CodeGeneration/stonegentool_test.py	Wed Feb 13 12:07:00 2019 +0100
@@ -1,5 +1,6 @@
-from stonegentool import EatToken
+from stonegentool import EatToken,SplitListOfTypes,ParseTemplateType,LoadSchema,CheckSchemaSchema
 import unittest
+import os
 
 class TestStonegentool(unittest.TestCase):
   def test_EatToken_empty(self):
@@ -35,6 +36,58 @@
     self.assertEqual(a,r"map<int32,string>")
     self.assertEqual(b,r"map<map<int32,string>,string>")
 
+  def test_SplitListOfTypes(self):
+    c = r"vector<map<vector<string>,map<int32,string>>>,map<int32,string>,map<map<int32,string>,string>"
+    lot = SplitListOfTypes(c)
+    self.assertEqual(3,len(lot))
+    self.assertEqual("vector<map<vector<string>,map<int32,string>>>",lot[0])
+    self.assertEqual("map<int32,string>",lot[1])
+    self.assertEqual("map<map<int32,string>,string>",lot[2])
+
+  def test_SplitListOfTypes_bogus(self):
+    c = r"vector<map<vector<string>,map<int32,string>>,map<int32,string>,map<map<int32,string>,string"
+    self.assertRaises(Exception,SplitListOfTypes,c) # the argument c must be passed to assertRaises, not as a normal call of SplitListOfTypes
+    
+  def test_ParseTemplateType_true(self):
+    c = "map<vector<map<int,vector<string>>>,map<vector<int>,vector<string>>>"
+    (ok,a,b) = ParseTemplateType(c)
+    self.assertEqual(ok,True)
+    self.assertEqual(a,"map")
+    self.assertEqual(b,["vector<map<int,vector<string>>>","map<vector<int>,vector<string>>"])
+
+    (ok2,a2,b2) = ParseTemplateType(b[0])
+    self.assertEqual(ok2,True)
+    self.assertEqual(a2,"vector")
+    self.assertEqual(b2,["map<int,vector<string>>"])
+
+    (ok3,a3,b3) = ParseTemplateType(b[1])
+    self.assertEqual(ok3,True)
+    self.assertEqual(a3,"map")
+    self.assertEqual(b3,["vector<int>","vector<string>"])
+
+    (ok4,a4,b4) = ParseTemplateType(b2[0])
+    self.assertEqual(ok4,True)
+    self.assertEqual(a4,"map")
+    self.assertEqual(b4,["int","vector<string>"])
+    
+  def test_ParseSchema(self):
+    fn = os.path.join(os.path.dirname(__file__), 'test', 'test1.jsonc')
+    obj = LoadSchema(fn)
+    # we're happy if it does not crash
+    CheckSchemaSchema(obj)
+
+  def test_ParseSchema_bogus_json(self):
+    fn = os.path.join(os.path.dirname(__file__), 'test', 'test1_bogus_json.jsonc')
+    self.assertRaises(Exception,LoadSchema,fn)
+
+  def test_ParseSchema_bogus_schema(self):
+    fn = os.path.join(os.path.dirname(__file__), 'test', 'test1_bogus_schema.jsonc')
+    obj = LoadSchema(fn)
+    self.assertRaises(Exception,CheckSchemaSchema,obj) 
+
+
+
+
 # def test(self):
 #   s = 'hello world'
 #   self.assertEqual(s.split(), ['hello', 'world'])
--- a/Resources/CodeGeneration/test/test1.json	Wed Feb 13 06:46:36 2019 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,34 +0,0 @@
-{
-  "root_name":"test1",
-  "types": [
-    {
-      "name":"B",
-      "kind":"struct",
-      "fields": [
-        {
-          "name":"someAs",
-          "type":"vector<A>"
-        },
-        {
-          "name":"someInts",
-          "type":"vector<int32>"
-        }
-      ]
-    },
-    {
-      "name":"A",
-      "kind":"struct",
-      "fields": [
-        {
-          "name":"someStrings",
-          "type":"vector<string>"
-        },
-        {
-          "name":"someInts2",
-          "type":"vector<int32>"
-        }
-      ]
-    }
-  ]
-}
-
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Resources/CodeGeneration/test/test1.jsonc	Wed Feb 13 12:07:00 2019 +0100
@@ -0,0 +1,55 @@
+/*
+         1         2         3         4         5         6         7
+12345678901234567890123456789012345678901234567890123456789012345678901234567890
+*/
+{
+  "root_name":"test1",
+  "types": [
+    {
+      "name":"B",
+      "kind":"struct",
+      "fields": [
+        {
+          "name":"someAs",
+          "type":"vector<A>"
+        },
+        {
+          "name":"someInts",
+          "type":"vector<int32>"
+        }
+      ]
+    },
+    {
+      "name":"A",
+      "kind":"struct",
+      "fields": [
+        {
+          "name":"someStrings",
+          "type":"vector<string>"
+        },
+        {
+          "name":"someInts2",
+          "type":"vector<int32>"
+        }
+      ]
+    },
+    {
+      "name":"MovieType",
+      "kind":"enum",
+      "fields": [
+        {
+          "name":"Romcom"
+        },
+        {
+          "name":"Horror"
+        },
+        {
+          "name":"ScienceFiction"
+        },
+        {
+          "name":"Vegetables"
+        }
+      ]
+    }
+  ]
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Resources/CodeGeneration/test/test1_bogus_json.jsonc	Wed Feb 13 12:07:00 2019 +0100
@@ -0,0 +1,55 @@
+{
+  "root_name":"test1",
+  "types": [
+    {
+      "name":"B",
+      "kind":"struct",
+      "fields": [
+        {
+          "name":"someAs",
+          "type":"vector<A>"
+        }},
+        {
+          "name":"someInts",
+          "type":"vector<int32>"
+        }
+      ]
+    },
+    {
+      "name":"A",
+      "kind":"struct",
+      "fields": [
+        {
+          "name":"someStrings",
+          "type":"vector<string>"
+        },
+        {
+          "name":"someInts2",
+          "type":"vector<int32>"
+        }
+      ]
+    },
+    {
+      "name":"MovieType",
+      "kind":"enum",
+      "fields": [
+        {
+          "name":"Romcom",
+        },
+        {
+          "name":"Horror",
+        },
+        {
+          "name":"ScienceFiction",
+        },
+        {
+          "name":"Vegetables",
+        }
+    }
+  ]
+}
+
+/*
+         1         2         3         4         5         6         7
+12345678901234567890123456789012345678901234567890123456789012345678901234567890
+*/
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Resources/CodeGeneration/test/test1_bogus_schema.jsonc	Wed Feb 13 12:07:00 2019 +0100
@@ -0,0 +1,56 @@
+{
+  "root_name":"test1",
+  "types": [
+    {
+      "name":"B",
+      "kind":"struct",
+      "fields": [
+        {
+          "name":"someAs",
+          "type":"vector<A>"
+        },
+        {
+          "name":"someInts",
+          "type":"vector<int32>"
+        }
+      ]
+    },
+    {
+      "name":"A",
+      "kiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiind":"struct",
+      "fields": [
+        {
+          "name":"someStrings",
+          "type":"vector<string>"
+        },
+        {
+          "name":"someInts2",
+          "type":"vector<int32>"
+        }
+      ]
+    },
+    {
+      "name":"MovieType",
+      "kind":"enum",
+      "fields": [
+        {
+          "naaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaame":"Romcom"
+        },
+        {
+          "name":"Horror"
+        },
+        {
+          "name":"ScienceFiction"
+        },
+        {
+          "name":"Vegetables"
+        }
+      ]
+    }
+  ]
+}
+
+/*
+         1         2         3         4         5         6         7
+12345678901234567890123456789012345678901234567890123456789012345678901234567890
+*/