From 19b2a51836700dd24538498a8cbdcdf5a1ca38bf Mon Sep 17 00:00:00 2001 From: Michael Becker Date: Sat, 24 Aug 2024 14:12:28 -0400 Subject: [PATCH] add yaml2mcl build script --- mocha-common/build | 7 + mocha-common/compilers/yaml2mcl/Program.py | 134 +++++ .../compilers/yaml2mcl/mocha/core/Guid.py | 417 +++++++++++++++ .../mocha/core/KnownAttributeGuids.py | 18 + .../yaml2mcl/mocha/core/KnownClassGuids.py | 5 + .../mocha/core/KnownRelationshipGuids.py | 3 + .../yaml2mcl/mocha/library/InstanceCache.py | 60 +++ .../mocha/library/manager/GuidCache.py | 46 ++ .../mocha/library/manager/LibraryOperation.py | 21 + .../library/manager/MemoryLibraryManager.py | 192 +++++++ .../library/manager/MochaLibraryManager.py | 482 ++++++++++++++++++ .../mocha/library/manager/Normalization.py | 5 + .../mocha/library/manager/SQLExpression.py | 47 ++ .../mocha/library/manager/SQLFunctionCall.py | 20 + .../mocha/library/manager/SQLParameter.py | 12 + .../mocha/library/manager/__init__.py | 16 + .../operations/AssignAttributeOperation.py | 21 + .../operations/AssignRelationshipOperation.py | 21 + .../operations/CreateClassOperation.py | 17 + .../operations/CreateInstanceOperation.py | 17 + .../operations/PrepareInstanceOperation.py | 41 ++ .../operations/StoredProcedureOperation.py | 24 + .../mocha/library/parser/JSONLibraryParser.py | 30 ++ .../mocha/library/parser/LibraryParser.py | 4 + .../mocha/library/parser/MochaSyntaxError.py | 5 + .../mocha/library/parser/XMLLibraryParser.py | 46 ++ .../mocha/library/parser/YAMLLibraryParser.py | 345 +++++++++++++ .../yaml2mcl/mocha/library/parser/__init__.py | 4 + mocha-common/yaml2mcl | 4 + 29 files changed, 2064 insertions(+) create mode 100755 mocha-common/build create mode 100644 mocha-common/compilers/yaml2mcl/Program.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/core/Guid.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/core/KnownAttributeGuids.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/core/KnownClassGuids.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/core/KnownRelationshipGuids.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/InstanceCache.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/GuidCache.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/LibraryOperation.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/MemoryLibraryManager.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/MochaLibraryManager.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/Normalization.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLExpression.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLFunctionCall.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLParameter.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/__init__.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/AssignAttributeOperation.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/AssignRelationshipOperation.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/CreateClassOperation.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/CreateInstanceOperation.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/PrepareInstanceOperation.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/StoredProcedureOperation.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/parser/JSONLibraryParser.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/parser/LibraryParser.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/parser/MochaSyntaxError.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/parser/XMLLibraryParser.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/parser/YAMLLibraryParser.py create mode 100644 mocha-common/compilers/yaml2mcl/mocha/library/parser/__init__.py create mode 100755 mocha-common/yaml2mcl diff --git a/mocha-common/build b/mocha-common/build new file mode 100755 index 0000000..2d729b6 --- /dev/null +++ b/mocha-common/build @@ -0,0 +1,7 @@ +#!/bin/bash + +if [ ! -d output ]; then + mkdir output +fi + +./yaml2mcl --export-entities=output/net.alcetech.Mocha.System.cs -o output/net.alcetech.Mocha.System.mcl data/libraries/yaml/net.alcetech.Mocha.System diff --git a/mocha-common/compilers/yaml2mcl/Program.py b/mocha-common/compilers/yaml2mcl/Program.py new file mode 100644 index 0000000..683a056 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/Program.py @@ -0,0 +1,134 @@ +# Copyright 2024 Michael Becker +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from glob import glob + +from mocha.library.parser import YAMLLibraryParser + +from mocha.library.manager import MochaLibraryManager, MemoryLibraryManager + +import os, sys + +class Yaml2Mcl: + + def __init__(self): + + pass + + def start(self): + + args = sys.argv[1:] + + filenames = [ ] + outputFileName = "" + exportEntitiesFileName = None + + while (len(args) > 0): + # print (args[0]) + if args[0] == "-o" or args[0] == "--output": + outputFileName = args[1] + args = args[2:] + elif args[0] == "--export-entities": + exportEntitiesFileName = "" + args = args[1:] + elif args[0].startswith("--export-entities="): + exportEntitiesFileName = args[0].split('=', 2)[1] + args = args[1:] + else: + filenames.append(args[0]) + args = args[1:] + + + print ("Mocha YAML to Mocha Class Library Compiler") + print ("Version 1.0") + print ("") + + print (filenames) + print ("output to: " + outputFileName) + + manager = MemoryLibraryManager() + yl = YAMLLibraryParser(manager) + + for filename in filenames: + if not os.path.isdir(filename): + print ("not a directory: '" + filename + "'") + continue + + yaml_files = sorted(glob(filename + "/**/*.yaml", recursive=True)) + if len(yaml_files) == 0: + print ("no YAML files found ; does the path exist?") + return 3 + + # first, load the entity defs + for yaml_file in yaml_files: + yl.load_entity_definitions_from_file(yaml_file) + + #try: + # then, load instance definitions (also loads sugar elements into memory for later use) + for yaml_file in yaml_files: + yl.load_instances_from_file(yaml_file) + + # finally, apply syntactic sugar + yl.apply_sugar() + + manager.filename = outputFileName + manager.commit() + + if not exportEntitiesFileName is None: + if exportEntitiesFileName == "": + exportEntitiesFileName = "entities.cs" + + print("export entities to file: '" + exportEntitiesFileName + "'") + manager.save_entities_to_file(exportEntitiesFileName) + + return True + + #except NameError as ex: + + print (ex) + + rgx = "&(.*);" + + # go through and get all entity references across all files + import re + + stuff = [] + + for yaml_file in yaml_files: + f = open(yaml_file, "r") + text = f.read() + matches = re.findall(rgx, text) + for match in matches: + stuff.append(match) + f.close() + + missingEntities = [] + for stuf in stuff: + if not stuf in manager.entityReferences: + if not stuf in missingEntities: + missingEntities.append(stuf) + + if len(missingEntities) > 0: + print("\nNOTE: there were undefined referenced entities:\n") + for missingEntity in missingEntities: + print("\t" + missingEntity) + print("\n") + + return False + + +if __name__ == "__main__": + + app = Yaml2Mcl() + app.start() \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/core/Guid.py b/mocha-common/compilers/yaml2mcl/mocha/core/Guid.py new file mode 100644 index 0000000..8b4a8bc --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/core/Guid.py @@ -0,0 +1,417 @@ + +class UUIDParseFailureKind: + FORMAT = 2 + +class UUIDParseNumbers: + NOSPACE = 2 + +class UUIDParseResult: + + def __init__(self): + self.parsedGuid = Guid() + self.kind = None + self.message = None + + def setFailure(self, kind, message): + self.kind = kind + self.message = message + +class Guid: + + __urand = None + + def __init__(self): + if Guid.__urand is None: + Guid.__urand = open("/dev/urandom", "rb") + + self.__a = 0 + self.__b = 0 + self.__c = 0 + self.__d = 0 + self.__e = 0 + self.__f = 0 + self.__g = 0 + self.__h = 0 + self.__i = 0 + self.__j = 0 + self.__k = 0 + + @staticmethod + def __stringToInt(guidString : str, parsePos : int, requiredLength : int, flags : int, result : int, parseResult): + parseWhat = guidString[parsePos:(parsePos+requiredLength)] + result = int(parseWhat, 16) + parsePos += requiredLength + return (parsePos, result) + + @staticmethod + def fromBytes(b : bytes): + guid = Guid() + + guid.__a = (b[3] << 24) | (b[2] << 16) | (b[1] << 8) | b[0] + guid.__b = (b[5] << 8) | b[4] + guid.__c = ((b[7] << 8) | b[6]) + guid.__d = b[8] + guid.__e = b[9] + guid.__f = b[10] + guid.__g = b[11] + guid.__h = b[12] + guid.__i = b[13] + guid.__j = b[14] + guid.__k = b[15] + + return guid + + @staticmethod + def __tryParseGuidWithDashes(guidString : str) -> UUIDParseResult: + + result = UUIDParseResult() + print (guidString) + if guidString == "": + return result + + guidString = guidString.replace("{", "").replace("}", "").replace("-", "") + + import codecs + try: + myBytes = codecs.decode(guidString, 'hex') + except: + return result + + guidBytes = [ myBytes[3], myBytes[2], myBytes[1], myBytes[0], myBytes[5], myBytes[4], myBytes[7], myBytes[6], myBytes[8], + myBytes[9], myBytes[10], myBytes[11], myBytes[12], myBytes[13], myBytes[14], myBytes[15] ] + + result.parsedGuid = Guid.fromBytes(guidBytes) + return result + + """ + + startPos = 0 + temp = 0 + templ = 0 + currentPos = 0 + result = UUIDParseResult() + hasDashes = True + + if (len(guidString) < 1): + print(guidString) + result.setFailure(UUIDParseFailureKind.FORMAT, "Format_GuidInvLen[?]") + return result + + # check to see that it's the proper length + if guidString[0] == '{': + + if len(guidString) != 38 or guidString[37] != '}': + print(guidString) + result.setFailure(UUIDParseFailureKind.FORMAT, "Format_GuidInvLen[38]") + return result + + startPos = 1 + + elif guidString[0] == '(': + + if len(guidString) != 38 or guidString[37] != ')': + + print(guidString) + result.setFailure(UUIDParseFailureKind.FORMAT, "Format_GuidInvLen[38]") + return result + + startPos = 1 + + elif len(guidString) != 36: + + if len(guidString) != 32: + + print(guidString) + result.setFailure(UUIDParseFailureKind.FORMAT, "Format_GuidInvLen[36]") + return result + + else: + + hasDashes = False + + if hasDashes: + + if (guidString[8 + startPos] != '-' or + guidString[13 + startPos] != '-' or + guidString[18 + startPos] != '-' or + guidString[23 + startPos] != '-'): + + result.setFailure(UUIDParseFailureKind.FORMAT, "Format_GuidDashes") + return result + + currentPos = startPos + + (parsePos, temp) = Guid.__stringToInt(guidString, currentPos, 8, UUIDParseNumbers.NOSPACE, temp, result) + result.parsedGuid.__a = temp + currentPos = parsePos + + if hasDashes: + currentPos = currentPos + 1; # Increment past the '-'; + + (parsePos, temp) = Guid.__stringToInt(guidString, currentPos, 4, UUIDParseNumbers.NOSPACE, temp, result) + result.parsedGuid.__b = temp + currentPos = parsePos + + if hasDashes: + currentPos = currentPos + 1 # Increment past the '-'; + + (parsePos, temp) = Guid.__stringToInt(guidString, currentPos, 4, UUIDParseNumbers.NOSPACE, temp, result) + result.parsedGuid.__c = temp + currentPos = parsePos + + if hasDashes: + currentPos = currentPos + 1 # Increment past the '-'; + + (parsePos, temp) = Guid.__stringToInt(guidString, currentPos, 4, UUIDParseNumbers.NOSPACE, temp, result) + result.parsedGuid.__d = temp >> 8 + result.parsedGuid.__e = temp - (temp >> 8) + currentPos = parsePos + + if hasDashes: + currentPos = currentPos + 1 # Increment past the '-'; + startPos = currentPos + + (parsePos, templ) = Guid.__stringToInt(guidString, currentPos, 8, UUIDParseNumbers.NOSPACE, templ, result) + currentPos = parsePos + + (parsePos, temp2) = Guid.__stringToInt(guidString, currentPos, 4, UUIDParseNumbers.NOSPACE, templ, result) + + # /* + # if ($currentPos - $startPos != 12) { + # $result->setFailure(UUIDParseFailureKind::FORMAT, "Format_GuidInvLen(*)") + # return $result; + # } + # */ + result.parsedGuid.__j = temp2 >> 8 + result.parsedGuid.__k = temp2 - (temp2 >> 8) + + temp = templ + result.parsedGuid.__f = temp >> 24 + result.parsedGuid.__g = (temp >> 16) - (temp >> 24) + result.parsedGuid.__h = (temp >> 8) - (temp >> 16) + result.parsedGuid.__i = temp - (temp >> 8) + return result + """ + + def parse(value : str): + if value is None: + return None + + result = Guid.__tryParseGuidWithDashes(value) + return result.parsedGuid + + def __eq__(self, other): + # /* + # if ( + # $uuid->clock_seq_hi_and_reserved == self.clock_seq_hi_and_reserved + # && $uuid->node == self.node + # && $uuid->time_hi_and_version == self.time_hi_and_version + # && $uuid->time_low == self.time_low + # && $uuid->time_mid == self.time_mid + # ) + # { + # return true; + # } + # */ + return ( + other.__a == self.__a + and other.__b == self.__b + and other.__c == self.__c + and other.__d == self.__d + and other.__e == self.__e + and other.__f == self.__f + and other.__g == self.__g + and other.__h == self.__h + and other.__i == self.__i + and other.__j == self.__j + and other.__k == self.__k + ) + + @staticmethod + def __init_bits(): + __pr_bits = False + if Guid.__urand is not None: + __pr_bits += Guid.__urand.read(16) + + if not self.__pr_bits: + fp = open ( '/dev/urandom', 'rb' ) + if fp is not False: + __pr_bits += fp.read(16) + fp.close() + + else: + + # If /dev/urandom isn't available (eg: in non-unix systems), use mt_rand(). + __pr_bits = "" + for cnt in range(0, 16): + __pr_bits += chr ( __mt_rand ( 0, 255 ) ) + + return __pr_bits + + """ + @brief Generates a Universally Unique IDentifier, version 4. + + This function generates a truly random UUID. The built in CakePHP String::uuid() function + is not cryptographically secure. You should uses this function instead. + + @see http://tools.ietf.org/html/rfc4122#section-4.4 + @see http://en.wikipedia.org/wiki/UUID + @return UUID A UUID, made up of 32 hex digits and 4 hyphens. + """ + @staticmethod + def generate(): + uuid = Guid() + __pr_bits = Guid.__init_bits() + + uuid.__a = (int(pr_bits[3]) << 24) | (int(pr_bits[2]) << 16) | (int(pr_bits[1]) << 8) | pr_bits[0] + uuid.__b = ((int(pr_bits[5]) << 8) | pr_bits[4]) + uuid.__c = ((int(pr_bits[7]) << 8) | pr_bits[6]) + uuid.__d = pr_bits[8] + uuid.__e = pr_bits[9] + uuid.__f = pr_bits[10] + uuid.__g = pr_bits[11] + uuid.__h = pr_bits[12] + uuid.__i = pr_bits[13] + uuid.__j = pr_bits[14] + uuid.__k = pr_bits[15] + + return uuid + + @staticmethod + def __hexToChar(a : int) -> str: + a = a & 0xf + + v = 0 + if a > 9: + v = a - 10 + 0x61 + else: + v = a + 0x30 + + return chr(v) + + @staticmethod + def __hexsToChars(a : int, b : int, hex : bool = False): + guidChars = "" + if hex: + guidChars = "0x" + + guidChars += Guid.__hexToChar(a >> 4) + guidChars += Guid.__hexToChar(a) + if hex: + guidChars += ",0x" + + guidChars += Guid.__hexToChar(b >> 4) + guidChars += Guid.__hexToChar(b) + return guidChars + + # public function __toString() + # { + # } + # public function __toStringFormat($includeDashes = true, $prefix = "{", $suffix = "}") + # { + # $guidChars = $prefix; + # $guidChars += Guid.__hexsToChars(self._a >> 24, self._a >> 16) + # $guidChars += Guid.__hexsToChars(self._a >> 8, self._a) + # if ($includeDashes) $guidChars += '-'; + # $guidChars += Guid.__hexsToChars(self._b >> 8, self._b) + # if ($includeDashes) $guidChars += '-'; + # $guidChars += Guid.__hexsToChars(self._c >> 8, self._c) + # if ($includeDashes) $guidChars += '-'; + # $guidChars += Guid.__hexsToChars(self._d, self._e) + # if ($includeDashes) $guidChars += '-'; + # $guidChars += Guid.__hexsToChars(self._f, self._g) + # $guidChars += Guid.__hexsToChars(self._h, self._i) + # $guidChars += Guid.__hexsToChars(self._j, self._k) + # $guidChars += $suffix; + # return $guidChars; + # } + + def format(input : str): + output = input + output = output[0:8] + "-" + output[8:(8+4)] + "-" + output[12:12+4] + "-" + output[16:16+4] + "-" + output[20:] + return "{" + output + "}" + + def strip(self): + guidChars = "" + guidChars += Guid.__hexsToChars(self.__a >> 24, self.__a >> 16) + guidChars += Guid.__hexsToChars(self.__a >> 8, self.__a) + guidChars += Guid.__hexsToChars(self.__b >> 8, self.__b) + guidChars += Guid.__hexsToChars(self.__c >> 8, self.__c) + guidChars += Guid.__hexsToChars(self.__d, self.__e) + guidChars += Guid.__hexsToChars(self.__f, self.__g) + guidChars += Guid.__hexsToChars(self.__h, self.__i) + guidChars += Guid.__hexsToChars(self.__j, self.__k) + return guidChars.lower() + + def __str__(self): + # //return self.format(strtoupper( sprintf ( '%08s%04s%04x%04x%012s', self.time_low, self.time_mid, self.time_hi_and_version, self.clock_seq_hi_and_reserved, self.node ) )) + dash = True + guidChars = "{" + guidChars += Guid.__hexsToChars(self.__a >> 24, self.__a >> 16) + guidChars += Guid.__hexsToChars(self.__a >> 8, self.__a) + + if (dash): + guidChars += '-' + + guidChars += Guid.__hexsToChars(self.__b >> 8, self.__b) + if (dash): + guidChars += '-' + + guidChars += Guid.__hexsToChars(self.__c >> 8, self.__c) + if (dash): + guidChars += '-' + + guidChars += Guid.__hexsToChars(self.__d, self.__e) + if (dash): + guidChars += '-' + + guidChars += Guid.__hexsToChars(self.__f, self.__g) + guidChars += Guid.__hexsToChars(self.__h, self.__i) + guidChars += Guid.__hexsToChars(self.__j, self.__k) + guidChars += "}" + return guidChars.lower() + + @staticmethod + def create(): + + from uuid import uuid4 + u = uuid4() + g = Guid.parse(str(u)) + return g + + def __key(self): + return (self.__a, self.__b, self.__c, self.__d, self.__e, self.__f, self.__g, self.__h, self.__i, self.__j, self.__k) + + def __hash__(self): + return hash(self.__key()) + + def __eq__(self, other): + if isinstance(other, Guid): + return self.__key() == other.__key() + return NotImplemented + + def to_bytes(self) -> bytes: + a = self.__a.to_bytes(4, 'little') + b = self.__b.to_bytes(2, 'little') + c = self.__c.to_bytes(2, 'little') + + a0 = a[0] + a1 = a[1] + a2 = a[2] + a3 = a[3] + b0 = b[0] + b1 = b[1] + c0 = c[0] + c1 = c[1] + d = self.__d + e = self.__e + f = self.__f + g = self.__g + h = self.__h + i = self.__i + j = self.__j + k = self.__k + + sss = [a0, a1, a2, a3, b0, b1, c0, c1, d, e, f, g, h, i, j, k] + + return bytes(sss) \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/core/KnownAttributeGuids.py b/mocha-common/compilers/yaml2mcl/mocha/core/KnownAttributeGuids.py new file mode 100644 index 0000000..e5ff01a --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/core/KnownAttributeGuids.py @@ -0,0 +1,18 @@ +from .Guid import Guid + +""" +Represents the Text Attribute `Name` +""" +IDA_Name = Guid.parse("{9153A637-992E-4712-ADF2-B03F0D9EDEA6}") + +""" +Represents the Text Attribute `Value` +""" +IDA_Value = Guid.parse("{041DD7FD-2D9C-412B-8B9D-D7125C166FE0}") + +""" +Represents the Text Attribute `Debug Definition File Name` +""" +IDA_DebugDefinitionFileName = Guid.parse("{03bf47c7-dc97-43c8-a8c9-c6147bee4e1f}") +IDA_DebugDefinitionLineNumber = Guid.parse("{822be9b7-531d-4aa1-818a-6e4de1609057}") +IDA_DebugDefinitionColumnNumber = Guid.parse("{0f75c750-e738-4410-9b4e-deb422efc7aa}") \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/core/KnownClassGuids.py b/mocha-common/compilers/yaml2mcl/mocha/core/KnownClassGuids.py new file mode 100644 index 0000000..d11b7b5 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/core/KnownClassGuids.py @@ -0,0 +1,5 @@ +from .Guid import Guid + +IDC_SourceDefinition = Guid.parse("{5d0b2f03-4886-4ba6-ac3c-8f9612963fa6}") +IDC_EntityDefinition = Guid.parse("{15ffa529-6aab-4f1f-8720-f2534951b045}") + diff --git a/mocha-common/compilers/yaml2mcl/mocha/core/KnownRelationshipGuids.py b/mocha-common/compilers/yaml2mcl/mocha/core/KnownRelationshipGuids.py new file mode 100644 index 0000000..a4125ab --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/core/KnownRelationshipGuids.py @@ -0,0 +1,3 @@ +from .Guid import Guid + +IDR_Instance__has__Source_Definition = Guid.parse("{57cbc351-0428-47e6-a6db-445e4503abab}") diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/InstanceCache.py b/mocha-common/compilers/yaml2mcl/mocha/library/InstanceCache.py new file mode 100644 index 0000000..2276f2b --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/InstanceCache.py @@ -0,0 +1,60 @@ +from .manager.Normalization import Normalization +from ..core.Guid import Guid + +class InstanceCache: + + def __init__(self): + + self.next_inst_id = dict() + self.next_inst_id[1] = 1 + + self.inst_indices = dict() + self.inst_guids = dict() + + self.dbids_gid = dict() + + def add_instance(self, inst_key, inst_gid : Guid): + self.inst_guids[inst_key] = inst_gid + self.inst_indices[inst_gid] = inst_key + + if not inst_key[0] in self.next_inst_id: + self.next_inst_id[inst_key[0]] = 1 + + if inst_key[1] >= self.next_inst_id[inst_key[0]]: + self.next_inst_id[inst_key[0]] = inst_key[1] + 1 + + def get_global_identifier(self, inst_key) -> Guid: + return self.inst_guids[inst_key] + + def get_instance_key(self, inst_gid): + return self.inst_indices[inst_gid] + + def get_next_instance_id(self, class_index): + if not class_index in self.next_inst_id: + # this is the first instance of this class + self.next_inst_id[class_index] = 1 + + val = self.next_inst_id[class_index] + self.next_inst_id[class_index] += 1 + return val + + def has_instance_by_global_identifier(self, gid): + return gid in self.inst_indices + + def count(self) -> int: + """ + Returns the number of instances stored in this OMS. + """ + return len(self.inst_indices) + + def get_instance_global_identifers(self): + return self.inst_guids + + def get_instance_keys(self): + return self.inst_indices + + def set_database_id_by_global_identifier(self, gid : Guid, dbid): + self.dbids_gid[Normalization.normalize_uuid(gid.get_value())] = dbid + # self.dbids_idx[self.inst_indices[gid]] = dbid + def get_database_id_by_global_identifier(self, gid : Guid): + return self.dbids_gid[Normalization.normalize_uuid(gid.get_value())] \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/GuidCache.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/GuidCache.py new file mode 100644 index 0000000..70e0a3f --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/GuidCache.py @@ -0,0 +1,46 @@ +# Copyright (C) 2024 Michael Becker +# +# This file is part of yaml2mcl. +# +# yaml2mcl is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# yaml2mcl is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with yaml2mcl. If not, see . + +from ...core.Guid import Guid + +class GuidCache: + + def __init__(self): + self.guid_db = dict() + self.indx_db = dict() + self.__index = 0 + + def add(self, guid : Guid): + if not guid in self.indx_db: + self.guid_db[self.__index] = guid + self.indx_db[guid] = self.__index + self.__index += 1 + + def count(self): + return len(self.indx_db) + + def get_guid(self, index : int) -> Guid: + return self.guid_db[index] + + def get_index(self, guid : Guid) -> int: + return self.indx_db[guid] + + def to_list(self) -> list: + ls = [ ] + for i in range(0, self.count()): + ls.append(self.get_guid(i)) + return ls \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/LibraryOperation.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/LibraryOperation.py new file mode 100644 index 0000000..faffb26 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/LibraryOperation.py @@ -0,0 +1,21 @@ +import MySQLdb + +from MySQLdb.connections import Connection + +class LibraryOperation: + def build_query(self): + return '' + + def print_error(self, cur): + rows = cur.fetchall() + for row in rows: + if 'error_description' in row: + print (row['error_description']) + + def execute(self, db : Connection): + cur = db.cursor() + + query = self.build_query() + print(query) + cur.execute(query) + self.print_error(cur) diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/MemoryLibraryManager.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/MemoryLibraryManager.py new file mode 100644 index 0000000..b592ffc --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/MemoryLibraryManager.py @@ -0,0 +1,192 @@ +# Copyright (C) 2024 Michael Becker +# +# This file is part of Mocha. +# +# Mocha is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Mocha is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Mocha. If not, see . + +from .MochaLibraryManager import MochaLibraryManager + +class MemoryLibraryManager (MochaLibraryManager): + + def __init__(self): + MochaLibraryManager.__init__(self) + self.filename = "" + self.codeDefinitionFileName = "" + + def process_attribute_ops(self, ops): + self.attribute_ops = ops + + def process_instance_ops(self, ops): + self.instance_ops = ops + + def process_relationship_ops(self, ops): + self.relationship_ops = ops + + def update_global_identifiers(self): + pass + + def update_creation_user(self): + pass + + def do_commit(self): + print("writing file '" + self.filename + "'") + + from .GuidCache import GuidCache + + guid_db = GuidCache() + strs_db = [ ] + + # *** FIRST PASS *** + + # first, go through and load all the instances GUIDs + for op in self.instance_ops: + guid_db.add(op.globalIdentifier) + guid_db.add(op.classGlobalIdentifier) + + # print ("INST: " + str(op.globalIdentifier) + " : " + str(op.classGlobalIdentifier) + " [ " + str(op.classIndex) + "$" + str(op.instanceIndex) + " ]") + + # next, go through and ensure instances and attributes are defined + for op in self.attribute_ops: + guid_db.add(op.instanceId) + guid_db.add(op.attributeInstanceId) + + if not str(op.value) in strs_db: + strs_db.append(str(op.value)) + + # print ("ATT: " + str(op.instanceId) + " . " + str(op.attributeInstanceId) + " = " + str(op.value)) + # finally, get the relationship instances + for op in self.relationship_ops: + guid_db.add(op.instanceId) + guid_db.add(op.relationshipInstanceId) + guid_db.add(op.targetInstanceId) + + # print ("REL: " + str(op.instanceId) + " . " + str(op.relationshipInstanceId) + " = " + str(op.targetInstanceId)) + + print ("processed " + str(guid_db.count()) + " instance references") + + # *** SECOND PASS *** + + # first, go through and load all the instances GUIDs + insts = [ ] + atts = [ ] + rels = [ ] + rsrc_db = [ ] + + rsrc_db.append(bytes([182, 128, 64, 48])) + + for op in self.instance_ops: + gi = guid_db.get_index(op.globalIdentifier) + cgi = guid_db.get_index(op.classGlobalIdentifier) + ckey = op.classIndex + ikey = op.instanceIndex + insts.append((gi, cgi, ckey, ikey)) + + print ("INST: " + str(op.globalIdentifier) + " : " + str(op.classGlobalIdentifier) + " [ " + str(op.classIndex) + "$" + str(op.instanceIndex) + " ]") + + # next, go through and ensure instances and attributes are defined + for op in self.attribute_ops: + srci = guid_db.get_index(op.instanceId) + atti = guid_db.get_index(op.attributeInstanceId) + vali = strs_db.index(str(op.value)) + + atts.append((srci, atti, vali, None)) + + print ("ATT: " + str(op.instanceId) + " . " + str(op.attributeInstanceId) + " = " + str(op.value)) + # finally, get the relationship instances + for op in self.relationship_ops: + srci = guid_db.get_index(op.instanceId) + reli = guid_db.get_index(op.relationshipInstanceId) + tgti = guid_db.get_index(op.targetInstanceId) + rels.append((srci, reli, tgti, None)) + + print ("REL: " + str(op.instanceId) + " . " + str(op.relationshipInstanceId) + " = " + str(op.targetInstanceId)) + + + guids = guid_db.to_list() + + f = open(self.filename, "wb") + f.write("MCX!".encode('utf-8')) + + # version + import struct + ba = bytearray(struct.pack("f", 2.0)) + f.write(ba) + + section_count = 6 + # section count + f.write(int.to_bytes(section_count, 4, 'little')) + + # sections = [ (offset, length) ] + sections = [ ("GUIDTable", guids, 16), ("Instances", insts, 16), ("Attributes", atts, 12), ("Relationships", rels, 12), ("StringTable", strs_db, None), ("Resources", rsrc_db, None) ] + offset = 12 + (section_count * 32) + + lastLength = 0 + for (name, sect, item_len) in sections: + offset += lastLength + length = 0 + + if item_len is None: + if sect == strs_db: + for itm in sect: + # FIXME: hard code for z-strings + length += len(itm) + 1 + elif sect == rsrc_db: + for itm in sect: + length += len(itm) + else: + length = item_len * len(sect) + + f.write(str(name).ljust(16, '\0').encode("utf-8")) + f.write(int.to_bytes(offset, 4, 'little')) + f.write(int.to_bytes(length, 4, 'little')) + f.write(int.to_bytes(len(sect), 4, 'little')) + f.write(int.to_bytes(0, 4, 'little')) # reserved or padding + + lastLength = length + + # guids + for guid in guids: + f.write(guid.to_bytes()) + + # instances + for (gi, cgi, ckey, ikey) in insts: + f.write(int.to_bytes(cgi, 4, 'little')) + f.write(int.to_bytes(gi, 4, 'little')) + f.write(int.to_bytes(ckey, 4, 'little')) + f.write(int.to_bytes(ikey, 4, 'little')) + + # attributes + for (srci, atti, vali, edate) in atts: + f.write(int.to_bytes(srci, 4, 'little')) + f.write(int.to_bytes(atti, 4, 'little')) + f.write(int.to_bytes(vali, 4, 'little')) + + # relationships + for (srci, reli, tgti, edate) in rels: + f.write(int.to_bytes(srci, 4, 'little')) + f.write(int.to_bytes(reli, 4, 'little')) + f.write(int.to_bytes(tgti, 4, 'little')) + + # string table + for strz in strs_db: + f.write(strz.encode("utf-8")) + f.write(int.to_bytes(0, 1, 'little')) + + # resource table + for rsrc in rsrc_db: + f.write(rsrc) + + f.close() + + print ("processed " + str(len(self.instance_ops)) + " instances, " + str(len(self.attribute_ops)) + " attributes, and " + str(len(self.relationship_ops)) + " relationships") \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/MochaLibraryManager.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/MochaLibraryManager.py new file mode 100644 index 0000000..3a1e17d --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/MochaLibraryManager.py @@ -0,0 +1,482 @@ +import MySQLdb + +from ..parser import XMLLibraryParser, JSONLibraryParser, YAMLLibraryParser + +from ..manager.LibraryOperation import LibraryOperation +from ..manager.operations.AssignAttributeOperation import AssignAttributeOperation +from ..manager.operations.AssignRelationshipOperation import AssignRelationshipOperation +from ..manager.operations.PrepareInstanceOperation import PrepareInstanceOperation +from ..manager.operations.StoredProcedureOperation import StoredProcedureOperation + +from .Normalization import Normalization + +from ...core.Guid import Guid +from ...core.KnownClassGuids import IDC_EntityDefinition, IDC_SourceDefinition +from ...core.KnownAttributeGuids import * +from ...core.KnownRelationshipGuids import * + +from ..InstanceCache import InstanceCache +from .SQLExpression import SQLExpression + +# from redis import Redis + +class MochaLibraryManager: + + def __init__(self): + self.entityReferences = dict() + self.entityReferenceFileNames = dict() + self.db = None + self._instances = [] + self._attOps = [] + self._relOps = [] + self.redis = None + + def connect(self, hostname, database, username, password): + self.db = MySQLdb.connect(host=hostname, user=username, passwd=password, db=database) + # self.redis = Redis(host=hostname, port=6379, decode_responses=True) + + def preprocess_instance_ops(self): + # this version is much faster as it prepares everything in advanced on the CPU before + # pushing it out to the database + + instances = InstanceCache() + + for key in self._instances: + (class_gid, inst_gid, class_index, inst_index) = key + # first import all the known classes + if class_gid is not None and class_index is not None: + # print ("known class import") + # print ("class_gid = " + class_gid) + # print ("inst_gid = " + inst_gid + " (" + str(class_index) + "$" + str(inst_index) + ")") + if not instances.has_instance_by_global_identifier(class_gid): + instances.add_instance((1, class_index), class_gid) + + for key in self._instances: + (class_gid, inst_gid, class_index, inst_index) = key + # now get all the classes without specified IIDs + if class_gid is not None and class_index is None: + # print ("unknown class import") + if not instances.has_instance_by_global_identifier(class_gid): + class_index = instances.get_next_instance_id(class_index) + else: + class_index = instances.get_instance_key(class_gid)[1] + + if inst_index is None: + inst_index = instances.get_next_instance_id(class_index) + + instances.add_instance((1, inst_index), inst_gid) + + # print ("class_gid = " + class_gid) + # print ("inst_gid = " + inst_gid + " (" + str(1) + "$" + str(inst_index) + ")") + + for key in self._instances: + (class_gid, inst_gid, class_index, inst_index) = key + if class_gid is not None and class_index is not None and inst_gid is not None and inst_index is not None: + # we know all there is to know about this one + pass + elif class_gid is not None and inst_gid is not None: + if class_index is None: + # we need to know the class index + if instances.has_instance_by_global_identifier(class_gid): + class_index = instances.get_instance_key(class_gid)[1] + else: + class_index = instances.get_next_instance_id(1) + + if inst_index is None: + # we need to know the instance index as well + if instances.has_instance_by_global_identifier(inst_gid): + inst_index = instances.get_instance_key(inst_gid)[1] + else: + inst_index = instances.get_next_instance_id(class_index) + else: + print("error: not implemened class_gid is None or inst_gid is None") + + instances.add_instance((class_index, inst_index), inst_gid) + + print (str(instances.count()) + " instances preloaded") + + self.instances = instances + return True + + def process_instance_ops(self, instances): + c = 0 + query = "INSERT INTO mocha_instances (tenant_id, class_id, inst_id, global_identifier) VALUES " + for inst_gid in instances.get_instance_keys(): + (class_index, inst_index) = instances.get_instance_key(inst_gid) + print("preparing inst " + str(class_index) + "$" + str(inst_index) + " with guid " + inst_gid) + + inst_key_str = str(class_index) + "$" + str(inst_index) + + if not self.redis is None: + self.redis.set(inst_key_str + ".gid", Normalization.normalize_uuid(inst_gid)) + self.redis.set(inst_key_str + ".attributes.count", "0") + self.redis.set(inst_key_str + ".relationships.count", "0") + self.redis.set(Normalization.normalize_uuid(inst_gid) + ".iid", inst_key_str) + self.redis.set(Normalization.normalize_uuid(inst_gid) + ".attributes.count", "0") + self.redis.set(Normalization.normalize_uuid(inst_gid) + ".relationships.count", "0") + + query += "(1, " + str(class_index) + ", " + str(inst_index) + ", '" + Normalization.normalize_uuid(inst_gid) + "')" + + if c < instances.count() - 1: + query += ", " + + c += 1 + + print ("process_instance_ops: query was: ") + print (query) + self.db.query(query) + + def process_attribute_ops(self, ops): + query = "INSERT INTO mocha_attributes (tenant_id, src_inst_id, att_inst_id, usr_inst_id, att_effective_date, att_value) VALUES " + for op in ops: + instanceId = op.instanceId + attributeInstanceId = op.attributeInstanceId + value = op.value + + if attributeInstanceId.get_value() == "": + print("skipping invalid blank attribute") + continue + + print (str(instanceId) + " : " + str(attributeInstanceId) + " = '" + str(value) + "'") + + # query += "(1, " \ + # + "mocha_get_instance_by_global_identifier('" + Normalization.normalize_uuid(instanceId.get_value()) + "'), " \ + # + "mocha_get_instance_by_global_identifier('" + Normalization.normalize_uuid(attributeInstanceId.get_value()) + "'), " \ + # + "NULL, NOW(), " + SQLExpression.normalize_parm(value) + ")" + + query += "(1, " \ + + str(self.instances.get_database_id_by_global_identifier(instanceId)) + ", " \ + + str(self.instances.get_database_id_by_global_identifier(attributeInstanceId)) + ", " \ + + "NULL, NOW(), " + SQLExpression.normalize_parm(value) + ")" + + if ops.index(op) < len(ops) - 1: + query += ", " + + # op.execute(self.db) + + print ("process_attribute_ops: query was: ") + print (query) + self.db.query(query) + + def process_relationship_ops(self, ops): + query = "INSERT INTO mocha_relationships (tenant_id, source_inst_id, relationship_inst_id, destination_inst_id, user_inst_id, effective_date) VALUES " + + for op in ops: + instanceId = op.instanceId + relationshipInstanceId = op.relationshipInstanceId + targetInstanceId = op.targetInstanceId + + print (str(instanceId) + " : " + str(relationshipInstanceId) + " = " + str(targetInstanceId)) + + query += "(1, " \ + + str(self.instances.get_database_id_by_global_identifier(instanceId)) + ", " \ + + str(self.instances.get_database_id_by_global_identifier(relationshipInstanceId)) + ", " \ + + str(self.instances.get_database_id_by_global_identifier(targetInstanceId)) + ", " \ + + "NULL, NOW() )" + + if ops.index(op) < len(ops) - 1: + query += ", " + + # op.execute(self.db) + + print ("process_relationship_ops: query was: ") + print (query) + self.db.query(query) + + def before_commit(self): + """ + Called before the unstructured information is sorted and written to the database. + --- + This is your last chance to add instances, set attributes, and assign + relationships before the compilation is finalized. After this stage, calls made + to modify the compilation are undefined and will almost certainly not do what you + want. + """ + + self.write_debug_information() + + def do_commit(self): + self.db.commit() + + def write_debug_information(self): + """ + Writes debugging information, such as entity definitions ('&IDC_...;') and source + code definitions (filename, line number, column number, etc.) + """ + + print ("preparing debug information (use --no-debug or undefine DEBUG or etc. [NOT IMPLEMENTED] to turn off)") + for key in self.entityReferences: + val = self.entityReferences[key] + + gidEntityDefinition = Guid.create() + self.add_instance(IDC_EntityDefinition, gidEntityDefinition, None, None) + self.set_attribute_value(gidEntityDefinition, IDA_Name, key) + self.set_attribute_value(gidEntityDefinition, IDA_Value, val) + + gidSourceDefinition = Guid.create() + self.add_instance(IDC_SourceDefinition, gidSourceDefinition, None, None) + self.set_attribute_value(gidSourceDefinition, IDA_DebugDefinitionFileName, self.entityReferenceFileNames[key]) + + self.assign_relationship(gidEntityDefinition, IDR_Instance__has__Source_Definition, gidSourceDefinition) + + # Source Definition.has X + # self.add_instance(KnownClassGuids.CLASS, Guid.parse("{dc0a5dd2-22e0-471f-87cf-a5ef1b764efa}"), 1, index) + # self.assign_relationship(instGid, Guid.parse("{dc0a5dd2-22e0-471f-87cf-a5ef1b764efa}"), targetInstanceId) + + def update_global_identifiers(self): + query = "SELECT id, global_identifier FROM mocha_instances" + + cur = self.db.cursor() + cur.execute(query) + rows = cur.fetchall() + + for (dbid, gid) in rows: + self.instances.set_database_id_by_global_identifier(Guid.parse(gid), dbid) + + def update_creation_user(self): + cur = self.db.cursor() + # cur.execute("UPDATE mocha_instances SET user_inst_id = mocha_get_instance_by_global_identifier(mocha_normalize_uuid('{B066A54B-B160-4510-A805-436D3F90C2E6}'))") + cur.execute("UPDATE mocha_attributes SET usr_inst_id = " + str(self.instances.get_database_id_by_global_identifier(Guid.parse("{B066A54B-B160-4510-A805-436D3F90C2E6}")))) + cur.execute("UPDATE mocha_relationships SET user_inst_id = " + str(self.instances.get_database_id_by_global_identifier(Guid.parse("{B066A54B-B160-4510-A805-436D3F90C2E6}")))) + + def commit(self): + + self.before_commit() + + # first record all the instances + if not self.preprocess_instance_ops(): + return False + + instance_ops = [ ] + for inst_gid in self.instances.get_instance_keys(): + (class_index, inst_index) = self.instances.get_instance_key(inst_gid) + class_gid = self.instances.get_global_identifier((1, class_index)) + + if not isinstance(class_gid, Guid): + print ("class_gid is not Guid") + + instance_ops.append(PrepareInstanceOperation(class_gid, inst_gid, class_index, inst_index)) + + self.process_instance_ops(instance_ops) + + print("reading database ids...") + self.update_global_identifiers() + + print("assigning attributes...") + self.process_attribute_ops(self._attOps) + + print("assigning relationships...") + self.process_relationship_ops(self._relOps) + + hasSiblingRelationships = dict() + for op in self._relOps: + relationshipKey = self.instances + + if str(op.relationshipInstanceId) == "{656110ff-4502-48b8-a7f3-d07f017aea3f}": + # Relationship.has sibling Relationship + hasSiblingRelationships[op.instanceId] = op.targetInstanceId + + siblingOps = [] + for op in self._relOps: + if op.relationshipInstanceId in hasSiblingRelationships: + siblingOp = AssignRelationshipOperation(op.targetInstanceId, hasSiblingRelationships[op.relationshipInstanceId], op.instanceId) + siblingOps.append(siblingOp) + # print ("assigning sibling relationship " + siblingOp.instanceId.get_value() + " . " + siblingOp.relationshipInstanceId.get_value() + " = " + siblingOp.instanceId.get_value()) + # siblingOp.execute(self.db) + + print("assigning sibling relationships...") + self.process_relationship_ops(siblingOps) + + print("setting creation user...") + self.update_creation_user() + + # rows = cur.fetchall() + # print(rows) + + # print ("ok , applying `Class.has Instance`...") + + # for row in rows: + # print(row) + # id = row[0] + # tenant_id = row[1] + # class_id = row[2] + # inst_id = row[3] + # global_identifier = row[4] + + # if class_id is not None and id is not None: + + # # Class.has Instance + # query2 = ("CALL mocha_assign_relationship (" + + # "mocha_get_instance_by_key(1, " + str(class_id) + "), " + + # "mocha_get_instance_by_global_identifier(mocha_normalize_uuid('7EB41D3C-2AE9-4884-83A4-E59441BCAEFB'))" + ", " + + # str(id) + ", NULL, NULL)") + # cur.execute(query2) + + # # Instance.for Class + # query3 = ("CALL mocha_assign_relationship (" + + # str(id) + ", " + + # "mocha_get_instance_by_global_identifier(mocha_normalize_uuid('494D5A6D-04BE-477B-8763-E3F57D0DD8C8'))" + ", " + + # "mocha_get_instance_by_key(1, " + str(class_id) + "), NULL, NULL)") + # cur.execute(query3) + self.do_commit() + print("done!") + + def close(self): + self.db.close() + + def print_error(self, cur): + rows = cur.fetchall() + for row in rows: + if 'error_description' in row: + print (row['error_description']) + + def select_tenant(self, tenantName): + cur = self.db.cursor() + cur.execute("CALL mocha_select_tenant(mocha_get_tenant_by_name('" + tenantName + "'))") + self.print_error(cur) + + def release_tenant(self): + cur = self.db.cursor() + cur.execute("CALL mocha_release_tenant()") + self.print_error(cur) + + def assign_relationship(self, instanceId : Guid, relationshipInstanceId : Guid, targetInstanceId : Guid): + print("-- assigning relationship " + str(relationshipInstanceId) + " = '" + str(targetInstanceId) + "'") + self._relOps.append(AssignRelationshipOperation(instanceId, relationshipInstanceId, targetInstanceId)) + # self._relOps.append(AssignRelationshipOperation(instanceId, relationshipInstanceId, targetInstanceId)) + # query = "CALL mocha_assign_relationship(mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + instanceId.get_value() + "')), mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + relationshipInstanceId.get_value() + "')), mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + targetInstanceId.get_value() + "')), NULL, NULL);" + # print(query) + # cur.execute(query) + # self.print_error(cur) + + def set_attribute_value(self, instanceId : Guid, attributeInstanceId : Guid, value): + print("-- assigning attribute " + str(attributeInstanceId) + " = '" + str(value) + "'") + self._attOps.append(AssignAttributeOperation(instanceId, attributeInstanceId, value)) + + def install_from_path(self, path): + + from glob import glob + + xl = XMLLibraryParser(self) + jl = JSONLibraryParser(self) + yl = YAMLLibraryParser(self) + + #xml_files = glob(path + "/**/*.xml", recursive=True) + #for xml_file in xml_files: + #s xl.load_file(xml_file) + + + #json_files = glob(path + "/**/*.json", recursive=True) + #for json_file in json_files: + # jl.load_file(json_file) + + yaml_files = sorted(glob(path + "/**/*.yaml", recursive=True)) + if len(yaml_files) == 0: + print ("no files found ; does the path exist?") + return 3 + + # first, load the entity defs + for yaml_file in yaml_files: + yl.load_entity_definitions_from_file(yaml_file) + + try: + # then, load instance definitions (also loads sugar elements into memory for later use) + for yaml_file in yaml_files: + yl.load_instances_from_file(yaml_file) + + # finally, apply syntactic sugar + yl.apply_sugar() + + return True + + except NameError as ex: + + print (ex) + + rgx = "&(.*);" + + # go through and get all entity references across all files + import re + import fileinput + + stuff = [] + + for yaml_file in yaml_files: + f = open(yaml_file, "r") + text = f.read() + matches = re.findall(rgx, text) + for match in matches: + stuff.append(match) + f.close() + + missingEntities = [] + for stuf in stuff: + if not stuf in self.entityReferences: + if not stuf in missingEntities: + missingEntities.append(stuf) + + if len(missingEntities) > 0: + print("\nNOTE: there were undefined referenced entities:\n") + for missingEntity in missingEntities: + print("\t" + missingEntity) + print("\n") + + return False + + + def exists_entity_reference(self, name): + return name in self.entityReferences + + def register_entity_reference(self, name, value, filename = None): + self.entityReferences[name] = value + self.entityReferenceFileNames[name] = filename + + def define_entity_reference(self, name): + return self.entityReferences[name] + def expand_entity_references(self, value): + insideName = False + name = "" + retval = "" + for i in range(0, len(value)): + if value[i] == "&": + insideName = True + elif value[i] == ';': + insideName = False + + if name in self.entityReferences: + retval += self.define_entity_reference(name) + else: + print("unknown entity ref '" + name + "'") + raise NameError("unknown entity ref '" + name + "'") + + name = "" + elif insideName: + name += value[i] + else: + retval += value[i] + return retval + + def add_instance(self, classGid : Guid, instGid : Guid, classIndex : int, index : int): + print("adding instance for class '" + str(classGid) + "' with globalid '" + str(instGid) + "' [" + str(index) + "]") + self._instances.append((classGid, instGid, classIndex, index)) + + # assign relationship `Instance.for Class` + self.assign_relationship(instGid, Guid.parse('494D5A6D04BE477B8763E3F57D0DD8C8'), classGid) + + # assign relationship `Class.has Instance` + self.assign_relationship(classGid, Guid.parse('7EB41D3C2AE9488483A4E59441BCAEFB'), instGid) + + def save_entities_to_file(self, filename : str): + f = open(filename, "w") + print("public static class KnownInstances", file=f) + print("{", file=f) + for name in self.entityReferences: + if name.startswith("IDC_") or name.startswith("IDA_") or name.startswith("IDI_"): + shortname = name + if name.startswith("ID") and "_" in name: + shortname = name[name.index("ID") + 2:] + + print("\tpublic static readonly Guid " + shortname + " = new Guid(\"" + self.entityReferences[name] + "\");", file=f) + + print("}", file=f) + f.close() diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/Normalization.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/Normalization.py new file mode 100644 index 0000000..d05c8e2 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/Normalization.py @@ -0,0 +1,5 @@ +class Normalization: + @staticmethod + def normalize_uuid(val : str): + val = val.replace("{", "").replace("}", "").replace("-", "").strip().lower() + return val \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLExpression.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLExpression.py new file mode 100644 index 0000000..4f40613 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLExpression.py @@ -0,0 +1,47 @@ +from ...core.Guid import Guid +from .Normalization import Normalization + +class SQLExpression: + + @staticmethod + def sqlescape(parm : str): + return parm.replace("'", "\\'") + + @staticmethod + def normalize_parm(parm : str): + query = "" + if parm == None: + query += "NULL" + elif isinstance(parm, Guid): + gid = None + if parm is not None: + gid = "'" + Normalization.normalize_uuid(parm.get_value()) + "'" + else: + gid = "NULL" + + query += gid + elif isinstance(parm, str): + query += "'" + SQLExpression.sqlescape(str(parm)) + "'" + else: + query += str(parm) + + return query + + @staticmethod + def to_string(parm): + return SQLExpression.normalize_parm(parm) + + @staticmethod + def array_to_string(parms): + i = 0 + query = '' + for parm in parms: + + query += SQLExpression.to_string(parm) + + if i < len(parms) - 1: + query += ", " + + i += 1 + + return query \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLFunctionCall.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLFunctionCall.py new file mode 100644 index 0000000..13dc71f --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLFunctionCall.py @@ -0,0 +1,20 @@ +from .SQLExpression import SQLExpression + +class SQLFunctionCall (SQLExpression): + + def __init__(self, funcname, parms): + self.name = funcname + self.parms = parms + + def get_name(self): + return self.name + + def get_parameters(self): + return self.parms + + def get_parameter_list(self): + return SQLExpression.array_to_string(self.parms) + + def __str__(self): + return self.get_name() + "(" + self.get_parameter_list() + ")" + diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLParameter.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLParameter.py new file mode 100644 index 0000000..70ac50c --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/SQLParameter.py @@ -0,0 +1,12 @@ +from .SQLExpression import SQLExpression + +class SQLParameter (SQLExpression): + + def __init__(self, name): + self.name = name + + def get_name(self): + return self.name + + def __str__(self): + return "@" + self.get_name() \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/__init__.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/__init__.py new file mode 100644 index 0000000..e4bd1d7 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2024 Michael Becker +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .MochaLibraryManager import MochaLibraryManager +from .MemoryLibraryManager import MemoryLibraryManager diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/AssignAttributeOperation.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/AssignAttributeOperation.py new file mode 100644 index 0000000..6b35c64 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/AssignAttributeOperation.py @@ -0,0 +1,21 @@ +from .StoredProcedureOperation import StoredProcedureOperation +from ....core.Guid import Guid +from ..SQLParameter import SQLParameter +from ..SQLFunctionCall import SQLFunctionCall + +class AssignAttributeOperation(StoredProcedureOperation): + + def __init__(self, instanceId : Guid, attributeInstanceId : Guid, value): + self.instanceId = instanceId + self.attributeInstanceId = attributeInstanceId + self.value = value + + def get_sp_name(self): + return "mocha_set_attribute_value" + + def get_sp_parameters(self): + return [ SQLFunctionCall('mocha_get_instance_by_global_identifier', [ self.instanceId ]), SQLFunctionCall('mocha_get_instance_by_global_identifier', [ self.attributeInstanceId ]), str(self.value), None, None ] + #query = "CALL mocha_set_attribute_value(mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + instanceId.get_value() + "')), mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + attributeInstanceId.get_value() + "')), '" + str(value) + "', NULL, NULL);" + #print(query) + #cur.execute(query) + #self.print_error(cur) \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/AssignRelationshipOperation.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/AssignRelationshipOperation.py new file mode 100644 index 0000000..eb48707 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/AssignRelationshipOperation.py @@ -0,0 +1,21 @@ +from .StoredProcedureOperation import StoredProcedureOperation +from ....core.Guid import Guid +from ..SQLParameter import SQLParameter +from ..SQLFunctionCall import SQLFunctionCall + +class AssignRelationshipOperation(StoredProcedureOperation): + + def __init__(self, instanceId : Guid, relationshipInstanceId : Guid, targetInstanceId : Guid): + self.instanceId = instanceId + self.relationshipInstanceId = relationshipInstanceId + self.targetInstanceId = targetInstanceId + + def get_sp_name(self): + return "mocha_assign_relationship" + + def get_sp_parameters(self): + return [ SQLFunctionCall('mocha_get_instance_by_global_identifier', [ self.instanceId ]), SQLFunctionCall('mocha_get_instance_by_global_identifier', [ self.relationshipInstanceId ]), SQLFunctionCall('mocha_get_instance_by_global_identifier', [ self.targetInstanceId ]), None, None ] + #query = "CALL mocha_set_attribute_value(mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + instanceId.get_value() + "')), mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + attributeInstanceId.get_value() + "')), '" + str(value) + "', NULL, NULL);" + #print(query) + #cur.execute(query) + #self.print_error(cur) diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/CreateClassOperation.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/CreateClassOperation.py new file mode 100644 index 0000000..1571628 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/CreateClassOperation.py @@ -0,0 +1,17 @@ +from .StoredProcedureOperation import StoredProcedureOperation +from ..SQLParameter import SQLParameter +from ..SQLFunctionCall import SQLFunctionCall +from ....core.Guid import Guid + +class CreateClassOperation (StoredProcedureOperation): + + def __init__(self, globalIdentifier : Guid, classIndex : int): + self.globalIdentifier = globalIdentifier + self.classIndex = classIndex + + def get_sp_name(self): + return 'mocha_create_class' + + def get_sp_parameters(self): + return [ self.classIndex, self.globalIdentifier, None, None, SQLParameter("dummy") ] + \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/CreateInstanceOperation.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/CreateInstanceOperation.py new file mode 100644 index 0000000..b954868 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/CreateInstanceOperation.py @@ -0,0 +1,17 @@ +from .StoredProcedureOperation import StoredProcedureOperation +from ....core.Guid import Guid +from ..SQLParameter import SQLParameter +from ..SQLFunctionCall import SQLFunctionCall + +class CreateInstanceOperation(StoredProcedureOperation): + + def __init__(self, globalIdentifier : Guid, classIndex : int, instanceIndex : int): + self.globalIdentifier = globalIdentifier + self.classIndex = classIndex + self.instanceIndex = instanceIndex + + def get_sp_name(self): + return 'mocha_create_instance' + + def get_sp_parameters(self): + return [ self.globalIdentifier, self.classIndex, self.instanceIndex, None, None, SQLParameter('assigned_inst_id') ] diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/PrepareInstanceOperation.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/PrepareInstanceOperation.py new file mode 100644 index 0000000..243a06c --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/PrepareInstanceOperation.py @@ -0,0 +1,41 @@ +from .StoredProcedureOperation import StoredProcedureOperation +from ....core.Guid import Guid +from ..SQLParameter import SQLParameter +from ..SQLFunctionCall import SQLFunctionCall + +class PrepareInstanceOperation(StoredProcedureOperation): + + def __init__(self, classGlobalIdentifier : Guid, globalIdentifier : Guid, classIndex : int, instanceIndex : int): + if not isinstance(classGlobalIdentifier, Guid): + raise TypeError + if not isinstance(globalIdentifier, Guid): + raise TypeError + + self.classGlobalIdentifier = classGlobalIdentifier + self.globalIdentifier = globalIdentifier + self.classIndex = classIndex + self.instanceIndex = instanceIndex + + def get_sp_name(self): + return "mocha_prepare_instance" + + def get_sp_parameters(self): + parms = [] + + globalId = None + if self.globalIdentifier is not None: + globalId = self.globalIdentifier + + classGlobalId = None + if self.classGlobalIdentifier is not None: + classGlobalId = self.classGlobalIdentifier + + strCid = None + if self.classIndex is not None: + strCid = self.classIndex + + strIid = None + if self.instanceIndex is not None: + strIid = self.instanceIndex + + return [strCid, strIid, classGlobalId, globalId, None, None, SQLParameter("p_assigned_inst_id")] diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/StoredProcedureOperation.py b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/StoredProcedureOperation.py new file mode 100644 index 0000000..3f16f8b --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/manager/operations/StoredProcedureOperation.py @@ -0,0 +1,24 @@ +import MySQLdb + +from ..LibraryOperation import LibraryOperation +from MySQLdb.connections import Connection +from ..SQLExpression import SQLExpression +from ..SQLParameter import SQLParameter +from ..SQLFunctionCall import SQLFunctionCall +from ....core.Guid import Guid + +class StoredProcedureOperation (LibraryOperation): + def get_sp_name(self): + return '' + + def get_sp_parameters(self): + return [] + + def build_query(self): + query = "CALL " + self.get_sp_name() + "(" + parms = self.get_sp_parameters() + query += SQLExpression.array_to_string(parms) + + query += ")" + return query + diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/parser/JSONLibraryParser.py b/mocha-common/compilers/yaml2mcl/mocha/library/parser/JSONLibraryParser.py new file mode 100644 index 0000000..be6b164 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/parser/JSONLibraryParser.py @@ -0,0 +1,30 @@ +from .LibraryParser import LibraryParser + +class JSONLibraryParser (LibraryParser): + + def cstyle_strip_comments(self, val): + val2 = "" + # possible values for comment: 0 = none, 1 = multiline, 2 = singleline + comment = 0 + + xr = iter(range(0, len(val) - 1)) + for i in xr: + if val[i] == '\n' and comment == 2: + comment = 0 + + check = val[i:i+2] + if check == "/*" and comment == 0: + comment = 1 + next(xr) + elif check == "*/" and comment == 1: + comment = 0 + next(xr) + elif check == "//": + comment = 2 + next(xr) + elif comment == 0: + val2 += val[i] + + val2 += val[-1:] + return val2 + diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/parser/LibraryParser.py b/mocha-common/compilers/yaml2mcl/mocha/library/parser/LibraryParser.py new file mode 100644 index 0000000..7c7ea67 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/parser/LibraryParser.py @@ -0,0 +1,4 @@ +class LibraryParser(): + def __init__(self, manager): + + self.__manager = manager diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/parser/MochaSyntaxError.py b/mocha-common/compilers/yaml2mcl/mocha/library/parser/MochaSyntaxError.py new file mode 100644 index 0000000..8b09f79 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/parser/MochaSyntaxError.py @@ -0,0 +1,5 @@ +class MochaSyntaxError(RuntimeError): + def __init__(self, *args): + RuntimeError.__init__(self, args) + + pass diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/parser/XMLLibraryParser.py b/mocha-common/compilers/yaml2mcl/mocha/library/parser/XMLLibraryParser.py new file mode 100644 index 0000000..acc9029 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/parser/XMLLibraryParser.py @@ -0,0 +1,46 @@ +from .LibraryParser import LibraryParser + +from xml.dom import minidom, Node +from xml.dom.minidom import Entity + +class XMLLibraryParser(LibraryParser): + + def load_instance(self, xmlnode): + print("instance : [" + xmlnode.getAttribute("id") + "]") + + def load_library(self, xmlnode): + """ + Loads a library from the specified XML node. + """ + + raise NotImplementedError() + + if xmlnode.tagName != "library": + return + + id = xmlnode.getAttribute("id") + for child in xmlnode.childNodes: + if child.tagName == "instances": + for child2 in child.childNodes: + self.load_library_instance_from_xml(child) + + + def load_file(self, filename): + print("loading xml from " + filename + "... ") + + dom = minidom.getDOMImplementation() + dt = dom.createDocumentType("mocha", "-//MBS//DTD Mocha 1.0 Dev//EN", "https://doctype.schemas.alcetech.net/Mocha/1.0/mocha-1.0.dtd") + dt.entities.setNamedItem(Entity("IDC_ReturnInstanceSetMethodBinding", "{AADC20F9-7559-429B-AEF0-97E059295C76}", None, None)) + + dom = minidom.parse(filename) + if (dom.documentElement.tagName != "mocha"): + print(filename + ": top-level tag is not 'mocha'") + return + + for child in dom.documentElement.childNodes: + if child.nodeType == Node.ELEMENT_NODE: + if child.tagName == "libraries": + for child2 in child.childNodes: + self.load_library(child) + + print ("\n") \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/parser/YAMLLibraryParser.py b/mocha-common/compilers/yaml2mcl/mocha/library/parser/YAMLLibraryParser.py new file mode 100644 index 0000000..d53722a --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/parser/YAMLLibraryParser.py @@ -0,0 +1,345 @@ +from .LibraryParser import LibraryParser +from ...core.Guid import Guid +from .MochaSyntaxError import MochaSyntaxError + +class YAMLLibraryParser (LibraryParser): + + def __init__(self, manager): + LibraryParser.__init__(self, manager) + + self.yamlIds = dict() + self.templates = dict() + + # these two are special reserved tagNames + # self.yamlIds["entityDefinitions"] = None + # self.yamlIds["instance"] = { } + + self.manager = manager + self._instances_awaiting_sugar = [] + + def apply_sugar(self): + print("applying syntactic sugar to remaining instances...") + # _instances_also_awaiting_sugar = [] + + for inst in self._instances_awaiting_sugar: + print("applying sugar from '" + inst["from_file_name"] + "'") + self.load_instance(inst) + print("\n") + + # do recursively until we're all out of instances awaiting sugar + # if len(self._instances_awaiting_sugar) > 0: + # self.apply_sugar() + + def load_library(self, elem): + print("loading library from elem") + + def load_tenant(self, elem): + print("loading tenant from elem") + + def find_custom_tag_name(self, elem) -> tuple[str, Guid]: + # return tuple (tagname, value) + if "instance" in elem: + return ("instance", Guid.parse(self.manager.expand_entity_references(elem["instance"]))) + + for key in self.yamlIds: + if key in elem: + k = str(key) + v = Guid.parse(self.manager.expand_entity_references(elem[key])) + return (k, v) + + return (None, None) + + + def load_entity_definitions_from_file(self, filename): + print("loading entity defs from " + filename) + + import yaml + with open(filename, 'r') as file: + content = yaml.safe_load_all(file) + for doc in content: + for elem in doc: + if "entityDefinitions" in elem: + entities = elem["entityDefinitions"] + if not entities is None: + for entity in entities: + for key in entity: + if self.manager.exists_entity_reference(key): + print("duplicate entity definition: '" + key + "'") + exit(1) + + self.manager.register_entity_reference(key, entity[key], filename) + + def load_instance_with_template(self, elem, template): + (parentCreatorKey, parentInstanceId) = self.find_custom_tag_name(template) + (creatorKey, instanceId) = self.find_custom_tag_name(elem) + + self.apply_template(elem, template, instanceId) + + def assign_relationship_value(self, instanceId, rel, rel_iid, relationshipValue): + if (isinstance(relationshipValue, str)): + # single instance, this should be a GUID or entityref + # we should only need to refer to existing instance in this case + relationshipValueInst = Guid.parse(self.manager.expand_entity_references(relationshipValue)) + if "instance" in rel: + self.manager.assign_relationship(instanceId, Guid.parse(self.manager.expand_entity_references(rel["instance"])), relationshipValueInst) + else: + print("no relationship instance specified for relationship sugar") + + else: + # dynamically created instance + #FIXME: this 'instance' isn't always the tag name, this can be subject to customTagName as well!and should be processed accordingly + (relinst_key, relinst_iid) = self.find_custom_tag_name(relationshipValue) + if relinst_key is not None: + # for example, 'relinst_key' is 'elementContent' and 'relinst_iid' is '{8ECA14A4...}' + + if rel_iid is not None: + print("found customTagName '" + str(relinst_key) + "' with value '" + str(relinst_iid) + "'") + + # relval = Guid.parse(self.manager.expand_entity_references(elem[customTagName])) + # we need to create the new instance and assign relationship + self.manager.assign_relationship(instanceId, rel_iid, relinst_iid) + else: + if "globalIdentifier" in relationshipValue: + globalIdentifier = Guid.parse(relationshipValue["globalIdentifier"]) + else: + globalIdentifier = Guid.create() + + print("creating new instance for relationship '%s' with globalid '%s'" % (rel_iid, globalIdentifier)) + + createsInstanceOf = None + if "type" in relationshipValue: + print("found relationship override type '%s'" % (relationshipValue['type'])) + createsInstanceOf = Guid.parse(self.manager.expand_entity_references(relationshipValue["type"])) + + elif "customTagNameCreatesInstanceOf" in rel: + createsInstanceOf = Guid.parse(self.manager.expand_entity_references(rel["customTagNameCreatesInstanceOf"])) + + if not createsInstanceOf is None: + # create the new instance + self.manager.add_instance(createsInstanceOf, globalIdentifier, None, None) + + # assign relationships + self.manager.assign_relationship(instanceId, rel_iid, globalIdentifier) + # self.manager.assign_relationship(globalIdentifier, relationshipInstanceId, instanceId) + + # FIXME: apply relationships from the parent class template + + #? NOTE: You MUST specify 'registerForTemplate: yes' on the template definition for + #? sibling relationships to be properly applied! + + if createsInstanceOf in self.templates: + print("applying template for createsInstanceOf '%s'" % (createsInstanceOf)) + createsInstanceOfTemplate = self.templates[createsInstanceOf] + self.apply_template(relationshipValue, createsInstanceOfTemplate, globalIdentifier) + else: + print("no template registered for createsInstanceOf '%s'" % (createsInstanceOf)) + else: + print ("neither createsInstanceOf nor override 'type' present for relationship value") + + def apply_template(self, elem, template, instanceId): + + if "value" in elem: + print("VALUE found in ELEM:") + print(elem["value"]) + + if "attributes" in template: + attrs = template["attributes"] + if isinstance(attrs, list): + for attr in attrs: + if "customTagName" in attr: + customTagName = attr["customTagName"] + if customTagName in elem: + attrinst = Guid.parse(self.manager.expand_entity_references(attr["instance"])) + attrval = elem[customTagName] + self.manager.set_attribute_value(instanceId, attrinst, attrval) + + if "relationships" in template: + rels = template["relationships"] + for rel in rels: + if "instance" in rel: + rel_iid = rel["instance"] # the globalId of the relationship instance + else: + print("no relationship instance specified for relationship sugar") + continue + + if "customTagName" in rel: + customTagName = rel["customTagName"] + if customTagName in elem: + relationshipValue = elem[customTagName] + if relationshipValue is None: + continue + + if isinstance(relationshipValue, list): + # multiple instances + for v in relationshipValue: + self.assign_relationship_value(instanceId, rel, Guid.parse(self.manager.expand_entity_references(rel_iid)), v) + + else: + self.assign_relationship_value(instanceId, rel, Guid.parse(self.manager.expand_entity_references(rel_iid)), relationshipValue) + + def get_instance_sugar_for_elem(self, elem): + if "instance" in elem: + return (elem["instance"], "instance", None) + else: + for key in self.yamlIds: + # no 'instance', see if we + if key in elem: + template = self.yamlIds[key] + templateKey = key + return (elem[templateKey], templateKey, template) + + return (None, None, None) # should never get here + + + def load_instance(self, elem, elemParent = None): + # 'class' and 'relationship' are both at the 'same level' + # so we can't define 'relationship' without first defining 'class' + + # we could try to keep looping as long as we're still getting "unknown tag name" + # (instanceID is None), but we could get stuck that way... + + # for now, let's just forward define important classes (e.g. IDC_Relationship) + + + # all of these problems could be avoided by simply + # LOADING EVERYTHING INTO MEMORY (like the .NET version does) + # and only committing to the DB at the very end + # this will resolve problems where we can't find instances, or use syntactic sugar + + (globalIdentifier, templateKey, template) = self.get_instance_sugar_for_elem(elem) + print ("globalIdentifier = " + str(globalIdentifier) + ", templateKey = " + str(templateKey)) + + if template is None: + # no sugar + pass + + customTagName = None + if "customTagName" in elem: + customTagName = elem["customTagName"] + self.yamlIds[customTagName] = elem + print("registering customTagName '" + customTagName + "'") + + if "registerForTemplate" in elem: + if elem["registerForTemplate"] == True: + self.templates[self.manager.expand_entity_references(globalIdentifier)] = elem + print("registering elem for template") + + classId = None + classIndex = None + + if template is None and elemParent is not None: + template = elemParent + + if template is not None: + if "instance" in template: + classId = Guid.parse(self.manager.expand_entity_references(template["instance"])) + else: + for key in self.yamlIds: + if key in template: + classId = Guid.parse(self.manager.expand_entity_references(template[key])) + break + + if "index" in template: + classIndex = template["index"] + + instanceId = None + creatorKey = None + if "instance" in elem: + if "templateOnly" in elem and elem["templateOnly"] == True: + # instanceId = None + instanceId = Guid.parse(self.manager.expand_entity_references(elem["instance"])) + else: + instanceId = Guid.parse(self.manager.expand_entity_references(elem["instance"])) + + creatorKey = 'instance' + else: + for key in self.yamlIds: + if key in elem: + instanceId = Guid.parse(self.manager.expand_entity_references(elem[key])) + creatorKey = key + break + + index = None + if "index" in elem: + index = elem["index"] + # else: + # print("index not found for element") + # print(elem) + + if classId is None and instanceId is not None: + #!!! HACK HACK HACK !!! + classId = instanceId + if classIndex is None and index is None: + classIndex = 1 + index = 1 + print("WARNING: class hack used for instanceId " + str(instanceId) + " ( index " + str(classIndex) + "$" + str(index) + ")") + + if instanceId is not None: + self.manager.add_instance(classId, instanceId, classIndex, index) + + if creatorKey == 'class': #FIXME: remove this special-case behavior + print("creating class " + str(instanceId)) + + # # assign relationship `Instance.for Class` + # self.manager.assign_relationship(instanceId, Guid.parse('494D5A6D04BE477B8763E3F57D0DD8C8'), Guid.parse('B9C9B9B7AD8A4CBDAA6BE05784630B6B')) + + # # assign relationship `Class.has Instance` + # self.manager.assign_relationship(Guid.parse('B9C9B9B7AD8A4CBDAA6BE05784630B6B'), Guid.parse('7EB41D3C2AE9488483A4E59441BCAEFB'), instanceId) + + if "instances" in elem: + classInsts = elem["instances"] + for inst in classInsts: + self.load_instance(inst, elem) + + else: + print("creating instance " + str(instanceId) + " (of class " + str(classId) + ")") + + # # assign relationship `Instance.for Class` + # self.manager.assign_relationship(instanceId, Guid.parse('494D5A6D04BE477B8763E3F57D0DD8C8'), classId) + + # # assign relationship `Class.has Instance` + # self.manager.assign_relationship(classId, Guid.parse('7EB41D3C2AE9488483A4E59441BCAEFB'), instanceId) + + # self.manager.create_instance_of(classId, instanceId, classIndex, index) + + if templateKey is not None: + if template is not None: + print("using template (" + templateKey + ")") + #print(template) + + self.load_instance_with_template(elem, template) + else: + #print("ignoring '" + templateKey + "'") + pass + + def load_instances_from_file(self, filename): + print("loading instances from " + filename) + + import yaml + with open(filename, 'r') as file: + content = yaml.safe_load_all(file) + for doc in content: + for elem in doc: + if "entityDefinitions" in elem: + continue + elif "library" in elem: + self.load_library(filename, elem) + elif "tenant" in elem: + self.load_tenant(filename, elem) + else: + raise MochaSyntaxError("neither 'library' nor 'tenant' top level element found") + + def load_library(self, filename, elem): + if "instances" in elem: + instances = elem["instances"] + for elem1 in instances: + if "instance" in elem1: + # this is an instance definition (no sugar), so just load it + self.load_instance(elem1) + else: + # this is a syntactic sugar definition, so store it for later use + elem1["from_file_name"] = filename + self._instances_awaiting_sugar.append(elem1) + + def load_tenant(self, filename, elem): + pass \ No newline at end of file diff --git a/mocha-common/compilers/yaml2mcl/mocha/library/parser/__init__.py b/mocha-common/compilers/yaml2mcl/mocha/library/parser/__init__.py new file mode 100644 index 0000000..76cde50 --- /dev/null +++ b/mocha-common/compilers/yaml2mcl/mocha/library/parser/__init__.py @@ -0,0 +1,4 @@ +from .LibraryParser import LibraryParser +from .JSONLibraryParser import JSONLibraryParser +from .XMLLibraryParser import XMLLibraryParser +from .YAMLLibraryParser import YAMLLibraryParser \ No newline at end of file diff --git a/mocha-common/yaml2mcl b/mocha-common/yaml2mcl new file mode 100755 index 0000000..0055a17 --- /dev/null +++ b/mocha-common/yaml2mcl @@ -0,0 +1,4 @@ +#!/bin/bash + +python3 compilers/yaml2mcl/Program.py ${1+"$@"} +