import MySQLdb from .libraryparser import XMLLibraryParser, JSONLibraryParser, YAMLLibraryParser from .LibraryOperation import LibraryOperation from .operations.AssignAttributeOperation import AssignAttributeOperation from .operations.AssignRelationshipOperation import AssignRelationshipOperation from .operations.PrepareInstanceOperation import PrepareInstanceOperation from .operations.StoredProcedureOperation import StoredProcedureOperation from .Normalization import Normalization from .core.Guid import Guid from .core.KnownClassGuids import IDC_EntityDefinition, IDC_SourceDefinition from .core.KnownAttributeGuids import * from .core.KnownRelationshipGuids import * from .InstanceCache import InstanceCache from .SQLExpression import SQLExpression from redis import Redis class MochaLibraryManager: def __init__(self): self.entityReferences = dict() self.entityReferenceFileNames = dict() self.db = None self._instances = [] self._attOps = [] self._relOps = [] self.redis = None def connect(self, hostname, database, username, password): self.db = MySQLdb.connect(host=hostname, user=username, passwd=password, db=database) self.redis = Redis(host=hostname, port=6379, decode_responses=True) def process_instance_ops(self): # this version is much faster as it prepares everything in advanced on the CPU before # pushing it out to the database instances = InstanceCache() for key in self._instances: (class_gid, inst_gid, class_index, inst_index) = key # first import all the known classes if class_gid is not None and class_index is not None: # print ("known class import") # print ("class_gid = " + class_gid) # print ("inst_gid = " + inst_gid + " (" + str(class_index) + "$" + str(inst_index) + ")") if not instances.has_instance_by_global_identifier(class_gid): instances.add_instance((1, class_index), class_gid) for key in self._instances: (class_gid, inst_gid, class_index, inst_index) = key # now get all the classes without specified IIDs if class_gid is not None and class_index is None: # print ("unknown class import") if not instances.has_instance_by_global_identifier(class_gid): class_index = instances.get_next_instance_id(class_index) else: class_index = instances.get_instance_key(class_gid)[1] if inst_index is None: inst_index = instances.get_next_instance_id(class_index) instances.add_instance((1, inst_index), inst_gid) # print ("class_gid = " + class_gid) # print ("inst_gid = " + inst_gid + " (" + str(1) + "$" + str(inst_index) + ")") for key in self._instances: (class_gid, inst_gid, class_index, inst_index) = key if class_gid is not None and class_index is not None and inst_gid is not None and inst_index is not None: # we know all there is to know about this one pass elif class_gid is not None and inst_gid is not None: if class_index is None: # we need to know the class index if instances.has_instance_by_global_identifier(class_gid): class_index = instances.get_instance_key(class_gid)[1] else: class_index = instances.get_next_instance_id(1) if inst_index is None: # we need to know the instance index as well if instances.has_instance_by_global_identifier(inst_gid): inst_index = instances.get_instance_key(inst_gid)[1] else: inst_index = instances.get_next_instance_id(class_index) else: print("error: not implemened class_gid is None or inst_gid is None") instances.add_instance((class_index, inst_index), inst_gid) print (str(instances.count()) + " instances preloaded") c = 0 query = "INSERT INTO mocha_instances (tenant_id, class_id, inst_id, global_identifier) VALUES " for inst_gid in instances.get_instance_keys(): (class_index, inst_index) = instances.get_instance_key(inst_gid) print("preparing inst " + str(class_index) + "$" + str(inst_index) + " with guid " + inst_gid) inst_key_str = str(class_index) + "$" + str(inst_index) self.redis.set(inst_key_str + ".gid", Normalization.normalize_uuid(inst_gid)) self.redis.set(inst_key_str + ".attributes.count", "0") self.redis.set(inst_key_str + ".relationships.count", "0") self.redis.set(Normalization.normalize_uuid(inst_gid) + ".iid", inst_key_str) self.redis.set(Normalization.normalize_uuid(inst_gid) + ".attributes.count", "0") self.redis.set(Normalization.normalize_uuid(inst_gid) + ".relationships.count", "0") query += "(1, " + str(class_index) + ", " + str(inst_index) + ", '" + Normalization.normalize_uuid(inst_gid) + "')" if c < instances.count() - 1: query += ", " c += 1 print ("process_instance_ops: query was: ") print (query) self.db.query(query) self.instances = instances return True def process_attribute_ops(self, ops): query = "INSERT INTO mocha_attributes (tenant_id, src_inst_id, att_inst_id, usr_inst_id, att_effective_date, att_value) VALUES " for op in ops: instanceId = op.instanceId attributeInstanceId = op.attributeInstanceId value = op.value if attributeInstanceId.get_value() == "": print("skipping invalid blank attribute") continue print (instanceId.get_value() + " : " + attributeInstanceId.get_value() + " = '" + str(value) + "'") # query += "(1, " \ # + "mocha_get_instance_by_global_identifier('" + Normalization.normalize_uuid(instanceId.get_value()) + "'), " \ # + "mocha_get_instance_by_global_identifier('" + Normalization.normalize_uuid(attributeInstanceId.get_value()) + "'), " \ # + "NULL, NOW(), " + SQLExpression.normalize_parm(value) + ")" query += "(1, " \ + str(self.instances.get_database_id_by_global_identifier(instanceId)) + ", " \ + str(self.instances.get_database_id_by_global_identifier(attributeInstanceId)) + ", " \ + "NULL, NOW(), " + SQLExpression.normalize_parm(value) + ")" if ops.index(op) < len(ops) - 1: query += ", " # op.execute(self.db) print ("process_attribute_ops: query was: ") print (query) self.db.query(query) def process_relationship_ops(self, ops): query = "INSERT INTO mocha_relationships (tenant_id, source_inst_id, relationship_inst_id, destination_inst_id, user_inst_id, effective_date) VALUES " for op in ops: instanceId = op.instanceId relationshipInstanceId = op.relationshipInstanceId targetInstanceId = op.targetInstanceId print (instanceId.get_value() + " : " + relationshipInstanceId.get_value() + " = " + targetInstanceId.get_value()) query += "(1, " \ + str(self.instances.get_database_id_by_global_identifier(instanceId)) + ", " \ + str(self.instances.get_database_id_by_global_identifier(relationshipInstanceId)) + ", " \ + str(self.instances.get_database_id_by_global_identifier(targetInstanceId)) + ", " \ + "NULL, NOW() )" if ops.index(op) < len(ops) - 1: query += ", " # op.execute(self.db) print ("process_relationship_ops: query was: ") print (query) self.db.query(query) def before_commit(self): """ Called before the unstructured information is sorted and written to the database. --- This is your last chance to add instances, set attributes, and assign relationships before the compilation is finalized. After this stage, calls made to modify the compilation are undefined and will almost certainly not do what you want. """ self.write_debug_information() def write_debug_information(self): """ Writes debugging information, such as entity definitions ('&IDC_...;') and source code definitions (filename, line number, column number, etc.) """ print ("preparing debug information (use --no-debug or undefine DEBUG or etc. [NOT IMPLEMENTED] to turn off)") for key in self.entityReferences: val = self.entityReferences[key] gidEntityDefinition = Guid.create() self.add_instance(IDC_EntityDefinition, gidEntityDefinition, None, None) self.set_attribute_value(gidEntityDefinition, IDA_Name, key) self.set_attribute_value(gidEntityDefinition, IDA_Value, val) gidSourceDefinition = Guid.create() self.add_instance(IDC_SourceDefinition, gidSourceDefinition, None, None) self.set_attribute_value(gidSourceDefinition, IDA_DebugDefinitionFileName, self.entityReferenceFileNames[key]) self.assign_relationship(gidEntityDefinition, IDR_Instance__has__Source_Definition, gidSourceDefinition) # Source Definition.has X # self.add_instance(KnownClassGuids.CLASS, Guid("{dc0a5dd2-22e0-471f-87cf-a5ef1b764efa}"), 1, index) # self.assign_relationship(instGid, Guid("{dc0a5dd2-22e0-471f-87cf-a5ef1b764efa}"), targetInstanceId) def commit(self): self.before_commit() # first record all the instances if not self.process_instance_ops(): return False print("reading database ids...") query = "SELECT id, global_identifier FROM mocha_instances" cur = self.db.cursor() cur.execute(query) rows = cur.fetchall() for (dbid, gid) in rows: self.instances.set_database_id_by_global_identifier(Guid(gid), dbid) print("assigning attributes...") self.process_attribute_ops(self._attOps) print("assigning relationships...") self.process_relationship_ops(self._relOps) hasSiblingRelationships = dict() for op in self._relOps: relationshipKey = self.instances if op.relationshipInstanceId.get_value() == "{656110FF-4502-48B8-A7F3-D07F017AEA3F}": # Relationship.has sibling Relationship hasSiblingRelationships[op.instanceId.get_value()] = op.targetInstanceId siblingOps = [] for op in self._relOps: if op.relationshipInstanceId.get_value() in hasSiblingRelationships: siblingOp = AssignRelationshipOperation(op.targetInstanceId, hasSiblingRelationships[op.relationshipInstanceId.get_value()], op.instanceId) siblingOps.append(siblingOp) # print ("assigning sibling relationship " + siblingOp.instanceId.get_value() + " . " + siblingOp.relationshipInstanceId.get_value() + " = " + siblingOp.instanceId.get_value()) # siblingOp.execute(self.db) print("assigning sibling relationships...") self.process_relationship_ops(siblingOps) print("setting creation user...") cur = self.db.cursor() # cur.execute("UPDATE mocha_instances SET user_inst_id = mocha_get_instance_by_global_identifier(mocha_normalize_uuid('{B066A54B-B160-4510-A805-436D3F90C2E6}'))") cur.execute("UPDATE mocha_attributes SET usr_inst_id = " + str(self.instances.get_database_id_by_global_identifier(Guid("{B066A54B-B160-4510-A805-436D3F90C2E6}")))) cur.execute("UPDATE mocha_relationships SET user_inst_id = " + str(self.instances.get_database_id_by_global_identifier(Guid("{B066A54B-B160-4510-A805-436D3F90C2E6}")))) # rows = cur.fetchall() # print(rows) # print ("ok , applying `Class.has Instance`...") # for row in rows: # print(row) # id = row[0] # tenant_id = row[1] # class_id = row[2] # inst_id = row[3] # global_identifier = row[4] # if class_id is not None and id is not None: # # Class.has Instance # query2 = ("CALL mocha_assign_relationship (" + # "mocha_get_instance_by_key(1, " + str(class_id) + "), " + # "mocha_get_instance_by_global_identifier(mocha_normalize_uuid('7EB41D3C-2AE9-4884-83A4-E59441BCAEFB'))" + ", " + # str(id) + ", NULL, NULL)") # cur.execute(query2) # # Instance.for Class # query3 = ("CALL mocha_assign_relationship (" + # str(id) + ", " + # "mocha_get_instance_by_global_identifier(mocha_normalize_uuid('494D5A6D-04BE-477B-8763-E3F57D0DD8C8'))" + ", " + # "mocha_get_instance_by_key(1, " + str(class_id) + "), NULL, NULL)") # cur.execute(query3) self.db.commit() def close(self): self.db.close() def print_error(self, cur): rows = cur.fetchall() for row in rows: if 'error_description' in row: print (row['error_description']) def select_tenant(self, tenantName): cur = self.db.cursor() cur.execute("CALL mocha_select_tenant(mocha_get_tenant_by_name('" + tenantName + "'))") self.print_error(cur) def release_tenant(self): cur = self.db.cursor() cur.execute("CALL mocha_release_tenant()") self.print_error(cur) def assign_relationship(self, instanceId : Guid, relationshipInstanceId : Guid, targetInstanceId : Guid): print("-- assigning relationship " + relationshipInstanceId.get_value() + " = '" + targetInstanceId.get_value() + "'") self._relOps.append(AssignRelationshipOperation(instanceId, relationshipInstanceId, targetInstanceId)) # self._relOps.append(AssignRelationshipOperation(instanceId, relationshipInstanceId, targetInstanceId)) # query = "CALL mocha_assign_relationship(mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + instanceId.get_value() + "')), mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + relationshipInstanceId.get_value() + "')), mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + targetInstanceId.get_value() + "')), NULL, NULL);" # print(query) # cur.execute(query) # self.print_error(cur) def set_attribute_value(self, instanceId : Guid, attributeInstanceId : Guid, value): print("-- assigning attribute " + attributeInstanceId.get_value() + " = '" + str(value) + "'") self._attOps.append(AssignAttributeOperation(instanceId, attributeInstanceId, value)) def install_from_path(self, path): from glob import glob xl = XMLLibraryParser(self) jl = JSONLibraryParser(self) yl = YAMLLibraryParser(self) #xml_files = glob(path + "/**/*.xml", recursive=True) #for xml_file in xml_files: #s xl.load_file(xml_file) #json_files = glob(path + "/**/*.json", recursive=True) #for json_file in json_files: # jl.load_file(json_file) yaml_files = sorted(glob(path + "/**/*.yaml", recursive=True)) if len(yaml_files) == 0: print ("no files found ; does the path exist?") return 3 # first, load the entity defs for yaml_file in yaml_files: yl.load_entity_definitions_from_file(yaml_file) try: # then, load instance definitions (also loads sugar elements into memory for later use) for yaml_file in yaml_files: yl.load_instances_from_file(yaml_file) # finally, apply syntactic sugar yl.apply_sugar() return True except NameError as ex: print (ex) rgx = "&(.*);" # go through and get all entity references across all files import re import fileinput stuff = [] for yaml_file in yaml_files: f = open(yaml_file, "r") text = f.read() matches = re.findall(rgx, text) for match in matches: stuff.append(match) f.close() missingEntities = [] for stuf in stuff: if not stuf in self.entityReferences: if not stuf in missingEntities: missingEntities.append(stuf) if len(missingEntities) > 0: print("\nNOTE: there were undefined referenced entities:\n") for missingEntity in missingEntities: print("\t" + missingEntity) print("\n") return False def exists_entity_reference(self, name): return name in self.entityReferences def register_entity_reference(self, name, value, filename = None): self.entityReferences[name] = value self.entityReferenceFileNames[name] = filename def define_entity_reference(self, name): return self.entityReferences[name] def expand_entity_references(self, value): insideName = False name = "" retval = "" for i in range(0, len(value)): if value[i] == "&": insideName = True elif value[i] == ';': insideName = False if name in self.entityReferences: retval += self.define_entity_reference(name) else: raise NameError("unknown entity ref '" + name + "'") name = "" elif insideName: name += value[i] else: retval += value[i] return retval def add_instance(self, classGid : Guid, instGid : Guid, classIndex : int, index : int): print("adding instance for class '" + classGid.get_value() + "' with globalid '" + instGid.get_value() + "' [" + str(index) + "]") self._instances.append((classGid.get_value(), instGid.get_value(), classIndex, index)) # assign relationship `Instance.for Class` self.assign_relationship(instGid, Guid('494D5A6D04BE477B8763E3F57D0DD8C8'), classGid) # assign relationship `Class.has Instance` self.assign_relationship(classGid, Guid('7EB41D3C2AE9488483A4E59441BCAEFB'), instGid)