396 lines
14 KiB
Python
396 lines
14 KiB
Python
import MySQLdb
|
|
|
|
from .libraryparser import XMLLibraryParser, JSONLibraryParser, YAMLLibraryParser
|
|
|
|
from .LibraryOperation import LibraryOperation
|
|
from .operations.AssignAttributeOperation import AssignAttributeOperation
|
|
from .operations.AssignRelationshipOperation import AssignRelationshipOperation
|
|
from .operations.PrepareInstanceOperation import PrepareInstanceOperation
|
|
from .operations.StoredProcedureOperation import StoredProcedureOperation
|
|
|
|
from .Normalization import Normalization
|
|
|
|
from .Guid import Guid
|
|
from .InstanceCache import InstanceCache
|
|
from .SQLExpression import SQLExpression
|
|
|
|
from redis import Redis
|
|
|
|
class MochaLibraryManager:
|
|
|
|
def __init__(self):
|
|
self.entityReferences = dict()
|
|
self.db = None
|
|
self._instances = []
|
|
self._attOps = []
|
|
self._relOps = []
|
|
self.redis = None
|
|
|
|
def connect(self, hostname, database, username, password):
|
|
self.db = MySQLdb.connect(host=hostname, user=username, passwd=password, db=database)
|
|
self.redis = Redis(host=hostname, port=6379, decode_responses=True)
|
|
|
|
def process_instance_ops(self):
|
|
# this version is much faster as it prepares everything in advanced on the CPU before
|
|
# pushing it out to the database
|
|
|
|
instances = InstanceCache()
|
|
|
|
for key in self._instances:
|
|
(class_gid, inst_gid, class_index, inst_index) = key
|
|
# first import all the known classes
|
|
if class_gid is not None and class_index is not None:
|
|
# print ("known class import")
|
|
# print ("class_gid = " + class_gid)
|
|
# print ("inst_gid = " + inst_gid + " (" + str(class_index) + "$" + str(inst_index) + ")")
|
|
if not instances.has_instance_by_global_identifier(class_gid):
|
|
instances.add_instance((1, class_index), class_gid)
|
|
|
|
for key in self._instances:
|
|
(class_gid, inst_gid, class_index, inst_index) = key
|
|
# now get all the classes without specified IIDs
|
|
if class_gid is not None and class_index is None:
|
|
# print ("unknown class import")
|
|
if not instances.has_instance_by_global_identifier(class_gid):
|
|
class_index = instances.get_next_instance_id(class_index)
|
|
else:
|
|
class_index = instances.get_instance_key(class_gid)[1]
|
|
|
|
if inst_index is None:
|
|
inst_index = instances.get_next_instance_id(class_index)
|
|
|
|
instances.add_instance((1, inst_index), inst_gid)
|
|
|
|
# print ("class_gid = " + class_gid)
|
|
# print ("inst_gid = " + inst_gid + " (" + str(1) + "$" + str(inst_index) + ")")
|
|
|
|
for key in self._instances:
|
|
(class_gid, inst_gid, class_index, inst_index) = key
|
|
if class_gid is not None and class_index is not None and inst_gid is not None and inst_index is not None:
|
|
# we know all there is to know about this one
|
|
pass
|
|
elif class_gid is not None and inst_gid is not None:
|
|
if class_index is None:
|
|
# we need to know the class index
|
|
if instances.has_instance_by_global_identifier(class_gid):
|
|
class_index = instances.get_instance_key(class_gid)[1]
|
|
else:
|
|
class_index = instances.get_next_instance_id(1)
|
|
|
|
if inst_index is None:
|
|
# we need to know the instance index as well
|
|
if instances.has_instance_by_global_identifier(inst_gid):
|
|
inst_index = instances.get_instance_key(inst_gid)[1]
|
|
else:
|
|
inst_index = instances.get_next_instance_id(class_index)
|
|
else:
|
|
print("error: not implemened class_gid is None or inst_gid is None")
|
|
|
|
instances.add_instance((class_index, inst_index), inst_gid)
|
|
|
|
print (str(instances.count()) + " instances preloaded")
|
|
|
|
c = 0
|
|
query = "INSERT INTO mocha_instances (tenant_id, class_id, inst_id, global_identifier) VALUES "
|
|
for inst_gid in instances.get_instance_keys():
|
|
(class_index, inst_index) = instances.get_instance_key(inst_gid)
|
|
print("preparing inst " + str(class_index) + "$" + str(inst_index) + " with guid " + inst_gid)
|
|
|
|
inst_key_str = str(class_index) + "$" + str(inst_index)
|
|
self.redis.set(inst_key_str + ".gid", Normalization.normalize_uuid(inst_gid))
|
|
self.redis.set(inst_key_str + ".attributes.count", "0")
|
|
self.redis.set(inst_key_str + ".relationships.count", "0")
|
|
self.redis.set(Normalization.normalize_uuid(inst_gid) + ".iid", inst_key_str)
|
|
self.redis.set(Normalization.normalize_uuid(inst_gid) + ".attributes.count", "0")
|
|
self.redis.set(Normalization.normalize_uuid(inst_gid) + ".relationships.count", "0")
|
|
|
|
query += "(1, " + str(class_index) + ", " + str(inst_index) + ", '" + Normalization.normalize_uuid(inst_gid) + "')"
|
|
|
|
if c < instances.count() - 1:
|
|
query += ", "
|
|
|
|
c += 1
|
|
|
|
print ("process_instance_ops: query was: ")
|
|
print (query)
|
|
self.db.query(query)
|
|
|
|
self.instances = instances
|
|
return True
|
|
|
|
def process_attribute_ops(self, ops):
|
|
query = "INSERT INTO mocha_attributes (tenant_id, src_inst_id, att_inst_id, usr_inst_id, att_effective_date, att_value) VALUES "
|
|
for op in ops:
|
|
instanceId = op.instanceId
|
|
attributeInstanceId = op.attributeInstanceId
|
|
value = op.value
|
|
|
|
if attributeInstanceId.get_value() == "":
|
|
print("skipping invalid blank attribute")
|
|
continue
|
|
|
|
print (instanceId.get_value() + " : " + attributeInstanceId.get_value() + " = '" + str(value) + "'")
|
|
|
|
# query += "(1, " \
|
|
# + "mocha_get_instance_by_global_identifier('" + Normalization.normalize_uuid(instanceId.get_value()) + "'), " \
|
|
# + "mocha_get_instance_by_global_identifier('" + Normalization.normalize_uuid(attributeInstanceId.get_value()) + "'), " \
|
|
# + "NULL, NOW(), " + SQLExpression.normalize_parm(value) + ")"
|
|
|
|
query += "(1, " \
|
|
+ str(self.instances.get_database_id_by_global_identifier(instanceId)) + ", " \
|
|
+ str(self.instances.get_database_id_by_global_identifier(attributeInstanceId)) + ", " \
|
|
+ "NULL, NOW(), " + SQLExpression.normalize_parm(value) + ")"
|
|
|
|
if ops.index(op) < len(ops) - 1:
|
|
query += ", "
|
|
|
|
# op.execute(self.db)
|
|
|
|
print ("process_attribute_ops: query was: ")
|
|
print (query)
|
|
self.db.query(query)
|
|
|
|
def process_relationship_ops(self, ops):
|
|
query = "INSERT INTO mocha_relationships (tenant_id, source_inst_id, relationship_inst_id, destination_inst_id, user_inst_id, effective_date) VALUES "
|
|
|
|
for op in ops:
|
|
instanceId = op.instanceId
|
|
relationshipInstanceId = op.relationshipInstanceId
|
|
targetInstanceId = op.targetInstanceId
|
|
|
|
print (instanceId.get_value() + " : " + relationshipInstanceId.get_value() + " = " + targetInstanceId.get_value())
|
|
|
|
query += "(1, " \
|
|
+ str(self.instances.get_database_id_by_global_identifier(instanceId)) + ", " \
|
|
+ str(self.instances.get_database_id_by_global_identifier(relationshipInstanceId)) + ", " \
|
|
+ str(self.instances.get_database_id_by_global_identifier(targetInstanceId)) + ", " \
|
|
+ "NULL, NOW() )"
|
|
|
|
if ops.index(op) < len(ops) - 1:
|
|
query += ", "
|
|
|
|
# op.execute(self.db)
|
|
|
|
print ("process_relationship_ops: query was: ")
|
|
print (query)
|
|
self.db.query(query)
|
|
|
|
def commit(self):
|
|
|
|
# first record all the instances
|
|
if not self.process_instance_ops():
|
|
return False
|
|
|
|
print("reading database ids...")
|
|
query = "SELECT id, global_identifier FROM mocha_instances"
|
|
|
|
cur = self.db.cursor()
|
|
cur.execute(query)
|
|
rows = cur.fetchall()
|
|
|
|
for (dbid, gid) in rows:
|
|
self.instances.set_database_id_by_global_identifier(Guid(gid), dbid)
|
|
|
|
print("assigning attributes...")
|
|
self.process_attribute_ops(self._attOps)
|
|
|
|
print("assigning relationships...")
|
|
self.process_relationship_ops(self._relOps)
|
|
|
|
hasSiblingRelationships = dict()
|
|
for op in self._relOps:
|
|
relationshipKey = self.instances
|
|
|
|
if op.relationshipInstanceId.get_value() == "{656110FF-4502-48B8-A7F3-D07F017AEA3F}":
|
|
# Relationship.has sibling Relationship
|
|
hasSiblingRelationships[op.instanceId.get_value()] = op.targetInstanceId
|
|
|
|
siblingOps = []
|
|
for op in self._relOps:
|
|
if op.relationshipInstanceId.get_value() in hasSiblingRelationships:
|
|
siblingOp = AssignRelationshipOperation(op.targetInstanceId, hasSiblingRelationships[op.relationshipInstanceId.get_value()], op.instanceId)
|
|
siblingOps.append(siblingOp)
|
|
# print ("assigning sibling relationship " + siblingOp.instanceId.get_value() + " . " + siblingOp.relationshipInstanceId.get_value() + " = " + siblingOp.instanceId.get_value())
|
|
# siblingOp.execute(self.db)
|
|
|
|
print("assigning sibling relationships...")
|
|
self.process_relationship_ops(siblingOps)
|
|
|
|
cur = self.db.cursor()
|
|
# cur.execute("UPDATE mocha_instances SET user_inst_id = mocha_get_instance_by_global_identifier(mocha_normalize_uuid('{B066A54B-B160-4510-A805-436D3F90C2E6}'))")
|
|
cur.execute("UPDATE mocha_attributes SET usr_inst_id = mocha_get_instance_by_global_identifier('" + Normalization.normalize_uuid("{B066A54B-B160-4510-A805-436D3F90C2E6}") + "')")
|
|
cur.execute("UPDATE mocha_relationships SET user_inst_id = mocha_get_instance_by_global_identifier('" + Normalization.normalize_uuid("{B066A54B-B160-4510-A805-436D3F90C2E6}") + "')")
|
|
# rows = cur.fetchall()
|
|
# print(rows)
|
|
|
|
# print ("ok , applying `Class.has Instance`...")
|
|
|
|
# for row in rows:
|
|
# print(row)
|
|
# id = row[0]
|
|
# tenant_id = row[1]
|
|
# class_id = row[2]
|
|
# inst_id = row[3]
|
|
# global_identifier = row[4]
|
|
|
|
# if class_id is not None and id is not None:
|
|
|
|
# # Class.has Instance
|
|
# query2 = ("CALL mocha_assign_relationship (" +
|
|
# "mocha_get_instance_by_key(1, " + str(class_id) + "), " +
|
|
# "mocha_get_instance_by_global_identifier(mocha_normalize_uuid('7EB41D3C-2AE9-4884-83A4-E59441BCAEFB'))" + ", " +
|
|
# str(id) + ", NULL, NULL)")
|
|
# cur.execute(query2)
|
|
|
|
# # Instance.for Class
|
|
# query3 = ("CALL mocha_assign_relationship (" +
|
|
# str(id) + ", " +
|
|
# "mocha_get_instance_by_global_identifier(mocha_normalize_uuid('494D5A6D-04BE-477B-8763-E3F57D0DD8C8'))" + ", " +
|
|
# "mocha_get_instance_by_key(1, " + str(class_id) + "), NULL, NULL)")
|
|
# cur.execute(query3)
|
|
|
|
self.db.commit()
|
|
|
|
def close(self):
|
|
self.db.close()
|
|
|
|
def print_error(self, cur):
|
|
rows = cur.fetchall()
|
|
for row in rows:
|
|
if 'error_description' in row:
|
|
print (row['error_description'])
|
|
|
|
def select_tenant(self, tenantName):
|
|
cur = self.db.cursor()
|
|
cur.execute("CALL mocha_select_tenant(mocha_get_tenant_by_name('" + tenantName + "'))")
|
|
self.print_error(cur)
|
|
|
|
def release_tenant(self):
|
|
cur = self.db.cursor()
|
|
cur.execute("CALL mocha_release_tenant()")
|
|
self.print_error(cur)
|
|
|
|
def assign_relationship(self, instanceId : Guid, relationshipInstanceId : Guid, targetInstanceId : Guid):
|
|
print("-- assigning relationship " + relationshipInstanceId.get_value() + " = '" + targetInstanceId.get_value() + "'")
|
|
self._relOps.append(AssignRelationshipOperation(instanceId, relationshipInstanceId, targetInstanceId))
|
|
# self._relOps.append(AssignRelationshipOperation(instanceId, relationshipInstanceId, targetInstanceId))
|
|
# query = "CALL mocha_assign_relationship(mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + instanceId.get_value() + "')), mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + relationshipInstanceId.get_value() + "')), mocha_get_instance_by_global_identifier(mocha_normalize_uuid('" + targetInstanceId.get_value() + "')), NULL, NULL);"
|
|
# print(query)
|
|
# cur.execute(query)
|
|
# self.print_error(cur)
|
|
|
|
def set_attribute_value(self, instanceId : Guid, attributeInstanceId : Guid, value):
|
|
print("-- assigning attribute " + attributeInstanceId.get_value() + " = '" + str(value) + "'")
|
|
self._attOps.append(AssignAttributeOperation(instanceId, attributeInstanceId, value))
|
|
|
|
def install_from_path(self, path):
|
|
|
|
from glob import glob
|
|
|
|
xl = XMLLibraryParser(self)
|
|
jl = JSONLibraryParser(self)
|
|
yl = YAMLLibraryParser(self)
|
|
|
|
#xml_files = glob(path + "/**/*.xml", recursive=True)
|
|
#for xml_file in xml_files:
|
|
#s xl.load_file(xml_file)
|
|
|
|
|
|
#json_files = glob(path + "/**/*.json", recursive=True)
|
|
#for json_file in json_files:
|
|
# jl.load_file(json_file)
|
|
|
|
yaml_files = sorted(glob(path + "/**/*.yaml", recursive=True))
|
|
if len(yaml_files) == 0:
|
|
print ("no files found ; does the path exist?")
|
|
return 3
|
|
|
|
# first, load the entity defs
|
|
for yaml_file in yaml_files:
|
|
yl.load_entity_definitions_from_file(yaml_file)
|
|
|
|
try:
|
|
# then, load instance definitions (also loads sugar elements into memory for later use)
|
|
for yaml_file in yaml_files:
|
|
yl.load_instances_from_file(yaml_file)
|
|
|
|
# finally, apply syntactic sugar
|
|
yl.apply_sugar()
|
|
|
|
return True
|
|
|
|
except NameError as ex:
|
|
|
|
print (ex)
|
|
|
|
rgx = "&(.*);"
|
|
|
|
# go through and get all entity references across all files
|
|
import re
|
|
import fileinput
|
|
|
|
stuff = []
|
|
|
|
for yaml_file in yaml_files:
|
|
f = open(yaml_file, "r")
|
|
text = f.read()
|
|
matches = re.findall(rgx, text)
|
|
for match in matches:
|
|
stuff.append(match)
|
|
f.close()
|
|
|
|
missingEntities = []
|
|
for stuf in stuff:
|
|
if not stuf in self.entityReferences:
|
|
if not stuf in missingEntities:
|
|
missingEntities.append(stuf)
|
|
|
|
if len(missingEntities) > 0:
|
|
print("\nNOTE: there were undefined referenced entities:\n")
|
|
for missingEntity in missingEntities:
|
|
print("\t" + missingEntity)
|
|
print("\n")
|
|
|
|
return False
|
|
|
|
|
|
def exists_entity_reference(self, name):
|
|
return name in self.entityReferences
|
|
|
|
def register_entity_reference(self, name, value):
|
|
self.entityReferences[name] = value
|
|
|
|
def define_entity_reference(self, name):
|
|
return self.entityReferences[name]
|
|
def expand_entity_references(self, value):
|
|
insideName = False
|
|
name = ""
|
|
retval = ""
|
|
for i in range(0, len(value)):
|
|
if value[i] == "&":
|
|
insideName = True
|
|
elif value[i] == ';':
|
|
insideName = False
|
|
|
|
if name in self.entityReferences:
|
|
retval += self.define_entity_reference(name)
|
|
else:
|
|
raise NameError("unknown entity ref '" + name + "'")
|
|
|
|
name = ""
|
|
elif insideName:
|
|
name += value[i]
|
|
else:
|
|
retval += value[i]
|
|
return retval
|
|
|
|
def add_instance(self, classGid : Guid, instGid : Guid, classIndex : int, index : int):
|
|
print("adding instance for class '" + classGid.get_value() + "' with globalid '" + instGid.get_value() + "' [" + str(index) + "]")
|
|
self._instances.append((classGid.get_value(), instGid.get_value(), classIndex, index))
|
|
|
|
# assign relationship `Instance.for Class`
|
|
self.assign_relationship(instGid, Guid('494D5A6D04BE477B8763E3F57D0DD8C8'), classGid)
|
|
|
|
# assign relationship `Class.has Instance`
|
|
self.assign_relationship(classGid, Guid('7EB41D3C2AE9488483A4E59441BCAEFB'), instGid)
|
|
|