diff --git a/.travis.yml b/.travis.yml index 76ad4e334..89615636d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,7 @@ install: # - cd $TRAVIS_BUILD_DIR # $TRAVIS_BUILD_DIR is set to the location of the cloned repository: # for example: /home/travis/build/gramps-project/gramps + - git clone -b master https://github.com/srossross/meta - python setup.py build before_script: @@ -45,7 +46,7 @@ before_script: script: # --exclude=TestUser because of older version of mock # without configure_mock - - GRAMPS_RESOURCES=. nosetests3 --nologcapture --with-coverage --cover-package=gramps --exclude=TestcaseGenerator --exclude=vcard --exclude=merge_ref_test --exclude=user_test gramps + - PYTHONPATH=meta GRAMPS_RESOURCES=. nosetests3 --nologcapture --with-coverage --cover-package=gramps --exclude=TestcaseGenerator --exclude=vcard --exclude=merge_ref_test --exclude=user_test gramps after_success: - codecov diff --git a/gramps/gen/db/base.py b/gramps/gen/db/base.py index ebfb1967b..7240c8e1e 100644 --- a/gramps/gen/db/base.py +++ b/gramps/gen/db/base.py @@ -1287,7 +1287,7 @@ class DbReadBase(object): if compare(item, op, value): return True return False - if op == "=": + if op in ["=", "=="]: matched = v == value elif op == ">": matched = v > value @@ -1430,15 +1430,15 @@ class DbReadBase(object): name = self.get_table_func(table,"class_func").get_field_alias(name) return name.replace(".", "__") - Person = property(lambda self:QuerySet(self, "Person")) - Family = property(lambda self:QuerySet(self, "Family")) - Note = property(lambda self:QuerySet(self, "Note")) - Citation = property(lambda self:QuerySet(self, "Citation")) - Source = property(lambda self:QuerySet(self, "Source")) - Repository = property(lambda self:QuerySet(self, "Repository")) - Place = property(lambda self:QuerySet(self, "Place")) - Event = property(lambda self:QuerySet(self, "Event")) - Tag = property(lambda self:QuerySet(self, "Tag")) + Person = property(lambda self: QuerySet(self, "Person")) + Family = property(lambda self: QuerySet(self, "Family")) + Note = property(lambda self: QuerySet(self, "Note")) + Citation = property(lambda self: QuerySet(self, "Citation")) + Source = property(lambda self: QuerySet(self, "Source")) + Repository = property(lambda self: QuerySet(self, "Repository")) + Place = property(lambda self: QuerySet(self, "Place")) + Event = property(lambda self: QuerySet(self, "Event")) + Tag = property(lambda self: QuerySet(self, "Tag")) class DbWriteBase(DbReadBase): """ @@ -2089,41 +2089,6 @@ class DbWriteBase(DbReadBase): """ return getattr(self, table_name) -class Operator(object): - """ - Base for QuerySet operators. - """ - op = "OP" - def __init__(self, *expressions, **kwargs): - if self.op in ["AND", "OR"]: - exprs = [expression.list for expression - in expressions] - for key in kwargs: - exprs.append( - _select_field_operator_value(key, "=", kwargs[key])) - else: # "NOT" - if expressions: - exprs = expressions.list - else: - key, value = list(kwargs.items())[0] - exprs = _select_field_operator_value(key, "=", value) - self.list = [self.op, exprs] - -class AND(Operator): - op = "AND" - -class OR(Operator): - """ - OR operator for QuerySet logical WHERE expressions. - """ - op = "OR" - -class NOT(Operator): - """ - NOT operator for QuerySet logical WHERE expressions. - """ - op = "NOT" - class QuerySet(object): """ A container for selection criteria before being actually @@ -2164,20 +2129,15 @@ class QuerySet(object): self.needs_to_run = True return self - def _add_where_clause(self, *args, **kwargs): + def _add_where_clause(self, *args): """ Add a condition to the where clause. """ # First, handle AND, OR, NOT args: and_expr = [] - for arg in args: - expr = arg.list + for expr in args: and_expr.append(expr) # Next, handle kwargs: - for keyword in kwargs: - and_expr.append( - _select_field_operator_value( - keyword, "=", kwargs[keyword])) if and_expr: if self.where_by: self.where_by = ["AND", [self.where_by] + and_expr] @@ -2260,20 +2220,32 @@ class QuerySet(object): self.database = proxy_class(self.database, *args, **kwargs) return self - def filter(self, *args, **kwargs): + def where(self, where_clause): + """ + Apply a where_clause (closure) to the selection process. + """ + from gramps.gen.db.where import eval_where + # if there is already a generator, then error: + if self.generator: + raise Exception("Queries in invalid order") + where_by = eval_where(where_clause) + self._add_where_clause(where_by) + return self + + def filter(self, *args): """ Apply a filter to the database. """ from gramps.gen.proxy import FilterProxyDb from gramps.gen.filters import GenericFilter + from gramps.gen.db.where import eval_where for i in range(len(args)): arg = args[i] if isinstance(arg, GenericFilter): self.database = FilterProxyDb(self.database, arg, *args[i+1:]) - if arg.where_by: - self._add_where_clause(arg.where_by) - elif isinstance(arg, Operator): - self._add_where_clause(arg) + if hasattr(arg, "where"): + where_by = eval_where(arg.where) + self._add_where_clause(where_by) elif callable(arg): if self.generator and self.needs_to_run: ## error @@ -2285,8 +2257,6 @@ class QuerySet(object): self.generator = filter(arg, self.generator) else: pass # ignore, may have been arg from previous Filter - if kwargs: - self._add_where_clause(**kwargs) return self def map(self, f): @@ -2329,33 +2299,3 @@ class QuerySet(object): item.add_tag(tag.handle) commit_func(item, trans) -def _to_dot_format(field): - """ - Convert a field keyword arg into a proper - dotted field name. - """ - return field.replace("__", ".") - -def _select_field_operator_value(field, op, value): - """ - Convert a field keyword arg into proper - field, op, and value. - """ - alias = { - "LT": "<", - "GT": ">", - "LTE": "<=", - "GTE": ">=", - "IS_NOT": "IS NOT", - "IS_NULL": "IS NULL", - "IS_NOT_NULL": "IS NOT NULL", - "NE": "<>", - } - for operator in ["LIKE", "IN"] + list(alias.keys()): - operator = "__" + operator - if field.endswith(operator): - op = field[-len(operator) + 2:] - field = field[:-len(operator)] - op = alias.get(op, op) - field = _to_dot_format(field) - return (field, op, value) diff --git a/gramps/gen/db/test/test_where.py b/gramps/gen/db/test/test_where.py new file mode 100644 index 000000000..c23b4ee63 --- /dev/null +++ b/gramps/gen/db/test/test_where.py @@ -0,0 +1,102 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2016 Gramps Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# + +from gramps.gen.db.where import eval_where +import unittest + +########## +# Tests: + +def make_closure(surname): + """ + Test closure. + """ + from gramps.gen.lib import Person + return (lambda person: + (person.primary_name.surname_list[0].surname == surname and + person.gender == Person.MALE)) + +class Thing(object): + def __init__(self): + self.list = ["I0", "I1", "I2"] + + def where(self): + return lambda person: person.gramps_id == self.list[1] + +class ClosureTest(unittest.TestCase): + def check(self, test): + result = eval_where(test[0]) + self.assertTrue(result == test[1], "%s is not %s" % (result, test[1])) + + def test_01(self): + self.check( + (lambda family: (family.private and + family.mother_handle.gramps_id != "I0001"), + ['AND', [['private', '==', True], + ['mother_handle.gramps_id', '!=', 'I0001']]])) + + def test_02(self): + self.check( + (lambda person: LIKE(person.gramps_id, "I0001"), + ['gramps_id', 'LIKE', 'I0001'])) + + def test_03(self): + self.check( + (lambda note: note.gramps_id == "N0001", + ['gramps_id', '==', 'N0001'])) + + def test_04(self): + self.check( + (lambda person: person.event_ref_list.ref.gramps_id == "E0001", + ['event_ref_list.ref.gramps_id', '==', 'E0001'])) + + def test_05(self): + self.check( + (lambda person: LIKE(person.gramps_id, "I0001") or person.private, + ["OR", [['gramps_id', 'LIKE', 'I0001'], + ["private", "==", True]]])) + + def test_06(self): + self.check( + (lambda person: person.event_ref_list <= 0, + ["event_ref_list", "<=", 0])) + + def test_07(self): + self.check( + (lambda person: person.primary_name.surname_list[0].surname == "Smith", + ["primary_name.surname_list.0.surname", "==", "Smith"])) + + def test_08(self): + self.check( + (make_closure("Smith"), + ["AND", [["primary_name.surname_list.0.surname", "==", "Smith"], + ["gender", "==", 1]]])) + + def test_09(self): + self.check( + [Thing().where(), ["gramps_id", "==", "I1"]]) + + def test_10(self): + self.check( + (lambda person: LIKE(person.gramps_id, "I000%"), + ["gramps_id", "LIKE", "I000%"])) + +if __name__ == "__main__": + unittest.main() diff --git a/gramps/gen/db/where.py b/gramps/gen/db/where.py new file mode 100644 index 000000000..e9641143b --- /dev/null +++ b/gramps/gen/db/where.py @@ -0,0 +1,151 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2016 Gramps Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# + +from meta.asttools import Visitor +from meta.decompiler import _ast, decompile_func + +import copy + +class ParseFilter(Visitor): + def visitName(self, node): + return node.id + + def visitNum(self, node): + return node.n + + def visitlong(self, node): + return node + + def process_expression(self, expr): + if isinstance(expr, str): + # boolean + return [self.process_field(expr), "==", True] + elif len(expr) == 3: + # (field, op, value) + return [self.process_field(expr[0]), + expr[1], + self.process_value(expr[2])] + else: + # list of exprs + return [self.process_expression(exp) for + exp in expr] + + def process_value(self, value): + try: + return eval(value, self.env) + except: + return value + + def process_field(self, field): + field = field.replace("[", ".").replace("]", "") + if field.startswith(self.parameter + "."): + return field[len(self.parameter) + 1:] + else: + return field + + def visitCall(self, node): + """ + Handle LIKE() + """ + return [self.process_field(self.visit(node.args[0])), + self.visit(node.func), + self.process_value(self.visit(node.args[1]))] + + def visitStr(self, node): + return node.s + + def visitlist(self, list): + return [self.visit(node) for node in list] + + def visitCompare(self, node): + return [self.process_field(self.visit(node.left)), + " ".join(self.visit(node.ops)), + self.process_value(self.visit(node.comparators[0]))] + + def visitAttribute(self, node): + return "%s.%s" % (self.visit(node.value), node.attr) + + def get_boolean_op(self, node): + if isinstance(node, _ast.And): + return "AND" + elif isinstance(node, _ast.Or): + return "OR" + else: + raise Exception("invalid boolean") + + def visitNotEq(self, node): + return "!=" + + def visitLtE(self, node): + return "<=" + + def visitGtE(self, node): + return ">=" + + def visitEq(self, node): + return "==" + + def visitBoolOp(self, node): + """ + BoolOp: boolean operator + """ + op = self.get_boolean_op(node.op) + values = list(node.values) + return [op, self.process_expression( + [self.visit(value) for value in values])] + + def visitLambda(self, node): + self.parameter = self.visit(node.args)[0] + return self.visit(node.body) + + def visitarguments(self, node): + return [self.visit(arg) for arg in node.args] + + def visitarg(self, node): + return node.arg + + def visitSubscript(self, node): + return "%s[%s]" % (self.visit(node.value), + self.visit(node.slice)) + + def visitIndex(self, node): + return self.visit(node.value) + +def make_env(closure): + """ + Create an environment from the closure. + """ + env = copy.copy(closure.__globals__) + if closure.__closure__: + for i in range(len(closure.__closure__)): + env[closure.__code__.co_freevars[i]] = closure.__closure__[i].cell_contents + return env + +def eval_where(closure): + """ + Given a closure, parse and evaluate it. + Return a WHERE expression. + """ + parser = ParseFilter() + parser.env = make_env(closure) + ast_top = decompile_func(closure) + result = parser.visit(ast_top) + return result + diff --git a/gramps/plugins/database/test/db_test.py b/gramps/plugins/database/test/db_test.py index 29e1faad4..95d857e49 100644 --- a/gramps/plugins/database/test/db_test.py +++ b/gramps/plugins/database/test/db_test.py @@ -123,12 +123,12 @@ class BSDDBTest(unittest.TestCase): self.assertTrue(len(result) == 60, len(result)) def test_queryset_2(self): - result = list(self.db.Person.filter(gramps_id__LIKE="I000%").select()) + result = list(self.db.Person.where(lambda person: LIKE(person.gramps_id, "I000%")).select()) self.assertTrue(len(result) == 10, len(result)) def test_queryset_3(self): result = list(self.db.Family - .filter(mother_handle__gramps_id__LIKE="I003%") + .where(lambda family: LIKE(family.mother_handle.gramps_id, "I003%")) .select()) self.assertTrue(len(result) == 6, result) @@ -138,7 +138,7 @@ class BSDDBTest(unittest.TestCase): def test_queryset_4b(self): result = list(self.db.Family - .filter(mother_handle__event_ref_list__ref__gramps_id='E0156') + .where(lambda family: family.mother_handle.event_ref_list.ref.gramps_id == 'E0156') .select()) self.assertTrue(len(result) == 1, len(result)) @@ -154,9 +154,8 @@ class BSDDBTest(unittest.TestCase): [r["mother_handle.event_ref_list.0"] for r in result]) def test_queryset_7(self): - from gramps.gen.db import NOT result = list(self.db.Family - .filter(NOT(mother_handle__event_ref_list__0=None)) + .where(lambda family: family.mother_handle.event_ref_list[0] != None) .select()) self.assertTrue(len(result) == 21, len(result)) @@ -188,22 +187,28 @@ class BSDDBTest(unittest.TestCase): self.assertTrue(result == 60, result) def test_tag_1(self): - self.db.Person.filter(gramps_id="I0001").tag("Test") - result = self.db.Person.filter(tag_list__name="Test").count() + self.db.Person.where(lambda person: person.gramps_id == "I0001").tag("Test") + result = self.db.Person.where(lambda person: person.tag_list.name == "Test").count() self.assertTrue(result == 1, result) - # def test_filter_1(self): - # from gramps.gen.filters.rules.person import (IsDescendantOf, - # IsAncestorOf) - # from gramps.gen.filters import GenericFilter - # filter = GenericFilter() - # filter.set_logical_op("or") - # filter.add_rule(IsDescendantOf([self.db.get_default_person().gramps_id, - # True])) - # filter.add_rule(IsAncestorOf([self.db.get_default_person().gramps_id, - # True])) - # result = self.db.Person.filter(filter).count() - # self.assertTrue(result == 15, result) + def test_filter_1(self): + from gramps.gen.filters.rules.person import (IsDescendantOf, + IsAncestorOf) + from gramps.gen.filters import GenericFilter + filter = GenericFilter() + filter.set_logical_op("or") + filter.add_rule(IsDescendantOf([self.db.get_default_person().gramps_id, + True])) + filter.add_rule(IsAncestorOf([self.db.get_default_person().gramps_id, + True])) + result = self.db.Person.filter(filter).count() + self.assertTrue(result == 15, result) + filter.where = lambda person: person.private == True + result = self.db.Person.filter(filter).count() + self.assertTrue(result == 1, result) + filter.where = lambda person: person.private != True + result = self.db.Person.filter(filter).count() + self.assertTrue(result == 14, result) def test_filter_2(self): result = self.db.Person.filter(lambda p: p.private).count()