From 34613160679a885a17fdfce68c161551383f5c0c Mon Sep 17 00:00:00 2001 From: kulath Date: Mon, 1 Aug 2016 16:34:46 +0100 Subject: [PATCH 1/3] 0009515: DBAPI: gramps.plugins.database.dbapi_support.sqlite.Sqlite.try_execute ignores errors Check if the table 'person' exists to determine whether the database is new. --- gramps/plugins/db/dbapi/dbapi.py | 128 ++++++++++++++------------ gramps/plugins/db/dbapi/mysql.py | 11 +-- gramps/plugins/db/dbapi/postgresql.py | 13 +-- gramps/plugins/db/dbapi/sqlite.py | 11 +-- 4 files changed, 80 insertions(+), 83 deletions(-) diff --git a/gramps/plugins/db/dbapi/dbapi.py b/gramps/plugins/db/dbapi/dbapi.py index d2bdfbf7c..391f3b9fa 100644 --- a/gramps/plugins/db/dbapi/dbapi.py +++ b/gramps/plugins/db/dbapi/dbapi.py @@ -167,14 +167,18 @@ class DBAPI(DbGeneric): code = compile(file.read(), settings_file, 'exec') exec(code, globals(), settings) self.dbapi = settings["dbapi"] - self.update_schema() + + # We use the existence of the person table as a proxy for the database + # being new + if not self.dbapi.table_exists("person"): + self.update_schema() def update_schema(self): """ Create and update schema. """ # make sure schema is up to date: - self.dbapi.try_execute("""CREATE TABLE person ( + self.dbapi.execute("""CREATE TABLE person ( handle VARCHAR(50) PRIMARY KEY NOT NULL, given_name TEXT , surname TEXT , @@ -183,171 +187,171 @@ class DBAPI(DbGeneric): gramps_id TEXT , blob_data BLOB );""") - self.dbapi.try_execute("""CREATE TABLE family ( + self.dbapi.execute("""CREATE TABLE family ( handle VARCHAR(50) PRIMARY KEY NOT NULL, father_handle VARCHAR(50), mother_handle VARCHAR(50), gramps_id TEXT , blob_data BLOB );""") - self.dbapi.try_execute("""CREATE TABLE source ( + self.dbapi.execute("""CREATE TABLE source ( handle VARCHAR(50) PRIMARY KEY NOT NULL, order_by TEXT , gramps_id TEXT , blob_data BLOB );""") - self.dbapi.try_execute("""CREATE TABLE citation ( + self.dbapi.execute("""CREATE TABLE citation ( handle VARCHAR(50) PRIMARY KEY NOT NULL, order_by TEXT , gramps_id TEXT , blob_data BLOB );""") - self.dbapi.try_execute("""CREATE TABLE event ( + self.dbapi.execute("""CREATE TABLE event ( handle VARCHAR(50) PRIMARY KEY NOT NULL, gramps_id TEXT , blob_data BLOB );""") - self.dbapi.try_execute("""CREATE TABLE media ( + self.dbapi.execute("""CREATE TABLE media ( handle VARCHAR(50) PRIMARY KEY NOT NULL, order_by TEXT , gramps_id TEXT , blob_data BLOB );""") - self.dbapi.try_execute("""CREATE TABLE place ( + self.dbapi.execute("""CREATE TABLE place ( handle VARCHAR(50) PRIMARY KEY NOT NULL, order_by TEXT , gramps_id TEXT , blob_data BLOB );""") - self.dbapi.try_execute("""CREATE TABLE repository ( + self.dbapi.execute("""CREATE TABLE repository ( handle VARCHAR(50) PRIMARY KEY NOT NULL, gramps_id TEXT , blob_data BLOB );""") - self.dbapi.try_execute("""CREATE TABLE note ( + self.dbapi.execute("""CREATE TABLE note ( handle VARCHAR(50) PRIMARY KEY NOT NULL, gramps_id TEXT , blob_data BLOB );""") - self.dbapi.try_execute("""CREATE TABLE tag ( + self.dbapi.execute("""CREATE TABLE tag ( handle VARCHAR(50) PRIMARY KEY NOT NULL, order_by TEXT , blob_data BLOB );""") # Secondary: - self.dbapi.try_execute("""CREATE TABLE reference ( + self.dbapi.execute("""CREATE TABLE reference ( obj_handle VARCHAR(50), obj_class TEXT, ref_handle VARCHAR(50), ref_class TEXT );""") - self.dbapi.try_execute("""CREATE TABLE name_group ( + self.dbapi.execute("""CREATE TABLE name_group ( name VARCHAR(50) PRIMARY KEY NOT NULL, grouping TEXT );""") - self.dbapi.try_execute("""CREATE TABLE metadata ( + self.dbapi.execute("""CREATE TABLE metadata ( setting VARCHAR(50) PRIMARY KEY NOT NULL, value BLOB );""") - self.dbapi.try_execute("""CREATE TABLE gender_stats ( + self.dbapi.execute("""CREATE TABLE gender_stats ( given_name TEXT, female INTEGER, male INTEGER, unknown INTEGER );""") ## Indices: - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX person_order_by ON person(order_by); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX person_gramps_id ON person(gramps_id); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX person_surname ON person(surname); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX person_given_name ON person(given_name); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX source_order_by ON source(order_by); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX source_gramps_id ON source(gramps_id); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX citation_order_by ON citation(order_by); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX citation_gramps_id ON citation(gramps_id); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX media_order_by ON media(order_by); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX media_gramps_id ON media(gramps_id); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX place_order_by ON place(order_by); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX place_gramps_id ON place(gramps_id); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX tag_order_by ON tag(order_by); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX reference_ref_handle ON reference(ref_handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX name_group_name ON name_group(name); """) # Fixes: - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX place_handle ON place(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX citation_handle ON citation(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX media_handle ON media(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX person_handle ON person(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX family_handle ON family(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX event_handle ON event(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX repository_handle ON repository(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX tag_handle ON tag(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX note_handle ON note(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX source_handle ON source(handle); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX family_gramps_id ON family(gramps_id); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX event_gramps_id ON event(gramps_id); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX repository_gramps_id ON repository(gramps_id); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX note_gramps_id ON note(gramps_id); """) - self.dbapi.try_execute("""CREATE INDEX + self.dbapi.execute("""CREATE INDEX reference_obj_handle ON reference(obj_handle); """) self.rebuild_secondary_fields() @@ -1898,23 +1902,25 @@ class DBAPI(DbGeneric): def drop_tables(self): """ - Useful in testing, reseting. + Useful in testing, reseting. If the test is unsure whether the tables + already exist, then the caller will need to catch the appropriate + exception """ - self.dbapi.try_execute("""DROP TABLE person;""") - self.dbapi.try_execute("""DROP TABLE family;""") - self.dbapi.try_execute("""DROP TABLE source;""") - self.dbapi.try_execute("""DROP TABLE citation""") - self.dbapi.try_execute("""DROP TABLE event;""") - self.dbapi.try_execute("""DROP TABLE media;""") - self.dbapi.try_execute("""DROP TABLE place;""") - self.dbapi.try_execute("""DROP TABLE repository;""") - self.dbapi.try_execute("""DROP TABLE note;""") - self.dbapi.try_execute("""DROP TABLE tag;""") + self.dbapi.execute("""DROP TABLE person;""") + self.dbapi.execute("""DROP TABLE family;""") + self.dbapi.execute("""DROP TABLE source;""") + self.dbapi.execute("""DROP TABLE citation""") + self.dbapi.execute("""DROP TABLE event;""") + self.dbapi.execute("""DROP TABLE media;""") + self.dbapi.execute("""DROP TABLE place;""") + self.dbapi.execute("""DROP TABLE repository;""") + self.dbapi.execute("""DROP TABLE note;""") + self.dbapi.execute("""DROP TABLE tag;""") # Secondary: - self.dbapi.try_execute("""DROP TABLE reference;""") - self.dbapi.try_execute("""DROP TABLE name_group;""") - self.dbapi.try_execute("""DROP TABLE metadata;""") - self.dbapi.try_execute("""DROP TABLE gender_stats;""") + self.dbapi.execute("""DROP TABLE reference;""") + self.dbapi.execute("""DROP TABLE name_group;""") + self.dbapi.execute("""DROP TABLE metadata;""") + self.dbapi.execute("""DROP TABLE gender_stats;""") def _sql_type(self, python_type): """ @@ -2002,7 +2008,7 @@ class DBAPI(DbGeneric): for field in self.get_table_func( table, "class_func").get_index_fields(): field = self._hash_name(table, field) - self.dbapi.try_execute("CREATE INDEX %s_%s ON %s(%s);" + self.dbapi.execute("CREATE INDEX %s_%s ON %s(%s);" % (table, field, table_name, field)) def update_secondary_values_all(self): diff --git a/gramps/plugins/db/dbapi/mysql.py b/gramps/plugins/db/dbapi/mysql.py index 2b8a215e0..c25630f92 100644 --- a/gramps/plugins/db/dbapi/mysql.py +++ b/gramps/plugins/db/dbapi/mysql.py @@ -87,13 +87,10 @@ class MySQL: def rollback(self): self.connection.rollback() - def try_execute(self, sql): - query = self._hack_query(sql) - try: - self.cursor.execute(sql) - except Exception as exc: - pass - #print(str(exc)) + def table_exists(self, table): + self.cursor.execute("SELECT COUNT(*) FROM information_schema.tables " + "WHERE table_name='%s';" % table) + return self.fetchone()[0] != 0 def close(self): self.connection.close() diff --git a/gramps/plugins/db/dbapi/postgresql.py b/gramps/plugins/db/dbapi/postgresql.py index 91ef4c97e..1eedfdd3d 100644 --- a/gramps/plugins/db/dbapi/postgresql.py +++ b/gramps/plugins/db/dbapi/postgresql.py @@ -94,15 +94,10 @@ class Postgresql: def rollback(self): self.connection.rollback() - def try_execute(self, sql): - sql = self._hack_query(sql) - sql = sql.replace("BLOB", "bytea") - try: - self.cursor.execute(sql) - except Exception as exc: - self.cursor.execute("rollback") - #print("ERROR:", sql) - #print(str(exc)) + def table_exists(self, table): + self.cursor.execute("SELECT COUNT(*) FROM information_schema.tables " + "WHERE table_name='%s';" % table) + return self.fetchone()[0] != 0 def close(self): self.connection.close() diff --git a/gramps/plugins/db/dbapi/sqlite.py b/gramps/plugins/db/dbapi/sqlite.py index 6e0d19317..e4d1bd8c5 100644 --- a/gramps/plugins/db/dbapi/sqlite.py +++ b/gramps/plugins/db/dbapi/sqlite.py @@ -60,6 +60,7 @@ class Sqlite: return self.cursor.fetchall() def begin(self): + self.log.debug("BEGIN TRANSACTION;") self.execute("BEGIN TRANSACTION;") def commit(self): @@ -70,12 +71,10 @@ class Sqlite: self.log.debug("ROLLBACK;") self.connection.rollback() - def try_execute(self, sql): - try: - self.cursor.execute(sql) - except Exception as exc: - #print(str(exc)) - pass + def table_exists(self, table): + self.execute("SELECT COUNT(*) FROM sqlite_master " + "WHERE type='table' AND name='%s';" % table) + return self.fetchone()[0] != 0 def close(self): self.log.debug("closing database...") From fcf1c1d13b08b7c59973cd974627745fb9d263a5 Mon Sep 17 00:00:00 2001 From: kulath Date: Mon, 1 Aug 2016 17:37:26 +0100 Subject: [PATCH 2/3] Improve pylint to 10.0 --- gramps/plugins/db/dbapi/sqlite.py | 83 +++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/gramps/plugins/db/dbapi/sqlite.py b/gramps/plugins/db/dbapi/sqlite.py index e4d1bd8c5..dac1ae920 100644 --- a/gramps/plugins/db/dbapi/sqlite.py +++ b/gramps/plugins/db/dbapi/sqlite.py @@ -18,19 +18,35 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # -import os +""" +Backend for sqlite database. +""" + +#------------------------------------------------------------------------- +# +# standard python modules +# +#------------------------------------------------------------------------- import sqlite3 import logging import re sqlite3.paramstyle = 'qmark' +#------------------------------------------------------------------------- +# +# Sqlite class +# +#------------------------------------------------------------------------- class Sqlite: + """ + The Sqlite class is an interface between the DBAPI class which is the Gramps + backend for the DBAPI interface and the sqlite3 python module. + """ @classmethod def get_summary(cls): """ - Return a diction of information about this database - backend. + Return a dictionary of information about this database backend. """ summary = { "DB-API version": "2.0", @@ -43,6 +59,18 @@ class Sqlite: return summary def __init__(self, *args, **kwargs): + """ + Create a new Sqlite instance. + + This connects to a sqlite3 database and creates a cursor instance. + + :param args: arguments to be passed to the sqlite3 connect class at + creation. + :type args: list + :param kwargs: arguments to be passed to the sqlite3 connect class at + creation. + :type kwargs: list + """ self.log = logging.getLogger(".sqlite") self.connection = sqlite3.connect(*args, **kwargs) self.cursor = self.connection.cursor() @@ -50,35 +78,84 @@ class Sqlite: self.connection.create_function("regexp", 2, regexp) def execute(self, *args, **kwargs): + """ + Executes an SQL statement. + + :param args: arguments to be passed to the sqlite3 execute statement + :type args: list + :param kwargs: arguments to be passed to the sqlite3 execute statement + :type kwargs: list + """ self.log.debug(args) self.cursor.execute(*args, **kwargs) def fetchone(self): + """ + Fetches the next row of a query result set, returning a single sequence, + or None when no more data is available. + """ return self.cursor.fetchone() def fetchall(self): + """ + Fetches the next set of rows of a query result, returning a list. An + empty list is returned when no more rows are available. + """ return self.cursor.fetchall() def begin(self): + """ + Start a transaction manually. This transactions usually persist until + the next COMMIT or ROLLBACK command. + """ self.log.debug("BEGIN TRANSACTION;") self.execute("BEGIN TRANSACTION;") def commit(self): + """ + Commit the current transaction. + """ self.log.debug("COMMIT;") self.connection.commit() def rollback(self): + """ + Roll back any changes to the database since the last call to commit(). + """ self.log.debug("ROLLBACK;") self.connection.rollback() def table_exists(self, table): + """ + Test whether the specified SQL database table exists. + + :param table: table name to check. + :type table: str + :returns: True if the table exists, false otherwise. + :rtype: bool + """ self.execute("SELECT COUNT(*) FROM sqlite_master " "WHERE type='table' AND name='%s';" % table) return self.fetchone()[0] != 0 def close(self): + """ + Close the current database. + """ self.log.debug("closing database...") self.connection.close() def regexp(expr, value): + """ + A user defined function that can be called from within an SQL statement. + + This function has two parameters. + + :param expr: pattern to look for. + :type expr: str + :param value: the string to search. + :type value: list + :returns: True if the expr exists within the value, false otherwise. + :rtype: bool + """ return re.search(expr, value, re.MULTILINE) is not None From c217eaae4d574e75636c44784ec317cee3c44af6 Mon Sep 17 00:00:00 2001 From: kulath Date: Mon, 1 Aug 2016 23:29:46 +0100 Subject: [PATCH 3/3] Replace %s with ? as requested by Doug Blank. --- gramps/plugins/db/dbapi/postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gramps/plugins/db/dbapi/postgresql.py b/gramps/plugins/db/dbapi/postgresql.py index 1eedfdd3d..fecd023a1 100644 --- a/gramps/plugins/db/dbapi/postgresql.py +++ b/gramps/plugins/db/dbapi/postgresql.py @@ -96,7 +96,7 @@ class Postgresql: def table_exists(self, table): self.cursor.execute("SELECT COUNT(*) FROM information_schema.tables " - "WHERE table_name='%s';" % table) + "WHERE table_name=?;", [table]) return self.fetchone()[0] != 0 def close(self):