Skip to content

Commit

Permalink
Database format 21: add JSON, remove pickle
Browse files Browse the repository at this point in the history
  • Loading branch information
@dsblank
dsblank committed Oct 10, 2024
1 parent e84b26d commit a05fe5a
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 34 deletions.
7 changes: 5 additions & 2 deletions gramps/gen/db/generic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ class DbGeneric(DbWriteBase, DbReadBase, UpdateCallback, Callback):

__callback_map = {}

VERSION = (20, 0, 0)
VERSION = (21, 0, 0)

def __init__(self, directory=None):
DbReadBase.__init__(self)
Expand Down Expand Up @@ -1744,7 +1744,7 @@ def _iter_raw_place_tree_data(self):

def _get_raw_data(self, obj_key, handle):
"""
Return raw (serialized and pickled) object from handle.
Return raw (serialized) object from handle.
"""
raise NotImplementedError

Expand Down Expand Up @@ -2663,6 +2663,7 @@ def _gramps_upgrade(self, version, directory, callback=None):
gramps_upgrade_18,
gramps_upgrade_19,
gramps_upgrade_20,
gramps_upgrade_21,
)

if version < 14:
Expand All @@ -2679,6 +2680,8 @@ def _gramps_upgrade(self, version, directory, callback=None):
gramps_upgrade_19(self)
if version < 20:
gramps_upgrade_20(self)
if version < 21:
gramps_upgrade_21(self)

self.rebuild_secondary(callback)
self.reindex_reference_map(callback)
Expand Down
46 changes: 46 additions & 0 deletions gramps/gen/db/upgrade.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,52 @@
LOG = logging.getLogger(".upgrade")


def gramps_upgrade_21(self):
"""
Add unpickled data to tables.
"""
length = (
self.get_number_of_events()
+ self.get_number_of_places()
+ self.get_number_of_citations()
+ self.get_number_of_sources()
+ self.get_number_of_repositories()
+ self.get_number_of_media()
+ self.get_number_of_notes()
+ self.get_number_of_tags()
+ self.get_number_of_people()
)
self.set_total(length)
self._txn_begin()

tables = [
(PERSON_KEY, self.get_person_handles, "person"),
(FAMILY_KEY, self.get_family_handles, "family"),
(EVENT_KEY, self.get_event_handles, "event"),
(MEDIA_KEY, self.get_media_handles, "media"),
(PLACE_KEY, self.get_place_handles, "place"),
(REPOSITORY_KEY, self.get_repository_handles, "repository"),
(CITATION_KEY, self.get_citation_handles, "citation"),
(SOURCE_KEY, self.get_source_handles, "source"),
(NOTE_KEY, self.get_note_handles, "note"),
(TAG_KEY, self.get_tag_handles, "tag"),
]
for (key, method, table_name) in tables:
try:
self.dbapi.execute("ALTER TABLE %s ADD COLUMN unblob TEXT;" % table_name)
self.dbapi.commit()
except Exception:
pass

for handle in method():
obj = self._get_raw_data(key, handle)
self._commit_raw(obj, key)
self.update()

self._txn_commit()
# Bump up database version. Separate transaction to save metadata.
self._set_metadata("version", 21)

def gramps_upgrade_20(self):
"""
Placeholder update.
Expand Down
65 changes: 33 additions & 32 deletions gramps/plugins/db/dbapi/dbapi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
# Python modules
#
# -------------------------------------------------------------------------
import json
import logging
import pickle
import time
Expand Down Expand Up @@ -103,71 +104,71 @@ def _create_schema(self):
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"given_name TEXT, "
"surname TEXT, "
"blob_data BLOB"
"unblob TEXT"
")"
)
self.dbapi.execute(
"CREATE TABLE family "
"("
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"blob_data BLOB"
"unblob TEXT"
")"
)
self.dbapi.execute(
"CREATE TABLE source "
"("
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"blob_data BLOB"
"unblob TEXT"
")"
)
self.dbapi.execute(
"CREATE TABLE citation "
"("
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"blob_data BLOB"
"unblob TEXT"
")"
)
self.dbapi.execute(
"CREATE TABLE event "
"("
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"blob_data BLOB"
"unblob TEXT"
")"
)
self.dbapi.execute(
"CREATE TABLE media "
"("
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"blob_data BLOB"
"unblob TEXT"
")"
)
self.dbapi.execute(
"CREATE TABLE place "
"("
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"enclosed_by VARCHAR(50), "
"blob_data BLOB"
"unblob TEXT"
")"
)
self.dbapi.execute(
"CREATE TABLE repository "
"("
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"blob_data BLOB"
"unblob TEXT"
")"
)
self.dbapi.execute(
"CREATE TABLE note "
"("
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"blob_data BLOB"
"unblob TEXT"
")"
)
self.dbapi.execute(
"CREATE TABLE tag "
"("
"handle VARCHAR(50) PRIMARY KEY NOT NULL, "
"blob_data BLOB"
"unblob TEXT"
")"
)
# Secondary:
Expand Down Expand Up @@ -579,10 +580,10 @@ def get_tag_from_name(self, name):
If no such Tag exists, None is returned.
"""
self.dbapi.execute("SELECT blob_data FROM tag WHERE name = ?", [name])
self.dbapi.execute("SELECT unblob FROM tag WHERE name = ?", [name])
row = self.dbapi.fetchone()
if row:
return Tag.create(pickle.loads(row[0]))
return Tag.create(json.loads(row[0]))
return None

def _get_number_of(self, obj_key):
Expand Down Expand Up @@ -635,14 +636,14 @@ def _commit_base(self, obj, obj_key, trans, change_time):
old_data = self._get_raw_data(obj_key, obj.handle)
# update the object:
self.dbapi.execute(
f"UPDATE {table} SET blob_data = ? WHERE handle = ?",
[pickle.dumps(obj.serialize()), obj.handle],
f"UPDATE {table} SET unblob = ? WHERE handle = ?",
[json.dumps(obj.serialize()), obj.handle],
)
else:
# Insert the object:
self.dbapi.execute(
f"INSERT INTO {table} (handle, blob_data) VALUES (?, ?)",
[obj.handle, pickle.dumps(obj.serialize())],
f"INSERT INTO {table} (handle, unblob) VALUES (?, ?)",
[obj.handle, json.dumps(obj.serialize())],
)
self._update_secondary_values(obj)
self._update_backlinks(obj, trans)
Expand All @@ -665,14 +666,14 @@ def _commit_raw(self, data, obj_key):
if self._has_handle(obj_key, handle):
# update the object:
self.dbapi.execute(
f"UPDATE {table} SET blob_data = ? WHERE handle = ?",
[pickle.dumps(data), handle],
f"UPDATE {table} SET unblob = ? WHERE handle = ?",
[json.dumps(data), handle],
)
else:
# Insert the object:
self.dbapi.execute(
f"INSERT INTO {table} (handle, blob_data) VALUES (?, ?)",
[handle, pickle.dumps(data)],
f"INSERT INTO {table} (handle, unblob) VALUES (?, ?)",
[handle, json.dumps(data)],
)

def _update_backlinks(self, obj, transaction):
Expand Down Expand Up @@ -829,11 +830,11 @@ def _iter_raw_data(self, obj_key):
"""
table = KEY_TO_NAME_MAP[obj_key]
with self.dbapi.cursor() as cursor:
cursor.execute(f"SELECT handle, blob_data FROM {table}")
cursor.execute(f"SELECT handle, unblob FROM {table}")
rows = cursor.fetchmany()
while rows:
for row in rows:
yield (row[0], pickle.loads(row[1]))
yield (row[0], json.loads(row[1]))
rows = cursor.fetchmany()

def _iter_raw_place_tree_data(self):
Expand All @@ -844,12 +845,12 @@ def _iter_raw_place_tree_data(self):
while to_do:
handle = to_do.pop()
self.dbapi.execute(
"SELECT handle, blob_data FROM place WHERE enclosed_by = ?", [handle]
"SELECT handle, unblob FROM place WHERE enclosed_by = ?", [handle]
)
rows = self.dbapi.fetchall()
for row in rows:
to_do.append(row[0])
yield (row[0], pickle.loads(row[1]))
yield (row[0], json.loads(row[1]))

def reindex_reference_map(self, callback):
"""
Expand Down Expand Up @@ -974,20 +975,20 @@ def _get_gramps_ids(self, obj_key):

def _get_raw_data(self, obj_key, handle):
table = KEY_TO_NAME_MAP[obj_key]
self.dbapi.execute(f"SELECT blob_data FROM {table} WHERE handle = ?", [handle])
self.dbapi.execute(f"SELECT unblob FROM {table} WHERE handle = ?", [handle])
row = self.dbapi.fetchone()
if row:
return pickle.loads(row[0])
return json.loads(row[0])
return None

def _get_raw_from_id_data(self, obj_key, gramps_id):
table = KEY_TO_NAME_MAP[obj_key]
self.dbapi.execute(
f"SELECT blob_data FROM {table} WHERE gramps_id = ?", [gramps_id]
f"SELECT unblob FROM {table} WHERE gramps_id = ?", [gramps_id]
)
row = self.dbapi.fetchone()
if row:
return pickle.loads(row[0])
return json.loads(row[0])
return None

def get_gender_stats(self):
Expand Down Expand Up @@ -1042,13 +1043,13 @@ def undo_data(self, data, handle, obj_key):
else:
if self._has_handle(obj_key, handle):
self.dbapi.execute(
f"UPDATE {table} SET blob_data = ? WHERE handle = ?",
[pickle.dumps(data), handle],
f"UPDATE {table} SET unblob = ? WHERE handle = ?",
[json.dumps(data), handle],
)
else:
self.dbapi.execute(
f"INSERT INTO {table} (handle, blob_data) VALUES (?, ?)",
[handle, pickle.dumps(data)],
f"INSERT INTO {table} (handle, unblob) VALUES (?, ?)",
[handle, json.dumps(data)],
)
obj = self._get_table_func(cls)["class_func"].create(data)
self._update_secondary_values(obj)
Expand Down

0 comments on commit a05fe5a

Please sign in to comment.