sqlitedict for persisting metadata

pyproject.toml

@@ -17,7 +17,7 @@ description = "Experiment specification & orchestration."
 dependencies = [
     "cycler",
     "event-model>=1.19.8",
-    "historydict",
+    # "historydict",
     "importlib-resources;python_version<'3.9'",
     "msgpack",
     "msgpack-numpy",
@@ -26,8 +26,9 @@ dependencies = [
     "toolz",
     "tqdm>=4.44",
     "typing-extensions>=4.0.0;python_version<'3.11'",
-    "dataclasses;python_version<'3.7'",
-    "zict<3.0.0",
+    # "dataclasses;python_version<'3.7'",
+    # "zict>=3.0.0",
+    "sqlitedict",
 ]
 dynamic = ["version"]
 license.file = "LICENSE"

src/bluesky/run_engine.py

@@ -1861,7 +1861,7 @@ async def _open_run(self, msg):
         self.md_validator(dict(md))
 
         # Apply normalizer at the same level of the validator
-        validated = self.md_normalizer(copy.deepcopy(md))
+        validated = self.md_normalizer(dict(md))
 
         current_run = self._run_bundlers[run_key] = type(self).RunBundler(
             validated,

src/bluesky/tests/conftest.py

@@ -76,3 +76,8 @@ def cleanup_any_figures(request):
 
     "Close any matplotlib figures that were opened during a test."
     plt.close("all")
+
+
+@pytest.fixture(scope="function")
+def tmp_file(tmp_path):
+    return str(tmp_path / "dict.sqlite")

src/bluesky/tests/test_persistent_dict.py

@@ -7,22 +7,26 @@
 
 from ..plans import count
 from ..utils import PersistentDict
+from sqlitedict import SqliteDict as PersistentDict
+from functools import partial
+import msgpack_numpy
 
+PersistentDict = partial(PersistentDict, autocommit=True, journal_mode="OFF")
 
-def test_persistent_dict(tmp_path):
-    d = PersistentDict(tmp_path)
+def test_persistent_dict(tmp_file):
+    d = PersistentDict(tmp_file)
     d["a"] = 1
     d["b"] = (1, 2)
     d["c"] = numpy.zeros((5, 5))
     d["d"] = {"a": 10, "b": numpy.ones((5, 5))}
     expected = dict(d)
-    actual = PersistentDict(tmp_path)
+    actual = PersistentDict(tmp_file)
     recursive_assert_equal(actual, expected)
 
     # Update a value and check again.
     d["a"] = 2
     expected = dict(d)
-    actual.reload()  # Force load changes from disk
+    # actual.reload()  # Force load changes from disk
     recursive_assert_equal(actual, expected)
 
     # Test element deletion
@@ -32,28 +36,33 @@ def test_persistent_dict(tmp_path):
     assert "b" not in d
 
     # Smoke test the accessor and the __repr__.
-    assert d.directory == tmp_path
+    assert d.filename == tmp_file
     d.__repr__()
 
 
-def test_persistent_dict_mutable_value(tmp_path):
-    d = PersistentDict(tmp_path)
+def test_persistent_dict_mutable_value(tmp_file):
+    d = PersistentDict(tmp_file)
+    print(f"1. {dict(d) = }")
     d["a"] = []
+    print(f"2. {dict(d) = }")
     d["a"].append(1)
+    print(f"3. {dict(d) = }")
     expected = {"a": [1]}
     # Check the in-memory version is updated
     recursive_assert_equal(d, expected)
     # Check that the __repr__ reflects this.
     assert "{'a': [1]}" in repr(d)
+    # d.sync()
     # Check that contents are synced to disk at exit.
-    del d
-    gc.collect()
-    actual = PersistentDict(tmp_path)
+    # del d
+    # 
+    # gc.collect()
+    actual = PersistentDict(tmp_file)
     recursive_assert_equal(actual, expected)
 
 
-def test_pop(tmp_path):
-    d = PersistentDict(tmp_path)
+def test_pop(tmp_file):
+    d = PersistentDict(tmp_file)
     d["a"] = 1
     d["b"] = 2
     d.pop("b")
@@ -65,12 +74,12 @@ def test_pop(tmp_path):
     # Check that contents are synced to disk at exit.
     del d
     gc.collect()
-    actual = PersistentDict(tmp_path)
+    actual = PersistentDict(tmp_file)
     recursive_assert_equal(actual, expected)
 
 
-def test_popitem(tmp_path):
-    d = PersistentDict(tmp_path)
+def test_popitem(tmp_file):
+    d = PersistentDict(tmp_file)
     d["a"] = 1
     d["b"] = 2
     d.popitem()
@@ -82,12 +91,12 @@ def test_popitem(tmp_path):
     # Check that contents are synced to disk at exit.
     del d
     gc.collect()
-    actual = PersistentDict(tmp_path)
+    actual = PersistentDict(tmp_file)
     recursive_assert_equal(actual, expected)
 
 
-def test_update(tmp_path):
-    d = PersistentDict(tmp_path)
+def test_update(tmp_file):
+    d = PersistentDict(tmp_file)
     d.update(a=1)
     expected = {"a": 1}
     # Check the in-memory version is updated
@@ -97,12 +106,12 @@ def test_update(tmp_path):
     # Check that contents are synced to disk at exit.
     del d
     gc.collect()
-    actual = PersistentDict(tmp_path)
+    actual = PersistentDict(tmp_file)
     recursive_assert_equal(actual, expected)
 
 
-def test_setdefault(tmp_path):
-    d = PersistentDict(tmp_path)
+def test_setdefault(tmp_file):
+    d = PersistentDict(tmp_file)
     d.setdefault("a", 1)
     expected = {"a": 1}
     # Check the in-memory version is updated
@@ -112,12 +121,12 @@ def test_setdefault(tmp_path):
     # Check that contents are synced to disk at exit.
     del d
     gc.collect()
-    actual = PersistentDict(tmp_path)
+    actual = PersistentDict(tmp_file)
     recursive_assert_equal(actual, expected)
 
 
-def test_clear(tmp_path):
-    d = PersistentDict(tmp_path)
+def test_clear(tmp_file):
+    d = PersistentDict(tmp_file)
     d["a"] = 1
     d.clear()
     expected = {}
@@ -128,19 +137,19 @@ def test_clear(tmp_path):
     # Check that contents are synced to disk at exit.
     del d
     gc.collect()
-    actual = PersistentDict(tmp_path)
+    actual = PersistentDict(tmp_file)
     recursive_assert_equal(actual, expected)
 
 
-def test_integration(tmp_path, RE, hw):
+def test_integration(tmp_file, RE, hw):
     """
     Test integration with RE.
 
     Not looking for anything *specific* here, just general paranoia in case
     unforseen future changes create a bad interaction between PersistentDict
     and RE, as happened with HistoryDict and RE.
     """
-    d = PersistentDict(tmp_path)
+    d = PersistentDict(tmp_file)
     d["a"] = 1
     d["b"] = (1, 2)
     d["c"] = numpy.zeros((5, 5))
@@ -152,7 +161,7 @@ def test_integration(tmp_path, RE, hw):
     RE(count([hw.det]))
     recursive_assert_equal(RE.md, expected)
 
-    reloaded = PersistentDict(tmp_path)
+    reloaded = PersistentDict(tmp_file)
     recursive_assert_equal(reloaded, expected)