Adding options to the Python plugin from_bytes() and to_bytes() metho…

…ds (#880)

Adding options to the Python plugin from_bytes() and to_bytes() methods.

Note, that this PR will not make old Python plugins with to_bytes() and/or from_bytes() methods without the options argument stop working.

bindings/python/dlite-entity-python.i

@@ -486,11 +486,11 @@ def get_instance(
         )
 
     @classmethod
-    def from_bytes(cls, driver, buffer, id=None):
+    def from_bytes(cls, driver, buffer, id=None, options=None):
         """Load the instance with ID `id` from bytes `buffer` using the
         given storage driver.
         """
-        return _from_bytes(driver, buffer, id=id)
+        return _from_bytes(driver, buffer, id=id, options=options)
 
     @classmethod
     def create_metadata(cls, uri, dimensions, properties, description):

bindings/python/dlite-entity.i

@@ -421,9 +421,10 @@ Call signatures:
   %feature("docstring",
            "Save instance to bytes using given storage driver.") to_bytes;
   %newobject to_bytes;
-  void to_bytes(const char *driver, unsigned char **ARGOUT_BYTES, size_t *LEN) {
+  void to_bytes(const char *driver, unsigned char **ARGOUT_BYTES, size_t *LEN,
+                const char *options=NULL) {
     unsigned char *buf=NULL;
-    int m, n = dlite_instance_memsave(driver, buf, 0, $self);
+    int m, n = dlite_instance_memsave(driver, buf, 0, $self, options);
     if (n < 0) {
       *ARGOUT_BYTES = NULL;
       *LEN = 0;
@@ -433,7 +434,8 @@ Call signatures:
       dlite_err(dliteMemoryError, "allocation failure");
       return;
     }
-    if ((m = dlite_instance_memsave(driver, buf, n, $self)) < 0) return;
+    if ((m = dlite_instance_memsave(driver, buf, n, $self, options)) < 0)
+      return;
     assert (m == n);
     *ARGOUT_BYTES = buf;
     *LEN = n;
@@ -836,9 +838,11 @@ bool dlite_instance_has_property(struct _DLiteInstance *inst, const char *name);
 
 %rename(_from_bytes) dlite_instance_memload;
 %feature("docstring", "Loads instance from string.") dlite_instance_memload;
-struct _DLiteInstance *dlite_instance_memload(const char *driver,
-                                      unsigned char *INPUT_BYTES, size_t LEN,
-                                      const char *id=NULL);
+struct _DLiteInstance *
+         dlite_instance_memload(const char *driver,
+                                unsigned char *INPUT_BYTES, size_t LEN,
+                                const char *id=NULL, const char *options=NULL);
+
 
 /* FIXME - how do we avoid duplicating these constants from dlite-schemas.h? */
 #define BASIC_METADATA_SCHEMA  "http://onto-ns.com/meta/0.1/BasicMetadataSchema"

bindings/python/dlite-python.i

@@ -1289,7 +1289,7 @@ int dlite_swig_set_property_by_index(DLiteInstance *inst, int i, obj_t *obj)
  * --------------- */
 
 /* Argout bytes */
-/* We assumes that *ARGOUT_BYTES is a malloc() by the wrapped function */
+/* Assumes that *ARGOUT_BYTES is malloc()'ed by the wrapped function */
 %typemap("doc") (unsigned char **ARGOUT_BYTES, size_t *LEN) "bytes"
 %typemap(in,numinputs=0) (unsigned char **ARGOUT_BYTES, size_t *LEN)
   (unsigned char *tmp, size_t n) {

bindings/python/tests/test_storage.py

@@ -105,11 +105,20 @@
 
     # Test to_bytes()/from_bytes()
     data = inst.to_bytes("yaml")
-    data2 = data.replace(b"uri: my-data", b"uri: my-data2")
+    data2 = data.replace(b"uri: my-data", b"uri: my-data2").replace(
+        dlite.get_uuid("my-data").encode(),
+        dlite.get_uuid("my-data2").encode()
+    )
+    assert data2.startswith(b"uri: my-data2")
+
     inst2 = dlite.Instance.from_bytes("yaml", data2)
     assert inst2.uuid != inst.uuid
     assert inst2.get_hash() == inst.get_hash()
 
+    # Test to_bytes() with options
+    data3 = inst.to_bytes("yaml", options="single=false")
+    assert data3.startswith(b"ce592ff9-a7dc-548a-bbe5-3c53800afaf3")
+
     s.flush()  # avoid calling flush() when the interpreter is teared down
 
 

doc/user_guide/storage_plugin.py

@@ -67,24 +67,26 @@ def queue(self, pattern=None):
         """
 
     @classmethod
-    def from_bytes(cls, buffer, id=None):
+    def from_bytes(cls, buffer, id=None, options=None):
         """Load instance with given `id` from `buffer`.
 
         Arguments:
             buffer: Bytes or bytearray object to load the instance from.
             id: ID of instance to load.  May be omitted if `buffer` only
                 holds one instance.
+            options: Options string for this storage driver.
 
         Returns:
             New instance.
         """
 
     @classmethod
-    def to_bytes(cls, inst):
+    def to_bytes(cls, inst, options=None):
         """Save instance `inst` to bytes (or bytearray) object.  Optional.
 
         Arguments:
             inst: Instance to save.
+            options: Options string for this storage driver.
 
         Returns:
             The bytes (or bytearray) object that the instance is saved to.

src/dlite-entity.c

@@ -1144,14 +1144,14 @@ int dlite_instance_save_url(const char *url, const DLiteInstance *inst)
  */
 DLiteInstance *dlite_instance_memload(const char *driver,
                                       const unsigned char *buf, size_t size,
-                                      const char *id)
+                                      const char *id, const char *options)
 {
   const DLiteStoragePlugin *api;
   if (!(api = dlite_storage_plugin_get(driver))) return NULL;
   if (!api->memLoadInstance)
     return err(dliteUnsupportedError, "driver does not support memload: %s",
                api->name), NULL;
-  return api->memLoadInstance(api, buf, size, id);
+  return api->memLoadInstance(api, buf, size, id, options);
 }
 
 /*
@@ -1162,22 +1162,23 @@ DLiteInstance *dlite_instance_memload(const char *driver,
   Returns a negative error code on error.
  */
 int dlite_instance_memsave(const char *driver, unsigned char *buf, size_t size,
-                           const DLiteInstance *inst)
+                           const DLiteInstance *inst, const char *options)
 {
   const DLiteStoragePlugin *api;
   if (!(api = dlite_storage_plugin_get(driver))) return dliteStorageSaveError;
   if (!api->memSaveInstance)
     return err(dliteUnsupportedError, "driver does not support memsave: %s",
                api->name);
-  return api->memSaveInstance(api, buf, size, inst);
+  return api->memSaveInstance(api, buf, size, inst, options);
 }
 
 /*
   Saves instance to a newly allocated memory buffer.
   Returns NULL on error.
  */
 unsigned char *dlite_instance_to_memory(const char *driver,
-                                        const DLiteInstance *inst)
+                                        const DLiteInstance *inst,
+                                        const char *options)
 {
   const DLiteStoragePlugin *api;
   int n=0, m;
@@ -1187,10 +1188,10 @@ unsigned char *dlite_instance_to_memory(const char *driver,
     return err(dliteUnsupportedError, "driver does not support memsave: %s",
                api->name), NULL;
 
-  if ((n = api->memSaveInstance(api, buf, n, inst)) < 0) return NULL;
+  if ((n = api->memSaveInstance(api, buf, n, inst, options)) < 0) return NULL;
   if (!(buf = malloc(n)))
     return err(dliteMemoryError, "allocation failure"), NULL;
-  if ((m = api->memSaveInstance(api, buf, n, inst)) != n) {
+  if ((m = api->memSaveInstance(api, buf, n, inst, options)) != n) {
     assert(m < 0);  // On success, should `m` always be equal to `n`!
     free(buf);
     return NULL;

src/dlite-entity.h

@@ -628,7 +628,7 @@ int dlite_instance_save_url(const char *url, const DLiteInstance *inst);
  */
 DLiteInstance *dlite_instance_memload(const char *driver,
                                       const unsigned char *buf, size_t size,
-                                      const char *id);
+                                      const char *id, const char *options);
 
 /**
   Stores instance `inst` to memory buffer `buf` of size `size`.
@@ -638,14 +638,15 @@ DLiteInstance *dlite_instance_memload(const char *driver,
   Returns a negative error code on error.
  */
 int dlite_instance_memsave(const char *driver, unsigned char *buf, size_t size,
-                           const DLiteInstance *inst);
+                           const DLiteInstance *inst, const char *options);
 
 /**
   Saves instance to a newly allocated memory buffer.
   Returns NULL on error.
  */
 unsigned char *dlite_instance_to_memory(const char *driver,
-                                        const DLiteInstance *inst);
+                                        const DLiteInstance *inst,
+                                        const char *options);
 
 /**
   Returns a pointer to instance UUID.

src/dlite-storage-plugins.h

@@ -379,7 +379,7 @@ typedef int (*DeleteInstance)(DLiteStorage *s, const char *id);
  */
 typedef DLiteInstance *(*MemLoadInstance)(const DLiteStoragePlugin *api,
                                           const unsigned char *buf, size_t size,
-                                          const char *id);
+                                          const char *id, const char *options);
 
 /**
   Stores instance `inst` to memory buffer `buf` of size `size`.
@@ -390,7 +390,8 @@ typedef DLiteInstance *(*MemLoadInstance)(const DLiteStoragePlugin *api,
  */
 typedef int (*MemSaveInstance)(const DLiteStoragePlugin *api,
                                unsigned char *buf, size_t size,
-                               const DLiteInstance *inst);
+                               const DLiteInstance *inst,
+                               const char *options);
 
 /** @} */
 

storages/python/dlite-plugins-python.c

@@ -328,7 +328,8 @@ int deleter(DLiteStorage *s, const char *id)
   Loads instance with given id from bytes object.
  */
 DLiteInstance *memloader(const DLiteStoragePlugin *api,
-                         const unsigned char *buf, size_t size, const char *id)
+                         const unsigned char *buf, size_t size,
+                         const char *id, const char *options)
 {
   DLiteInstance *inst = NULL;
   PyObject *v = NULL;
@@ -337,8 +338,16 @@ DLiteInstance *memloader(const DLiteStoragePlugin *api,
   PyErr_Clear();
   if (!(classname = dlite_pyembed_classname(class)))
     dlite_warnx("cannot get class name for storage plugin '%s'", api->name);
-  v = PyObject_CallMethod(class, "from_bytes", "y#s",
-                          (const char *)buf, (Py_ssize_t) size, id);
+
+  /* Keep backward compatibility with Python plugins that do not have an
+     `options` argument in their `from_bytes()` method. */
+  if (options)
+    v = PyObject_CallMethod(class, "from_bytes", "y#ss",
+                            (const char *)buf, (Py_ssize_t) size, id, options);
+  else
+    v = PyObject_CallMethod(class, "from_bytes", "y#s",
+                            (const char *)buf, (Py_ssize_t) size, id);
+
   if (dlite_pyembed_err_check("calling from_bytes() in Python plugin '%s'",
                               classname)) {
     Py_XDECREF(v);
@@ -357,7 +366,7 @@ DLiteInstance *memloader(const DLiteStoragePlugin *api,
   Saves instance to bytes object.
  */
 int memsaver(const DLiteStoragePlugin *api, unsigned char *buf, size_t size,
-             const DLiteInstance *inst)
+             const DLiteInstance *inst, const char *options)
 {
   Py_ssize_t length = 0;
   char *buffer = NULL;
@@ -370,7 +379,14 @@ int memsaver(const DLiteStoragePlugin *api, unsigned char *buf, size_t size,
   if (!pyinst) goto fail;
   if (!(classname = dlite_pyembed_classname(class)))
     dlite_warnx("cannot get class name for storage plugin '%s'", api->name);
-  v = PyObject_CallMethod(class, "to_bytes", "O", pyinst);
+
+  /* Keep backward compatibility with Python plugins that do not have an
+     `options` argument in their `to_bytes()` method. */
+  if (options)
+    v = PyObject_CallMethod(class, "to_bytes", "Os", pyinst, options);
+  else
+    v = PyObject_CallMethod(class, "to_bytes", "O", pyinst);
+
   if (dlite_pyembed_err_check("calling to_bytes() in Python plugin '%s'%s",
                               classname, failmsg()))
     goto fail;

storages/python/python-storage-plugins/yaml.py

@@ -130,7 +130,7 @@ def queue(self, pattern=None):
             yield uuid
 
     @classmethod
-    def from_bytes(cls, buffer, id=None):
+    def from_bytes(cls, buffer, id=None, options=None):
         """Load instance with given `id` from `buffer`.
 
         Arguments:
@@ -148,19 +148,28 @@ def from_bytes(cls, buffer, id=None):
         )
 
     @classmethod
-    def to_bytes(cls, inst, soft7=True, with_uuid=False):
+    def to_bytes(cls, inst, options=None):
         """Save instance `inst` to bytes (or bytearray) object.  Optional.
 
         Arguments:
             inst: Instance to save.
-            soft7: Whether to structure metadata as SOFT7.
-            with_uuid: Whether to include UUID in the output.
+            options: Supported options:
+            - `soft7`: Whether to structure metadata as SOFT7.
+            - `with_uuid`: Whether to include UUID in the output.
+            - `single`: Whether to include UUID in the output.
 
         Returns:
             The bytes (or bytearray) object that the instance is saved to.
         """
+        opts = Options(
+            options, defaults="soft7=true;with_uuid=false;single=true"
+        )
         return pyyaml.safe_dump(
-            inst.asdict(soft7=soft7, uuid=with_uuid, single=True),
+            inst.asdict(
+                soft7=dlite.asbool(opts.soft7),
+                uuid=dlite.asbool(opts.with_uuid),
+                single=dlite.asbool(opts.single),
+            ),
             default_flow_style=False,
             sort_keys=False,
         ).encode()