Implement more of the bdev api

src/bdev_lsvd.cc

@@ -1,9 +1,12 @@
 #include "rados/librados.h"
 #include "spdk/bdev_module.h"
+#include <future>
 
 #include "bdev_lsvd.h"
 #include "image.h"
-#include <future>
+#include "smartiov.h"
+#include "spdk/thread.h"
+#include "utils.h"
 
 static int bdev_lsvd_init(void);
 static void bdev_lsvd_finish(void);
@@ -18,31 +21,22 @@ SPDK_BDEV_MODULE_REGISTER(ext_lsvd, &lsvd_if);
 static int bdev_lsvd_init(void) { return 0; }
 static void bdev_lsvd_finish(void) {}
 
-static bool lsvd_bdev_io_type_supported(void *ctx, spdk_bdev_io_type io_type);
+/**
+ * Function table for the LSVD bdev module.
+ */
+
+static int lsvd_destroy_bdev(void *);
+static void lsvd_submit_io(spdk_io_channel *c, spdk_bdev_io *io);
+static bool lsvd_io_type_supported(void *ctx, spdk_bdev_io_type io_type);
+static spdk_io_channel *lsvd_get_io_channel(void *ctx);
 
 static const spdk_bdev_fn_table lsvd_fn_table = {
-    .destruct = nullptr,
-    .submit_request = nullptr,
-    .io_type_supported = &lsvd_bdev_io_type_supported,
-    .get_io_channel = nullptr,
-    .dump_info_json = nullptr,
+    .destruct = lsvd_destroy_bdev,
+    .submit_request = lsvd_submit_io,
+    .io_type_supported = lsvd_io_type_supported,
+    .get_io_channel = lsvd_get_io_channel,
 };
 
-static bool lsvd_bdev_io_type_supported(void *ctx, spdk_bdev_io_type io_type)
-{
-    switch (io_type) {
-    case SPDK_BDEV_IO_TYPE_READ:
-    case SPDK_BDEV_IO_TYPE_WRITE:
-    case SPDK_BDEV_IO_TYPE_FLUSH:        // we only use this to ensure ordering
-    case SPDK_BDEV_IO_TYPE_RESET:        // block until all pending io aborts
-    case SPDK_BDEV_IO_TYPE_UNMAP:        // trim
-    case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: // also just trim
-        return true;
-    default:
-        return false;
-    }
-}
-
 class lsvd_iodevice
 {
   public:
@@ -63,8 +57,76 @@ class lsvd_iodevice
     ~lsvd_iodevice() {}
 };
 
-class lsvd_bdev_io_channel
+static int lsvd_destroy_bdev(void *ctx)
 {
+    auto iodev = reinterpret_cast<lsvd_iodevice *>(ctx);
+    delete iodev;
+    return 0;
+}
+
+static bool lsvd_io_type_supported(void *ctx, spdk_bdev_io_type io_type)
+{
+    switch (io_type) {
+    case SPDK_BDEV_IO_TYPE_READ:
+    case SPDK_BDEV_IO_TYPE_WRITE:
+    case SPDK_BDEV_IO_TYPE_FLUSH:        // we only use this to ensure ordering
+    case SPDK_BDEV_IO_TYPE_UNMAP:        // trim
+    case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: // also just trim
+        return true;
+    case SPDK_BDEV_IO_TYPE_RESET: // block until all pending io aborts
+    default:
+        return false;
+    }
+}
+
+static spdk_io_channel *lsvd_get_io_channel(void *ctx)
+{
+    lsvd_iodevice *iodev = reinterpret_cast<lsvd_iodevice *>(ctx);
+    // SPDK will pass this to the iodevice's registered create/destroy
+    // io_channel functions that were passed in when the device was registered.
+    // We don't need to do anything special here, so just return the iodevice.
+    return spdk_get_io_channel(iodev);
+}
+
+static void lsvd_submit_io(spdk_io_channel *c, spdk_bdev_io *io)
+{
+    auto dev = static_cast<lsvd_iodevice *>(io->bdev->ctxt);
+    auto &img = dev->img;
+
+    // io details
+    auto offset = io->u.bdev.offset_blocks * io->bdev->blocklen;
+    auto len = io->u.bdev.num_blocks * io->bdev->blocklen;
+    smartiov iov(io->u.bdev.iovs, io->u.bdev.iovcnt);
+
+    request *r;
+    switch (io->type) {
+    case SPDK_BDEV_IO_TYPE_READ:
+        r = img->read(offset, iov, nullptr);
+        break;
+    case SPDK_BDEV_IO_TYPE_WRITE:
+        r = img->write(offset, iov, nullptr);
+        break;
+    case SPDK_BDEV_IO_TYPE_FLUSH:
+        r = img->flush(nullptr);
+        break;
+    case SPDK_BDEV_IO_TYPE_UNMAP:
+        r = img->trim(offset, len, nullptr);
+        break;
+    case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
+        r = img->trim(offset, len, nullptr);
+        break;
+    default:
+        log_error("Unknown request type: {}", io->type);
+        return;
+    }
+
+    r->run(nullptr);
+}
+
+// Just copying from bdev_rbd, not sure where this is actually used
+struct lsvd_bdev_io_channel {
+    lsvd_iodevice *lsvd_dev;
+    spdk_io_channel *io_channel;
 };
 
 int bdev_lsvd_create(std::string img_name, rados_ioctx_t ioctx)
@@ -74,12 +136,24 @@ int bdev_lsvd_create(std::string img_name, rados_ioctx_t ioctx)
     auto img = lsvd_image::open_image(img_name, ioctx);
     auto iodev = new lsvd_iodevice(std::move(img));
 
-    spdk_io_device_register(iodev, nullptr, nullptr,
-                            sizeof(lsvd_bdev_io_channel), img_name.c_str());
+    spdk_io_device_register(
+        iodev,
+        [](void *iodev, void *ctx_buf) {
+            auto *ch = static_cast<lsvd_bdev_io_channel *>(ctx_buf);
+            ch->lsvd_dev = static_cast<lsvd_iodevice *>(iodev);
+            ch->io_channel = spdk_get_io_channel(&lsvd_if);
+            return 0;
+        },
+        [](void *iodev, void *ctx_buf) {
+            auto *ch = static_cast<lsvd_bdev_io_channel *>(ctx_buf);
+            spdk_put_io_channel(ch->io_channel);
+        },
+        sizeof(lsvd_bdev_io_channel), img_name.c_str());
     auto err = spdk_bdev_register(&iodev->bdev);
     if (err) {
         log_error("Failed to register bdev: err {}", (err));
-        spdk_io_device_unregister(iodev, nullptr);
+        spdk_io_device_unregister(
+            iodev, [](void *ctx) { delete (lsvd_iodevice *)ctx; });
         return err;
     }
 

src/image.cc

@@ -30,7 +30,7 @@ lsvd_image::~lsvd_image()
 
 uptr<lsvd_image> lsvd_image::open_image(std::string name, rados_ioctx_t io)
 {
-    uptr<lsvd_image> img;
+    uptr<lsvd_image> img(new lsvd_image());
     try {
         img->try_open(name, io);
         return img;

src/spdk_frontend.cc

@@ -1,11 +1,33 @@
-#include "spdk/env.h"
 #include "spdk/event.h"
 
+#include "bdev_lsvd.h"
 #include "utils.h"
 
 static void start_lsvd(void *arg)
 {
     log_info("Starting LSVD SPDK program ...");
+
+    setenv("LSVD_RCACHE_DIR", "/tmp/lsvd-read", 1);
+    setenv("LSVD_WCACHE_DIR", "/tmp/lsvd-write", 1);
+    setenv("LSVD_CACHE_SIZE", "2147483648", 1);
+
+    std::string pool_name = "pone";
+
+    rados_t cluster;
+    int err = rados_create2(&cluster, "ceph", "client.admin", 0);
+    check_ret_neg(err, "Failed to create cluster handle");
+
+    err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
+    check_ret_neg(err, "Failed to read config file");
+
+    err = rados_connect(cluster);
+    check_ret_neg(err, "Failed to connect to cluster");
+
+    rados_ioctx_t io_ctx;
+    err = rados_ioctx_create(cluster, pool_name.c_str(), &io_ctx);
+    check_ret_neg(err, "Failed to connect to pool {}", pool_name);
+
+    err = bdev_lsvd_create("test-image", io_ctx);
 }
 
 int main(int argc, char **argv)