vfs: add a VFS wrapper used in deterministic simulation tests (#910)

* dst/vfs: add vfs wrapper over wazero's sys.FS

This allows us to control filesystem operations. Currently, the implementation
is purely pass-through.

* dst/vfs: add WASM host hooks to simulate hard shutdown

This commit also instantiates these hooks in the wazero runtime.

dst/dst_test.go

@@ -339,6 +339,12 @@ func newTestLogger(t testing.TB) log.Logger {
 	return logger
 }
 
+// Remove unused warnings for now.
+var (
+	_ = vfsShutdown
+	_ = vfsRestart
+)
+
 // TestDST runs deterministic simulation tests against FrostDB. For true
 // determinism and reproducibility, this test needs to be run with
 // GORANDSEED set, the modified go runtime found at github.com/polarsignals/go,

dst/runtime/run.go

@@ -8,6 +8,8 @@ import (
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/experimental/sysfs"
 	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
+
+	"github.com/polarsignals/frostdb/dst/vfs"
 )
 
 const (
@@ -42,7 +44,7 @@ func run(modulePath string) error {
 		WithStdout(os.Stdout).
 		WithStderr(os.Stderr).
 		// Mount filesystem. This is taken from wazero's CLI implementation.
-		WithFSConfig(wazero.NewFSConfig().(sysfs.FSConfig).WithSysFSMount(sysfs.DirFS("/"), "/")).
+		WithFSConfig(wazero.NewFSConfig().(sysfs.FSConfig).WithSysFSMount(vfs.New("/"), "/")).
 		// All these time-related configuration options are to allow the module
 		// to access "real" time on the host. We could use this as a source of
 		// determinisme, but we currently compile the module with -faketime
@@ -53,6 +55,8 @@ func run(modulePath string) error {
 		WithSysWalltime().
 		WithArgs(os.Args...)
 
+	vfs.MustInstantiate(ctx, r)
+
 	moduleBytes, err := os.ReadFile(modulePath)
 	if err != nil {
 		return fmt.Errorf("reading module: %w", err)

dst/vfs/file.go

@@ -0,0 +1,135 @@
+package vfs
+
+import (
+	experimentalsys "github.com/tetratelabs/wazero/experimental/sys"
+	"github.com/tetratelabs/wazero/sys"
+)
+
+type file struct {
+	// internal is the underlying file system to delegate to. This is
+	// purposefully not embedded so that any new methods need to be explicitly
+	// added.
+	internal experimentalsys.File
+}
+
+var _ experimentalsys.File = (*file)(nil)
+
+func newFile(f experimentalsys.File) *file {
+	return &file{internal: f}
+}
+
+func (f *file) Dev() (uint64, experimentalsys.Errno) {
+	if isShutdown {
+		return 0, experimentalsys.EIO
+	}
+	return f.internal.Dev()
+}
+
+func (f *file) Ino() (sys.Inode, experimentalsys.Errno) {
+	if isShutdown {
+		return 0, experimentalsys.EIO
+	}
+	return f.internal.Ino()
+}
+
+func (f *file) IsDir() (bool, experimentalsys.Errno) {
+	if isShutdown {
+		return false, experimentalsys.EIO
+	}
+	return f.internal.IsDir()
+}
+
+func (f *file) IsAppend() bool {
+	return f.internal.IsAppend()
+}
+
+func (f *file) SetAppend(enable bool) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return f.internal.SetAppend(enable)
+}
+
+func (f *file) Stat() (sys.Stat_t, experimentalsys.Errno) {
+	if isShutdown {
+		return sys.Stat_t{}, experimentalsys.EIO
+	}
+	return f.internal.Stat()
+}
+
+func (f *file) Read(buf []byte) (n int, errno experimentalsys.Errno) {
+	if isShutdown {
+		return 0, experimentalsys.EIO
+	}
+	return f.internal.Read(buf)
+}
+
+func (f *file) Pread(buf []byte, off int64) (n int, errno experimentalsys.Errno) {
+	if isShutdown {
+		return 0, experimentalsys.EIO
+	}
+	return f.internal.Pread(buf, off)
+}
+
+func (f *file) Seek(offset int64, whence int) (newOffset int64, errno experimentalsys.Errno) {
+	if isShutdown {
+		return 0, experimentalsys.EIO
+	}
+	return f.internal.Seek(offset, whence)
+}
+
+func (f *file) Readdir(n int) (dirents []experimentalsys.Dirent, errno experimentalsys.Errno) {
+	if isShutdown {
+		return nil, experimentalsys.EIO
+	}
+	return f.internal.Readdir(n)
+}
+
+func (f *file) Write(buf []byte) (n int, errno experimentalsys.Errno) {
+	if isShutdown {
+		return 0, experimentalsys.EIO
+	}
+	return f.internal.Write(buf)
+}
+
+func (f *file) Pwrite(buf []byte, off int64) (n int, errno experimentalsys.Errno) {
+	if isShutdown {
+		return 0, experimentalsys.EIO
+	}
+	return f.internal.Pwrite(buf, off)
+}
+
+func (f *file) Truncate(size int64) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return f.internal.Truncate(size)
+}
+
+func (f *file) Sync() experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return f.internal.Sync()
+}
+
+func (f *file) Datasync() experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return f.internal.Datasync()
+}
+
+func (f *file) Utimens(atim, mtim int64) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return f.internal.Utimens(atim, mtim)
+}
+
+func (f *file) Close() experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return f.internal.Close()
+}

dst/vfs/fs.go

@@ -0,0 +1,111 @@
+package vfs
+
+import (
+	"io/fs"
+
+	experimentalsys "github.com/tetratelabs/wazero/experimental/sys"
+	"github.com/tetratelabs/wazero/experimental/sysfs"
+	"github.com/tetratelabs/wazero/sys"
+)
+
+type dstfs struct {
+	// internal is the underlying file system to delegate to. This is
+	// purposefully not embedded so that any new methods need to be explicitly
+	// added.
+	internal experimentalsys.FS
+}
+
+var _ experimentalsys.FS = (*dstfs)(nil)
+
+func New(dir string) experimentalsys.FS {
+	return &dstfs{internal: sysfs.DirFS(dir)}
+}
+
+func (d *dstfs) OpenFile(path string, flag experimentalsys.Oflag, perm fs.FileMode) (experimentalsys.File, experimentalsys.Errno) {
+	if isShutdown {
+		return nil, experimentalsys.EIO
+	}
+
+	f, err := d.internal.OpenFile(path, flag, perm)
+	if err != 0 {
+		return nil, err
+	}
+	return newFile(f), 0
+}
+
+func (d *dstfs) Lstat(path string) (sys.Stat_t, experimentalsys.Errno) {
+	if isShutdown {
+		return sys.Stat_t{}, experimentalsys.EIO
+	}
+	return d.internal.Lstat(path)
+}
+
+func (d *dstfs) Stat(path string) (sys.Stat_t, experimentalsys.Errno) {
+	if isShutdown {
+		return sys.Stat_t{}, experimentalsys.EIO
+	}
+	return d.internal.Stat(path)
+}
+
+func (d *dstfs) Mkdir(path string, perm fs.FileMode) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return d.internal.Mkdir(path, perm)
+}
+
+func (d *dstfs) Chmod(path string, perm fs.FileMode) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return d.internal.Chmod(path, perm)
+}
+
+func (d *dstfs) Rename(from, to string) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return d.internal.Rename(from, to)
+}
+
+func (d *dstfs) Rmdir(path string) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return d.internal.Rmdir(path)
+}
+
+func (d *dstfs) Unlink(path string) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return d.internal.Unlink(path)
+}
+
+func (d *dstfs) Link(oldPath, newPath string) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return d.internal.Link(oldPath, newPath)
+}
+
+func (d *dstfs) Symlink(oldPath, linkName string) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return d.internal.Symlink(oldPath, linkName)
+}
+
+func (d *dstfs) Readlink(path string) (string, experimentalsys.Errno) {
+	if isShutdown {
+		return "", experimentalsys.EIO
+	}
+	return d.internal.Readlink(path)
+}
+
+func (d *dstfs) Utimens(path string, atim, mtim int64) experimentalsys.Errno {
+	if isShutdown {
+		return experimentalsys.EIO
+	}
+	return d.internal.Utimens(path, atim, mtim)
+}

dst/vfs/wasm.go

@@ -0,0 +1,28 @@
+package vfs
+
+import (
+	"context"
+
+	"github.com/tetratelabs/wazero"
+)
+
+const wasmModuleName = "vfs"
+
+var isShutdown = false
+
+func shutdown() {
+	isShutdown = true
+}
+
+func restart() {
+	isShutdown = false
+}
+
+func MustInstantiate(ctx context.Context, r wazero.Runtime) {
+	if _, err := r.NewHostModuleBuilder(wasmModuleName).
+		NewFunctionBuilder().WithFunc(shutdown).Export("shutdown").
+		NewFunctionBuilder().WithFunc(restart).Export("restart").
+		Instantiate(ctx); err != nil {
+		panic(err)
+	}
+}

dst/vfs_hooks_nowasm.go

@@ -0,0 +1,7 @@
+//go:build !wasm
+
+package dst
+
+func vfsShutdown() {}
+
+func vfsRestart() {}

dst/vfs_hooks_wasm.go

@@ -0,0 +1,9 @@
+//go:build wasm
+
+package dst
+
+//go:wasmimport vfs shutdown
+func vfsShutdown()
+
+//go:wasmimport vfs restart
+func vfsRestart()