Merge pull request #37 from input-output-hk/wip-write-buffer

Write buffer

lsm-tree.cabal

@@ -33,8 +33,11 @@ library
   exposed-modules:
     Data.Map.Range
     Database.LSMTree.Common
+    Database.LSMTree.Internal.BlobRef
     Database.LSMTree.Internal.Integration
     Database.LSMTree.Internal.KMerge
+    Database.LSMTree.Internal.Monoidal
+    Database.LSMTree.Internal.Normal
     Database.LSMTree.Internal.Run
     Database.LSMTree.Internal.Run.BloomFilter
     Database.LSMTree.Internal.Run.Index.Compact

src/Database/LSMTree/Internal/BlobRef.hs

@@ -0,0 +1,13 @@
+module Database.LSMTree.Internal.BlobRef (
+    BlobRef (..),
+) where
+
+-- | A reference to an on-disk blob.
+--
+-- The blob can be retrieved based on the reference.
+--
+-- Blob comes from the acronym __Binary Large OBject (BLOB)__ and in many
+-- database implementations refers to binary data that is larger than usual
+-- values and is handled specially. In our context we will allow optionally a
+-- blob associated with each value in the table.
+data BlobRef blob = BlobRef

src/Database/LSMTree/Internal/Monoidal.hs

@@ -0,0 +1,28 @@
+module Database.LSMTree.Internal.Monoidal (
+    LookupResult (..),
+    RangeLookupResult (..),
+    Update (..),
+) where
+
+-- | Result of a single point lookup.
+data LookupResult k v =
+    NotFound      !k
+  | Found         !k !v
+  deriving (Eq, Show)
+
+
+-- | A result for one point in a range lookup.
+data RangeLookupResult k v =
+    FoundInRange         !k !v
+  deriving (Eq, Show)
+
+-- | Monoidal tables support insert, delete and monoidal upsert operations.
+--
+-- An __update__ is a term that groups all types of table-manipulating
+-- operations, like inserts, deletes and mupserts.
+data Update v =
+    Insert !v
+  | Delete
+    -- | TODO: should be given a more suitable name.
+  | Mupsert !v
+  deriving (Show, Eq)

src/Database/LSMTree/Internal/Normal.hs

@@ -0,0 +1,28 @@
+{-# LANGUAGE DeriveTraversable #-}
+module Database.LSMTree.Internal.Normal (
+    LookupResult (..),
+    RangeLookupResult (..),
+    Update (..),
+) where
+
+-- | Result of a single point lookup.
+data LookupResult k v blobref =
+    NotFound      !k
+  | Found         !k !v
+  | FoundWithBlob !k !v !blobref
+  deriving (Eq, Show, Functor, Foldable, Traversable)
+
+-- | A result for one point in a range lookup.
+data RangeLookupResult k v blobref =
+    FoundInRange         !k !v
+  | FoundInRangeWithBlob !k !v !blobref
+  deriving (Eq, Show, Functor, Foldable, Traversable)
+
+-- | Normal tables support insert and delete operations.
+--
+-- An __update__ is a term that groups all types of table-manipulating
+-- operations, like inserts and deletes.
+data Update v blob =
+    Insert !v !(Maybe blob)
+  | Delete
+  deriving (Show, Eq)

src/Database/LSMTree/Internal/WriteBuffer.hs

@@ -1,3 +1,5 @@
+{-# LANGUAGE DeriveTraversable   #-}
+{-# LANGUAGE ScopedTypeVariables #-}
 -- | The in-memory LSM level 0.
 --
 -- === TODO
@@ -18,4 +20,108 @@
 -- The above list is a sketch. Functionality may move around, and the list is
 -- not exhaustive.
 --
-module Database.LSMTree.Internal.WriteBuffer () where
+module Database.LSMTree.Internal.WriteBuffer (
+    WriteBuffer,
+    emptyWriteBuffer,
+    addEntryMonoidal,
+    addEntryNormal,
+    lookups,
+    rangeLookups,
+) where
+
+import qualified Data.Map.Range as Map.R
+import           Data.Map.Strict (Map)
+import qualified Data.Map.Strict as Map
+import           Database.LSMTree.Common (Range (..))
+import qualified Database.LSMTree.Internal.Monoidal as Monoidal
+import qualified Database.LSMTree.Internal.Normal as Normal
+
+{-------------------------------------------------------------------------------
+  Writebuffer type
+-------------------------------------------------------------------------------}
+
+data Entry v blobref
+    = Insert !v
+    | InsertWithBlob !v !blobref
+    | Mupdate !v
+    | Delete
+  deriving (Show, Functor, Foldable, Traversable)
+
+newtype WriteBuffer k v blobref = WB (Map k (Entry v blobref))
+
+emptyWriteBuffer :: WriteBuffer k v blobref
+emptyWriteBuffer = WB Map.empty
+
+{-------------------------------------------------------------------------------
+  Updates
+-------------------------------------------------------------------------------}
+
+combine :: (v -> v -> v) -> Entry v blobref -> Entry v blobref -> Entry v blobref
+combine _ e@Delete            _                       = e
+combine _ e@Insert {}         _                       = e
+combine _ e@InsertWithBlob {} _                       = e
+combine _   (Mupdate u)       Delete                  = Insert u
+combine f   (Mupdate u)       (Insert v)              = Insert (f u v)
+combine f   (Mupdate u)       (InsertWithBlob v blob) = InsertWithBlob (f u v) blob
+combine f   (Mupdate u)       (Mupdate v)             = Mupdate (f u v)
+
+addEntryMonoidal :: Ord k
+  => (v -> v -> v) -- ^ merge function
+  -> k -> Monoidal.Update v -> WriteBuffer k v blobref -> WriteBuffer k v blobref
+addEntryMonoidal f k e (WB wb) = WB (Map.insertWith (combine f) k (g e) wb) where
+  g :: Monoidal.Update v -> Entry v blobref
+  g (Monoidal.Insert v)  = Insert v
+  g (Monoidal.Mupsert v) = Mupdate v
+  g (Monoidal.Delete)    = Delete
+
+addEntryNormal :: Ord k
+  => k -> Normal.Update v blobref -> WriteBuffer k v blobref -> WriteBuffer k v blobref
+addEntryNormal k e (WB wb) = WB (Map.insert k (g e) wb) where
+  g :: Normal.Update v blobref -> Entry v blobref
+  g (Normal.Insert v Nothing)     = Insert v
+  g (Normal.Insert v (Just bref)) = InsertWithBlob v bref
+  g Normal.Delete                 = Delete
+
+{-------------------------------------------------------------------------------
+  Querying
+-------------------------------------------------------------------------------}
+
+-- We return 'Entry', so it can be properly combined with the lookups in other
+-- runs.
+--
+-- Note: the entry may be 'Delete'.
+--
+lookups :: forall k v blobref. Ord k
+  => WriteBuffer k v blobref
+  -> [k]
+  -> [(k, Maybe (Entry v blobref))]
+lookups (WB m) = fmap f where
+   f :: k -> (k, Maybe (Entry v blobref))
+   f k = (k, Map.lookup k m)
+
+{-------------------------------------------------------------------------------
+  RangeQueries
+-------------------------------------------------------------------------------}
+
+-- | We return 'Entry' instead of either @angeLookupResult@,
+-- so we can properly combine lookup results.
+--
+-- Note: 'Delete's are not filtered out.
+--
+rangeLookups :: Ord k
+  => WriteBuffer k v blobref
+  -> Range k
+  -> [(k, Entry v blobref)]
+rangeLookups (WB m) r =
+    [ (k, e)
+    | let (lb, ub) = convertRange r
+    , (k, e) <- Map.R.rangeLookup lb ub m
+    ]
+
+convertRange :: Range k -> (Map.R.Bound k, Map.R.Bound k)
+convertRange (FromToExcluding lb ub) =
+    ( Map.R.Bound lb Map.R.Inclusive
+    , Map.R.Bound ub Map.R.Exclusive )
+convertRange (FromToIncluding lb ub) =
+    ( Map.R.Bound lb Map.R.Inclusive
+    , Map.R.Bound ub Map.R.Inclusive )

src/Database/LSMTree/Monoidal.hs

@@ -86,6 +86,7 @@ import           Database.LSMTree.Common (IOLike, Range (..), Session,
                      SnapshotName, SomeSerialisationConstraint,
                      SomeUpdateConstraint, closeSession, deleteSnapshot,
                      listSnapshots, openSession)
+import           Database.LSMTree.Internal.Monoidal
 
 -- $resource-management
 -- See "Database.LSMTree.Normal#g:resource"
@@ -158,12 +159,6 @@ close = undefined
   Table querying and updates
 -------------------------------------------------------------------------------}
 
--- | Result of a single point lookup.
-data LookupResult k v =
-    NotFound      !k
-  | Found         !k !v
-  deriving (Eq, Show)
-
 -- | Perform a batch of lookups.
 --
 -- Lookups can be performed concurrently from multiple Haskell threads.
@@ -178,11 +173,6 @@ lookups ::
   -> m [LookupResult k v]
 lookups = undefined
 
--- | A result for one point in a range lookup.
-data RangeLookupResult k v =
-    FoundInRange         !k !v
-  deriving (Eq, Show)
-
 -- | Perform a range lookup.
 --
 -- Range lookups can be performed concurrently from multiple Haskell threads.
@@ -197,17 +187,6 @@ rangeLookup ::
   -> m [RangeLookupResult k v]
 rangeLookup = undefined
 
--- | Monoidal tables support insert, delete and monoidal upsert operations.
---
--- An __update__ is a term that groups all types of table-manipulating
--- operations, like inserts, deletes and mupserts.
-data Update v =
-    Insert !v
-  | Delete
-    -- | TODO: should be given a more suitable name.
-  | Mupsert !v
-  deriving (Show, Eq)
-
 -- | Perform a mixed batch of inserts, deletes and monoidal upserts.
 --
 -- Updates can be performed concurrently from multiple Haskell threads.

src/Database/LSMTree/Normal.hs

@@ -1,4 +1,3 @@
-{-# LANGUAGE DeriveTraversable        #-}
 {-# LANGUAGE StandaloneDeriving       #-}
 {-# LANGUAGE StandaloneKindSignatures #-}
 {-# LANGUAGE TupleSections            #-}
@@ -83,6 +82,8 @@ import           Data.Word (Word64)
 import           Database.LSMTree.Common (IOLike, Range (..), Session,
                      SnapshotName, SomeSerialisationConstraint, closeSession,
                      deleteSnapshot, listSnapshots, openSession)
+import           Database.LSMTree.Internal.BlobRef
+import           Database.LSMTree.Internal.Normal
 
 -- $resource-management
 -- Table handles use resources and as such need to be managed. In particular
@@ -212,13 +213,6 @@ close = undefined
   Table querying and updates
 -------------------------------------------------------------------------------}
 
--- | Result of a single point lookup.
-data LookupResult k v blobref =
-    NotFound      !k
-  | Found         !k !v
-  | FoundWithBlob !k !v !blobref
-  deriving (Eq, Show, Functor, Foldable, Traversable)
-
 -- | Perform a batch of lookups.
 --
 -- Lookups can be performed concurrently from multiple Haskell threads.
@@ -229,12 +223,6 @@ lookups ::
   -> m [LookupResult k v (BlobRef blob)]
 lookups = undefined
 
--- | A result for one point in a range lookup.
-data RangeLookupResult k v blobref =
-    FoundInRange         !k !v
-  | FoundInRangeWithBlob !k !v !blobref
-  deriving (Eq, Show, Functor, Foldable, Traversable)
-
 -- | Perform a range lookup.
 --
 -- Range lookups can be performed concurrently from multiple Haskell threads.
@@ -245,15 +233,6 @@ rangeLookup ::
   -> m [RangeLookupResult k v (BlobRef blob)]
 rangeLookup = undefined
 
--- | Normal tables support insert and delete operations.
---
--- An __update__ is a term that groups all types of table-manipulating
--- operations, like inserts and deletes.
-data Update v blob =
-    Insert !v !(Maybe blob)
-  | Delete
-  deriving (Show, Eq)
-
 -- | Perform a mixed batch of inserts and deletes.
 --
 -- Updates can be performed concurrently from multiple Haskell threads.
@@ -296,16 +275,6 @@ deletes ::
   -> m ()
 deletes = updates . fmap (,Delete)
 
--- | A reference to an on-disk blob.
---
--- The blob can be retrieved based on the reference.
---
--- Blob comes from the acronym __Binary Large OBject (BLOB)__ and in many
--- database implementations refers to binary data that is larger than usual
--- values and is handled specially. In our context we will allow optionally a
--- blob associated with each value in the table.
-data BlobRef blob = BlobRef
-
 -- | Perform a batch of blob retrievals.
 --
 -- This is a separate step from 'lookups' and 'rangeLookups'. The result of a