Move orderbook fetching to src folder and expose it (#619)

@anxolin suggested to reuse the fetch orderbook logic in the dex-price-estimator. In order to do this we need to move this logic into `/src` and expose it on `index.js` and `index.d.ts`.

### Test Plan

local price estimation script still works, and `yalc publish` allows using this method in dex-price-estimator

package.json

@@ -118,6 +118,7 @@
     "typechain": "^1.0.5",
     "typechain-target-web3-v1": "^1.0.4",
     "typescript": "^3.8.3",
+    "web3-core": "^1.2.6",
     "yargs": "^15.3.1"
   }
 }

scripts/stablex/ensure_owl_liquidity.js

@@ -1,7 +1,8 @@
 const BatchExchange = artifacts.require("BatchExchange")
 
 const BN = require("bn.js")
-const { maxUint32, sendLiquidityOrders, getOrdersPaginated } = require("./utilities")
+const { maxUint32, sendLiquidityOrders } = require("./utilities")
+const { getOrdersPaginated } = require("../../src/onchain_reading")
 
 const MINIMAL_LIQUIDITY_FOR_OWL = new BN(10).pow(new BN(17))
 const SELL_ORDER_AMOUNT_OWL = new BN(10).pow(new BN(18)).mul(new BN(5))
@@ -41,7 +42,7 @@ module.exports = async (callback) => {
     // Get the order data
     const numberOfToken = await instance.numTokens.call()
     const batchId = (await instance.getCurrentBatchId()).toNumber()
-    let orders = await getOrdersPaginated(instance, 100)
+    let orders = await getOrdersPaginated(instance.contract, 100)
     orders = orders.filter((order) => order.validUntil >= batchId && order.validFrom <= batchId)
 
     // Ensure OWL-liquidity is given

scripts/stablex/get_auction_elements.js

@@ -18,7 +18,7 @@ const argv = require("yargs")
   })
   .version(false).argv
 
-const { getOrdersPaginated } = require("./utilities")
+const { getOrdersPaginated } = require("../../src/onchain_reading")
 
 const COLORS = {
   NONE: "\x1b[0m",
@@ -83,7 +83,7 @@ const printOrder = function (order, currentBatchId) {
 module.exports = async (callback) => {
   try {
     const instance = await BatchExchange.deployed()
-    let auctionElementsDecoded = await getOrdersPaginated(instance, argv.pageSize)
+    let auctionElementsDecoded = await getOrdersPaginated(instance.contract, argv.pageSize)
 
     const batchId = (await instance.getCurrentBatchId()).toNumber()
     if (!argv.expired) {

scripts/stablex/transitive_orderbook.js

@@ -1,5 +1,5 @@
 const BatchExchangeViewer = artifacts.require("BatchExchangeViewer")
-const { getOpenOrdersPaginated } = require("./utilities.js")
+const { getOpenOrdersPaginated } = require("../../src/onchain_reading.js")
 const BN = require("bn.js")
 
 const { Orderbook, Offer, transitiveOrderbook } = require("../../typescript/common/orderbook.js")
@@ -41,11 +41,13 @@ const addItemToOrderbooks = function (orderbooks, item) {
 }
 
 const getAllOrderbooks = async function (instance, pageSize) {
-  const elements = await getOpenOrdersPaginated(instance, pageSize)
   const orderbooks = new Map()
-  elements.forEach((item) => {
-    addItemToOrderbooks(orderbooks, item)
-  })
+  for await (const page of getOpenOrdersPaginated(instance.contract, pageSize)) {
+    console.log("Fetched Page")
+    page.forEach((item) => {
+      addItemToOrderbooks(orderbooks, item)
+    })
+  }
   return orderbooks
 }
 

scripts/stablex/utilities.js

@@ -99,48 +99,6 @@ const closeAuction = async (instance, web3Provider = web3) => {
   await waitForNSeconds(time_remaining + 1, web3Provider)
 }
 
-const getOrdersPaginated = async (instance, pageSize) => {
-  const { decodeOrdersBN } = require("../../src/encoding")
-  let orders = []
-  let currentUser = "0x0000000000000000000000000000000000000000"
-  let currentOffSet = 0
-  let lastPageSize = pageSize
-  while (lastPageSize == pageSize) {
-    const page = decodeOrdersBN(await instance.getEncodedUsersPaginated(currentUser, currentOffSet, pageSize))
-    orders = orders.concat(page)
-    for (const index in page) {
-      if (page[index].user != currentUser) {
-        currentUser = page[index].user
-        currentOffSet = 0
-      }
-      currentOffSet += 1
-    }
-    lastPageSize = page.length
-  }
-  return orders
-}
-
-const getOpenOrdersPaginated = async function (instance, pageSize) {
-  const { decodeOrdersBN } = require("../../src/encoding")
-  let orders = []
-  let nextPageUser = "0x0000000000000000000000000000000000000000"
-  let nextPageUserOffset = 0
-  let lastPageSize = pageSize
-
-  while (lastPageSize == pageSize) {
-    console.log("Fetching Page")
-    const page = await instance.getOpenOrderBookPaginated([], nextPageUser, nextPageUserOffset, pageSize)
-    const elements = decodeOrdersBN(page.elements)
-    orders = orders.concat(elements)
-
-    //Update page info
-    lastPageSize = elements.length
-    nextPageUser = page.nextPageUser
-    nextPageUserOffset = page.nextPageUserOffset
-  }
-  return orders
-}
-
 const sendLiquidityOrders = async function (
   instance,
   tokenIds,
@@ -281,8 +239,6 @@ module.exports = {
   token_list_url,
   fetchTokenInfo,
   sendLiquidityOrders,
-  getOrdersPaginated,
-  getOpenOrdersPaginated,
   maxUint32,
   setAllowances,
   mintOwl,

src/index.d.ts

@@ -1,4 +1,5 @@
 import BN from "bn.js";
+import {BatchExchangeViewer} from "../build/types/BatchExchangeViewer";
 export * from "./orderbook";
 export * from "./fraction";
 
@@ -82,3 +83,8 @@ export interface OrderBN {
 
 export declare function decodeOrders(bytes: string): Order[];
 export declare function decodeOrdersBN(bytes: string): OrderBN[];
+
+export declare function getOpenOrdersPaginated(
+  contract: BatchExchangeViewer,
+  pageSize: number
+): AsyncIterable<OrderBN[]>;

src/index.js

@@ -13,4 +13,5 @@ module.exports = {
   ...require("../typescript/common/fraction.js"),
   ...require("../typescript/common/orderbook.js"),
   ...require("./encoding.js"),
+  ...require("./onchain_reading.js"),
 }

src/onchain_reading.js

@@ -0,0 +1,53 @@
+const { decodeOrdersBN } = require("./encoding")
+
+/**
+ * Returns an iterator yielding an item for each page of order in the orderbook that is currently being collected.
+ * @param {BatchExchangeViewer} contract to query from
+ * @param {number} pageSize the number of items to fetch per page
+ */
+const getOpenOrdersPaginated = async function* (contract, pageSize) {
+  let nextPageUser = "0x0000000000000000000000000000000000000000"
+  let nextPageUserOffset = 0
+  let lastPageSize = pageSize
+
+  while (lastPageSize == pageSize) {
+    const page = await contract.methods.getOpenOrderBookPaginated([], nextPageUser, nextPageUserOffset, pageSize).call()
+    const elements = decodeOrdersBN(page.elements)
+    yield elements
+
+    //Update page info
+    lastPageSize = elements.length
+    nextPageUser = page.nextPageUser
+    nextPageUserOffset = page.nextPageUserOffset
+  }
+}
+
+/**
+ * Returns all orders in the orderbook.
+ * @param {BatchExchange} contract to query from
+ * @param {number} pageSize the number of items to fetch per page
+ */
+const getOrdersPaginated = async (contract, pageSize) => {
+  let orders = []
+  let currentUser = "0x0000000000000000000000000000000000000000"
+  let currentOffSet = 0
+  let lastPageSize = pageSize
+  while (lastPageSize == pageSize) {
+    const page = decodeOrdersBN(await contract.methods.getEncodedUsersPaginated(currentUser, currentOffSet, pageSize).call())
+    orders = orders.concat(page)
+    for (const index in page) {
+      if (page[index].user != currentUser) {
+        currentUser = page[index].user
+        currentOffSet = 0
+      }
+      currentOffSet += 1
+    }
+    lastPageSize = page.length
+  }
+  return orders
+}
+
+module.exports = {
+  getOpenOrdersPaginated,
+  getOrdersPaginated,
+}

test/stablex/send_liquidity_orders.js

@@ -1,4 +1,5 @@
-const { closeAuction, sendLiquidityOrders, getOrdersPaginated } = require("../../scripts/stablex/utilities.js")
+const { closeAuction, sendLiquidityOrders } = require("../../scripts/stablex/utilities.js")
+const { getOrdersPaginated } = require("../../src/onchain_reading")
 const BatchExchange = artifacts.require("BatchExchange")
 const MockContract = artifacts.require("MockContract")
 const IdToAddressBiMap = artifacts.require("IdToAddressBiMap")
@@ -37,7 +38,7 @@ contract("Liquidity order placement test", async (accounts) => {
       const tokenIds = [1, 3, 5]
       await sendLiquidityOrders(batchExchange, tokenIds, PRICE_FOR_LIQUIDITY_PROVISION, SELL_ORDER_AMOUNT_OWL, artifacts)
 
-      const orders = await getOrdersPaginated(batchExchange, 100)
+      const orders = await getOrdersPaginated(batchExchange.contract, 100)
       assert.deepEqual(
         orders.map((o) => o.buyToken),
         tokenIds

tsconfig.json

@@ -53,7 +53,7 @@
     /* Source Map Options */
     // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
     // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
-    "inlineSourceMap": false,                 /* Emit a single file with source maps instead of having a separate file. */
+    "inlineSourceMap": false /* Emit a single file with source maps instead of having a separate file. */,
     // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
 
     /* Experimental Options */
@@ -63,5 +63,5 @@
     /* Advanced Options */
     "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
   },
-  "include": ["src/**/*.ts", "./node_modules/bn.js-typings/index.d.ts"]
+  "files": ["src/orderbook.ts", "src/fraction.ts"]
 }

yarn.lock

@@ -8882,7 +8882,7 @@ [email protected]:
     web3-core-requestmanager "1.2.2"
     web3-utils "1.2.2"
 
-[email protected]:
+[email protected], web3-core@^1.2.6:
   version "1.2.6"
   resolved "https://registry.yarnpkg.com/web3-core/-/web3-core-1.2.6.tgz#bb42a1d7ae49a7258460f0d95ddb00906f59ef92"
   integrity sha512-y/QNBFtr5cIR8vxebnotbjWJpOnO8LDYEAzZjeRRUJh2ijmhjoYk7dSNx9ExgC0UCfNFRoNCa9dGRu/GAxwRlw==