fix: use generators for getting receipts in w3up-client tests

packages/upload-client/src/index.js

@@ -1,5 +1,4 @@
 import * as PieceHasher from '@web3-storage/data-segment/multihash'
-import { codec as carCodec } from '@ucanto/transport/car'
 import { Storefront } from '@web3-storage/filecoin-client'
 import { ShardedDAGIndex } from '@web3-storage/blob-index'
 import * as Link from 'multiformats/link'
@@ -11,6 +10,7 @@ import * as Upload from './upload.js'
 import * as UnixFS from './unixfs.js'
 import * as CAR from './car.js'
 import { ShardingStream, defaultFileComparator } from './sharding.js'
+import { sha256 } from '@ucanto/core'
 
 export { Blob, Index, Store, Upload, UnixFS, CAR }
 export * from './sharding.js'
@@ -148,7 +148,7 @@ async function uploadBlockStream(
             // @ts-ignore Element
             const { multihash } = commitment.capabilities[0].nb.content
             // Should this be raw instead?
-            const cid = Link.create(carCodec.code, multihash)
+            const cid = Link.create(CAR.code, multihash)
             let piece
             if (pieceHasher) {
               const multihashDigest = await pieceHasher.digest(bytes)
@@ -201,12 +201,7 @@ async function uploadBlockStream(
   /* c8 ignore next */
   if (!root) throw new Error('missing root CID')
 
-  const index = ShardedDAGIndex.create(root)
-  for (const [i, shard] of shards.entries()) {
-    const slices = shardIndexes[i]
-    index.shards.set(shard.multihash, slices)
-  }
-  const indexBytes = await index.archive()
+  const indexBytes = await indexShardedDAG(root, shards, shardIndexes)
   /* c8 ignore next 3 */
   if (!indexBytes.ok) {
     throw new Error('failed to archive DAG index', { cause: indexBytes.error })
@@ -215,7 +210,7 @@ async function uploadBlockStream(
   // Store the index in the space
   const commitment = await Blob.add(conf, indexBytes.ok, options)
   const indexLink = Link.create(
-    carCodec.code,
+    CAR.code,
     // @ts-ignore Element
     commitment.capabilities[0].nb.content.multihash
   )
@@ -227,3 +222,19 @@ async function uploadBlockStream(
 
   return root
 }
+
+/**
+ * Indexes a sharded DAG
+ *
+ * @param {import('multiformats').Link} root
+ * @param {import('./types.js').CARLink[]} shards
+ * @param {Array<Map<import('./types.js').SliceDigest, import('./types.js').Position>>} shardIndexes
+ */
+export async function indexShardedDAG(root, shards, shardIndexes) {
+  const index = ShardedDAGIndex.create(root)
+  for (const [i, shard] of shards.entries()) {
+    const slices = shardIndexes[i]
+    index.shards.set(shard.multihash, slices)
+  }
+  return await index.archive()
+}

packages/upload-client/test/helpers/utils.js

@@ -25,13 +25,11 @@ export const setupGetReceipt = (contentGenFn) => {
     ) {
       return await fetch(url, options)
     }
-    console.log(options?.method, url)
 
     const taskID = url.pathname.replace('/receipt/', '')
     const issuer = await Signer.generate()
 
-    const { value: content } = contentGen.next()
-    console.log(content)
+    const { value: content } = await contentGen.next()
     const locationClaim = await Assert.location.delegate({
       issuer,
       audience: issuer,

packages/w3up-client/test/capability/blob.test.js

@@ -37,7 +37,9 @@ export const BlobClient = Test.withContext({
     const bytesHash = await sha256.digest(bytes)
     const link = createLink(car.code, bytesHash)
     const commitment = await alice.capability.blob.add(new Blob([bytes]), {
-      fetch: setupGetReceipt(link),
+      fetch: setupGetReceipt(function* () {
+        yield link
+      }),
     })
 
     // TODO we should check blobsStorage as well
@@ -87,7 +89,9 @@ export const BlobClient = Test.withContext({
     const bytesHash = await sha256.digest(bytes)
     const link = createLink(car.code, bytesHash)
     const commitment = await alice.capability.blob.add(new Blob([bytes]), {
-      fetch: setupGetReceipt(link),
+      fetch: setupGetReceipt(function* () {
+        yield link
+      }),
     })
     assert.deepEqual(
       // @ts-ignore Element
@@ -131,7 +135,9 @@ export const BlobClient = Test.withContext({
     const bytesHash = await sha256.digest(bytes)
     const link = createLink(car.code, bytesHash)
     const commitment = await alice.capability.blob.add(new Blob([bytes]), {
-      fetch: setupGetReceipt(link),
+      fetch: setupGetReceipt(function* () {
+        yield link
+      }),
     })
 
     const result = await alice.capability.blob.remove(

packages/w3up-client/test/capability/index.test.js

@@ -1,9 +1,11 @@
 import { ShardedDAGIndex } from '@web3-storage/blob-index'
+import { codec as CAR } from '@ucanto/transport/car'
 import * as Link from 'multiformats/link'
 import * as Result from '../../src/result.js'
 import { randomCAR } from '../helpers/random.js'
 import * as Test from '../test.js'
 import { setupGetReceipt } from '../helpers/utils.js'
+import { sha256 } from '@ucanto/server'
 
 export const IndexClient = Test.withContext({
   add: {
@@ -28,17 +30,21 @@ export const IndexClient = Test.withContext({
       const index = ShardedDAGIndex.create(car.cid)
       const indexBytes = Result.unwrap(await index.archive())
 
+      const bytesHash = await sha256.digest(indexBytes)
+      const link = Link.create(CAR.code, bytesHash)
       const commitment = await alice.capability.blob.add(
         new Blob([indexBytes]),
         {
-          fetch: setupGetReceipt(car.cid),
+          fetch: setupGetReceipt(function* () {
+            yield link
+          }),
         }
       )
 
       assert.ok(
         await alice.capability.index.add(
           Link.create(
-            0x0202,
+            CAR.code,
             // @ts-ignore Element
             commitment.capabilities[0].nb.content.multihash
           )

packages/w3up-client/test/capability/usage.test.js

@@ -1,16 +1,17 @@
-import { create as createLink } from 'multiformats/link'
+import * as Link from 'multiformats/link'
 import * as car from 'multiformats/codecs/raw'
 import { sha256 } from 'multiformats/hashes/sha2'
 import { AgentData } from '@web3-storage/access/agent'
 import { Client } from '../../src/client.js'
 import * as Test from '../test.js'
 import { setupGetReceipt } from '../helpers/utils.js'
+import { CID } from 'multiformats'
 
 export const UsageClient = Test.withContext({
   report: {
     'should fetch usage report': async (
       assert,
-      { connection, provisionsStorage }
+      { connection, provisionsStorage, allocationsStorage }
     ) => {
       const alice = new Client(await AgentData.create(), {
         // @ts-ignore
@@ -32,13 +33,35 @@ export const UsageClient = Test.withContext({
         consumer: space.did(),
       })
 
-      const content = new Blob(['hello world'])
-      const bytesHash = await sha256.digest(
-        new Uint8Array(await content.arrayBuffer())
-      )
-      const link = createLink(car.code, bytesHash)
+      const bytes = Buffer.from('hello world')
+      const bytesHash = await sha256.digest(bytes)
+      const link = Link.create(car.code, bytesHash)
+
+      // hardcoded the index shards so as not to reimplement sharding logic
+      // TODO there's probably a better way to do this
+      const digest = new Uint8Array([
+        18, 32, 185, 77, 39, 185, 147, 77, 62, 8, 165, 46, 82, 215, 218, 125,
+        171, 250, 196, 132, 239, 227, 122, 83, 128, 238, 144, 136, 247, 172,
+        226, 239, 205, 233,
+      ])
+      // @ts-ignore Argument
+      await allocationsStorage.insert({
+        space: space.did(),
+        blob: {
+          digest: digest,
+          size: digest.length,
+        },
+      })
+
+      const content = new Blob([bytes])
       await alice.uploadFile(content, {
-        fetch: setupGetReceipt(link),
+        fetch: setupGetReceipt(async function* () {
+          yield link
+          // hardcoded the CID so as not to reimplement index logic
+          yield CID.parse(
+            'bagbaiera34t5e64wf7evi3fagz5kspk7p5fnjjvfauyrziy6npvos2mpzdzq'
+          )
+        }),
       })
 
       const period = { from: new Date(0), to: new Date() }

packages/w3up-client/test/client.test.js

@@ -1,4 +1,6 @@
 import assert from 'assert'
+import { codec as CAR } from '@ucanto/transport/car'
+import * as Link from 'multiformats/link'
 import { create as createLink } from 'multiformats/link'
 import * as raw from 'multiformats/codecs/raw'
 import { sha256 } from 'multiformats/hashes/sha2'
@@ -10,6 +12,7 @@ import { File } from './helpers/shims.js'
 import { Client } from '../src/client.js'
 import * as Test from './test.js'
 import { setupGetReceipt } from './helpers/utils.js'
+import { indexShardedDAG } from '@web3-storage/upload-client'
 
 /** @type {Test.Suite} */
 export const testClient = {
@@ -49,7 +52,10 @@ export const testClient = {
         onShardStored: (meta) => {
           carCID = meta.cid
         },
-        fetch: setupGetReceipt(expectedCar.cid),
+        fetch: setupGetReceipt(function* () {
+          yield expectedCar.cid
+          yield expectedCar.cid
+        }),
       })
 
       assert.deepEqual(await uploadTable.exists(space.did(), dataCID), {
@@ -108,6 +114,12 @@ export const testClient = {
 
       /** @type {import('@web3-storage/upload-client/types').CARLink|undefined} */
       let carCID
+      /** @type {Array<Map<import('@web3-storage/blob-index/types').SliceDigest, import('@web3-storage/blob-index/types').Position>>} */
+      const shardIndexes = []
+      /** @type {import('@web3-storage/capabilities/types').CARLink[]} */
+      const shards = []
+      /** @type {import('@web3-storage/upload-client/types').AnyLink?} */
+      let root = null
 
       const alice = new Client(await AgentData.create(), {
         // @ts-ignore
@@ -134,8 +146,20 @@ export const testClient = {
       const dataCID = await alice.uploadDirectory(files, {
         onShardStored: (meta) => {
           carCID = meta.cid
+
+          root = meta.roots[0]
+          shards.push(meta.cid)
+          shardIndexes.push(meta.slices)
         },
-        fetch: setupGetReceipt(links[0]),
+        fetch: setupGetReceipt(async function* () {
+          yield links[0]
+          // @ts-ignore Argument
+          const index = await indexShardedDAG(root, shards, shardIndexes)
+          // @ts-ignore Argument
+          yield Link.create(CAR.code, await sha256.digest(index.ok))
+          // @ts-ignore Argument
+          yield Link.create(CAR.code, await sha256.digest(index.ok))
+        }),
       })
 
       assert.deepEqual(await uploadTable.exists(space.did(), dataCID), {
@@ -178,7 +202,10 @@ export const testClient = {
         onShardStored: (meta) => {
           carCID = meta.cid
         },
-        fetch: setupGetReceipt(car.cid),
+        fetch: setupGetReceipt(function* () {
+          yield car.cid
+          yield car.cid
+        }),
       })
 
       assert.deepEqual(await uploadTable.exists(space.did(), root), {
@@ -419,7 +446,10 @@ export const testClient = {
       })
 
       const root = await alice.uploadFile(new Blob([bytes]), {
-        fetch: setupGetReceipt(link),
+        fetch: setupGetReceipt(function* () {
+          yield link
+          yield link
+        }),
       })
 
       assert.deepEqual(await uploadTable.exists(space.did(), root), {
@@ -470,7 +500,10 @@ export const testClient = {
         })
 
         const root = await alice.uploadFile(new Blob([bytes]), {
-          fetch: setupGetReceipt(link),
+          fetch: setupGetReceipt(function* () {
+            yield link
+            yield link
+          }),
         })
 
         assert.deepEqual(await uploadTable.exists(space.did(), root), {
@@ -552,7 +585,10 @@ export const testClient = {
       })
 
       const root = await alice.uploadFile(new Blob(bytesArray), {
-        fetch: setupGetReceipt(links[0]),
+        fetch: setupGetReceipt(function* () {
+          yield links[0]
+          yield links[1]
+        }),
       })
 
       const upload = await uploadTable.get(space.did(), root)

packages/w3up-client/test/helpers/utils.js

@@ -13,14 +13,15 @@ import { Assert } from '@web3-storage/content-claims/capability'
 
 export const validateAuthorization = () => ({ ok: {} })
 
-/**
- * @param {import('multiformats').Link} content
- */
-export const setupGetReceipt = (content) => {
+// @ts-ignore Parameter
+export const setupGetReceipt = (contentGenFn) => {
+  // @type {Generator<import('multiformats').Link,void,unknown>} contentGenFn
+  const contentGen = contentGenFn()
   // @ts-ignore Parameter
   return async (url, options) => {
     // need to handle using regular fetch when not actually getting a receipt
     if (
+      options ||
       !url.pathname ||
       (url.pathname.contains && !url.pathname.contains('/receipt/'))
     ) {
@@ -30,6 +31,8 @@ export const setupGetReceipt = (content) => {
     const taskID = url.pathname.replace('/receipt/', '')
     const issuer = await Signer.generate()
 
+    const { value: content } = await contentGen.next()
+    console.log('get receipt content', content.multihash.bytes)
     const locationClaim = await Assert.location.delegate({
       issuer,
       audience: issuer,