fix: pass w3up-client tests

packages/upload-api/src/blob/add.js

@@ -5,7 +5,6 @@ import * as Blob from '@web3-storage/capabilities/blob'
 import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob'
 import * as HTTP from '@web3-storage/capabilities/http'
 import * as API from '../types.js'
-
 import { createConcludeInvocation } from '../ucan/conclude.js'
 import { AwaitError } from './lib.js'
 

packages/upload-client/src/index.js

@@ -10,7 +10,6 @@ import * as Upload from './upload.js'
 import * as UnixFS from './unixfs.js'
 import * as CAR from './car.js'
 import { ShardingStream, defaultFileComparator } from './sharding.js'
-import { sha256 } from '@ucanto/core'
 
 export { Blob, Index, Store, Upload, UnixFS, CAR }
 export * from './sharding.js'

packages/w3up-client/package.json

@@ -133,6 +133,7 @@
   },
   "devDependencies": {
     "@ipld/car": "^5.1.1",
+    "@ipld/unixfs": "^2.1.1",
     "@types/assert": "^1.5.6",
     "@types/mocha": "^10.0.1",
     "@types/node": "^20.8.4",

packages/w3up-client/test/capability/usage.test.js

@@ -1,17 +1,19 @@
 import * as Link from 'multiformats/link'
-import * as car from 'multiformats/codecs/raw'
+import * as UnixFS from '@web3-storage/upload-client/unixfs'
+import { codec as CAR } from '@ucanto/transport/car'
 import { sha256 } from 'multiformats/hashes/sha2'
 import { AgentData } from '@web3-storage/access/agent'
 import { Client } from '../../src/client.js'
 import * as Test from '../test.js'
 import { setupGetReceipt } from '../helpers/utils.js'
-import { CID } from 'multiformats'
+import { indexShardedDAG } from '@web3-storage/upload-client'
+import { encode } from '@web3-storage/upload-client/car'
 
 export const UsageClient = Test.withContext({
   report: {
     'should fetch usage report': async (
       assert,
-      { connection, provisionsStorage, allocationsStorage }
+      { connection, provisionsStorage }
     ) => {
       const alice = new Client(await AgentData.create(), {
         // @ts-ignore
@@ -33,34 +35,31 @@ export const UsageClient = Test.withContext({
         consumer: space.did(),
       })
 
-      const bytes = Buffer.from('hello world')
-      const bytesHash = await sha256.digest(bytes)
-      const link = Link.create(car.code, bytesHash)
+      /** @type {Array<Map<import('@web3-storage/blob-index/types').SliceDigest, import('@web3-storage/blob-index/types').Position>>} */
+      const shardIndexes = []
+      /** @type {import('@web3-storage/capabilities/types').CARLink[]} */
+      const shards = []
+      /** @type {import('@web3-storage/upload-client/types').AnyLink?} */
+      let root = null
 
-      // hardcoded the index shards so as not to reimplement sharding logic
-      // TODO there's probably a better way to do this
-      const digest = new Uint8Array([
-        18, 32, 185, 77, 39, 185, 147, 77, 62, 8, 165, 46, 82, 215, 218, 125,
-        171, 250, 196, 132, 239, 227, 122, 83, 128, 238, 144, 136, 247, 172,
-        226, 239, 205, 233,
-      ])
-      // @ts-ignore Argument
-      await allocationsStorage.insert({
-        space: space.did(),
-        blob: {
-          digest: digest,
-          size: digest.length,
-        },
-      })
-
-      const content = new Blob([bytes])
+      const content = new Blob(['hello world'])
       await alice.uploadFile(content, {
+        onShardStored: (meta) => {
+          root = meta.roots[0]
+          shards.push(meta.cid)
+          shardIndexes.push(meta.slices)
+        },
         fetch: setupGetReceipt(async function* () {
-          yield link
-          // hardcoded the CID so as not to reimplement index logic
-          yield CID.parse(
-            'bagbaiera34t5e64wf7evi3fagz5kspk7p5fnjjvfauyrziy6npvos2mpzdzq'
+          const { cid, blocks } = await UnixFS.encodeFile(content)
+          const car = await encode(blocks, cid)
+          yield Link.create(
+            CAR.code,
+            await sha256.digest(new Uint8Array(await car.arrayBuffer()))
           )
+          // @ts-ignore Argument
+          const index = await indexShardedDAG(root, shards, shardIndexes)
+          // @ts-ignore Argument
+          yield Link.create(CAR.code, await sha256.digest(index.ok))
         }),
       })
 

packages/w3up-client/test/client.test.js

@@ -1,8 +1,7 @@
 import assert from 'assert'
+import * as UnixFS from '@web3-storage/upload-client/unixfs'
 import { codec as CAR } from '@ucanto/transport/car'
 import * as Link from 'multiformats/link'
-import { create as createLink } from 'multiformats/link'
-import * as raw from 'multiformats/codecs/raw'
 import { sha256 } from 'multiformats/hashes/sha2'
 import { parseLink } from '@ucanto/server'
 import { AgentData } from '@web3-storage/access/agent'
@@ -13,6 +12,7 @@ import { Client } from '../src/client.js'
 import * as Test from './test.js'
 import { setupGetReceipt } from './helpers/utils.js'
 import { indexShardedDAG } from '@web3-storage/upload-client'
+import { encode } from '@web3-storage/upload-client/car'
 
 /** @type {Test.Suite} */
 export const testClient = {
@@ -102,12 +102,6 @@ export const testClient = {
       { connection, provisionsStorage, uploadTable }
     ) => {
       const bytesList = [await randomBytes(128), await randomBytes(32)]
-      const bytesHash = await Promise.all(
-        bytesList.map(async (bytes) => await sha256.digest(bytes))
-      )
-      const links = await Promise.all(
-        bytesHash.map((hash) => createLink(raw.code, hash))
-      )
       const files = bytesList.map(
         (bytes, index) => new File([bytes], `${index}.txt`)
       )
@@ -152,13 +146,16 @@ export const testClient = {
           shardIndexes.push(meta.slices)
         },
         fetch: setupGetReceipt(async function* () {
-          yield links[0]
+          const { cid, blocks } = await UnixFS.encodeDirectory(files)
+          const car = await encode(blocks, cid)
+          yield Link.create(
+            CAR.code,
+            await sha256.digest(new Uint8Array(await car.arrayBuffer()))
+          )
           // @ts-ignore Argument
           const index = await indexShardedDAG(root, shards, shardIndexes)
           // @ts-ignore Argument
           yield Link.create(CAR.code, await sha256.digest(index.ok))
-          // @ts-ignore Argument
-          yield Link.create(CAR.code, await sha256.digest(index.ok))
         }),
       })
 
@@ -420,8 +417,13 @@ export const testClient = {
       { connection, provisionsStorage, uploadTable }
     ) => {
       const bytes = await randomBytes(128)
-      const bytesHash = await sha256.digest(bytes)
-      const link = createLink(raw.code, bytesHash)
+
+      /** @type {Array<Map<import('@web3-storage/blob-index/types').SliceDigest, import('@web3-storage/blob-index/types').Position>>} */
+      const shardIndexes = []
+      /** @type {import('@web3-storage/capabilities/types').CARLink[]} */
+      const shards = []
+      /** @type {import('@web3-storage/upload-client/types').AnyLink?} */
+      let root = null
 
       const alice = new Client(await AgentData.create(), {
         // @ts-ignore
@@ -445,37 +447,56 @@ export const testClient = {
         consumer: space.did(),
       })
 
-      const root = await alice.uploadFile(new Blob([bytes]), {
-        fetch: setupGetReceipt(function* () {
-          yield link
-          yield link
+      const content = new Blob([bytes])
+      const fileLink = await alice.uploadFile(content, {
+        onShardStored: (meta) => {
+          root = meta.roots[0]
+          shards.push(meta.cid)
+          shardIndexes.push(meta.slices)
+        },
+        fetch: setupGetReceipt(async function* () {
+          const { cid, blocks } = await UnixFS.encodeFile(content)
+          const car = await encode(blocks, cid)
+          yield Link.create(
+            CAR.code,
+            await sha256.digest(new Uint8Array(await car.arrayBuffer()))
+          )
+          // @ts-ignore Argument
+          const index = await indexShardedDAG(root, shards, shardIndexes)
+          // @ts-ignore Argument
+          yield Link.create(CAR.code, await sha256.digest(index.ok))
         }),
       })
 
-      assert.deepEqual(await uploadTable.exists(space.did(), root), {
+      assert.deepEqual(await uploadTable.exists(space.did(), fileLink), {
         ok: true,
       })
 
       assert.deepEqual(
         await alice
-          .remove(root, { shards: true })
+          .remove(fileLink, { shards: true })
           .then((ok) => ({ ok: {} }))
           .catch((error) => {
             error
           }),
         { ok: {} }
       )
 
-      assert.deepEqual(await uploadTable.exists(space.did(), root), {
+      assert.deepEqual(await uploadTable.exists(space.did(), fileLink), {
         ok: false,
       })
     },
 
     'should remove an uploaded file from the service without its shards by default':
       async (assert, { connection, provisionsStorage, uploadTable }) => {
         const bytes = await randomBytes(128)
-        const bytesHash = await sha256.digest(bytes)
-        const link = createLink(raw.code, bytesHash)
+
+        /** @type {Array<Map<import('@web3-storage/blob-index/types').SliceDigest, import('@web3-storage/blob-index/types').Position>>} */
+        const shardIndexes = []
+        /** @type {import('@web3-storage/capabilities/types').CARLink[]} */
+        const shards = []
+        /** @type {import('@web3-storage/upload-client/types').AnyLink?} */
+        let root = null
 
         const alice = new Client(await AgentData.create(), {
           // @ts-ignore
@@ -499,28 +520,42 @@ export const testClient = {
           consumer: space.did(),
         })
 
-        const root = await alice.uploadFile(new Blob([bytes]), {
-          fetch: setupGetReceipt(function* () {
-            yield link
-            yield link
+        const content = new Blob([bytes])
+        const fileLink = await alice.uploadFile(content, {
+          onShardStored: (meta) => {
+            root = meta.roots[0]
+            shards.push(meta.cid)
+            shardIndexes.push(meta.slices)
+          },
+          fetch: setupGetReceipt(async function* () {
+            const { cid, blocks } = await UnixFS.encodeFile(content)
+            const car = await encode(blocks, cid)
+            yield Link.create(
+              CAR.code,
+              await sha256.digest(new Uint8Array(await car.arrayBuffer()))
+            )
+            // @ts-ignore Argument
+            const index = await indexShardedDAG(root, shards, shardIndexes)
+            // @ts-ignore Argument
+            yield Link.create(CAR.code, await sha256.digest(index.ok))
           }),
         })
 
-        assert.deepEqual(await uploadTable.exists(space.did(), root), {
+        assert.deepEqual(await uploadTable.exists(space.did(), fileLink), {
           ok: true,
         })
 
         assert.deepEqual(
           await alice
-            .remove(root)
+            .remove(fileLink)
             .then((ok) => ({ ok: {} }))
             .catch((error) => {
               error
             }),
           { ok: {} }
         )
 
-        assert.deepEqual(await uploadTable.exists(space.did(), root), {
+        assert.deepEqual(await uploadTable.exists(space.did(), fileLink), {
           ok: false,
         })
       },
@@ -555,12 +590,13 @@ export const testClient = {
       { connection, provisionsStorage, uploadTable }
     ) => {
       const bytesArray = [await randomBytes(128), await randomBytes(128)]
-      const bytesHash = await Promise.all(
-        bytesArray.map(async (bytes) => await sha256.digest(bytes))
-      )
-      const links = await Promise.all(
-        bytesHash.map(async (hash) => createLink(raw.code, hash))
-      )
+
+      /** @type {Array<Map<import('@web3-storage/blob-index/types').SliceDigest, import('@web3-storage/blob-index/types').Position>>} */
+      const shardIndexes = []
+      /** @type {import('@web3-storage/capabilities/types').CARLink[]} */
+      const shards = []
+      /** @type {import('@web3-storage/upload-client/types').AnyLink?} */
+      let root = null
 
       const alice = new Client(await AgentData.create(), {
         // @ts-ignore
@@ -584,14 +620,28 @@ export const testClient = {
         consumer: space.did(),
       })
 
-      const root = await alice.uploadFile(new Blob(bytesArray), {
-        fetch: setupGetReceipt(function* () {
-          yield links[0]
-          yield links[1]
+      const content = new Blob(bytesArray)
+      const fileLink = await alice.uploadFile(content, {
+        onShardStored: (meta) => {
+          root = meta.roots[0]
+          shards.push(meta.cid)
+          shardIndexes.push(meta.slices)
+        },
+        fetch: setupGetReceipt(async function* () {
+          const { cid, blocks } = await UnixFS.encodeFile(content)
+          const car = await encode(blocks, cid)
+          yield Link.create(
+            CAR.code,
+            await sha256.digest(new Uint8Array(await car.arrayBuffer()))
+          )
+          // @ts-ignore Argument
+          const index = await indexShardedDAG(root, shards, shardIndexes)
+          // @ts-ignore Argument
+          yield Link.create(CAR.code, await sha256.digest(index.ok))
         }),
       })
 
-      const upload = await uploadTable.get(space.did(), root)
+      const upload = await uploadTable.get(space.did(), fileLink)
 
       const shard = upload.ok?.shards?.[0]
       if (!shard) {
@@ -603,7 +653,7 @@ export const testClient = {
 
       assert.deepEqual(
         await alice
-          .remove(root, { shards: true })
+          .remove(fileLink, { shards: true })
           .then(() => ({ ok: {} }))
           .catch((error) => ({ error })),
         { ok: {} }

packages/w3up-client/test/helpers/utils.js

@@ -32,7 +32,6 @@ export const setupGetReceipt = (contentGenFn) => {
     const issuer = await Signer.generate()
 
     const { value: content } = await contentGen.next()
-    console.log('get receipt content', content.multihash.bytes)
     const locationClaim = await Assert.location.delegate({
       issuer,
       audience: issuer,