diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index ccbf53fd1..35269e42b 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -1,5 +1,4 @@ import * as PieceHasher from '@web3-storage/data-segment/multihash' -import { codec as carCodec } from '@ucanto/transport/car' import { Storefront } from '@web3-storage/filecoin-client' import { ShardedDAGIndex } from '@web3-storage/blob-index' import * as Link from 'multiformats/link' @@ -11,6 +10,7 @@ import * as Upload from './upload.js' import * as UnixFS from './unixfs.js' import * as CAR from './car.js' import { ShardingStream, defaultFileComparator } from './sharding.js' +import { sha256 } from '@ucanto/core' export { Blob, Index, Store, Upload, UnixFS, CAR } export * from './sharding.js' @@ -148,7 +148,7 @@ async function uploadBlockStream( // @ts-ignore Element const { multihash } = commitment.capabilities[0].nb.content // Should this be raw instead? - const cid = Link.create(carCodec.code, multihash) + const cid = Link.create(CAR.code, multihash) let piece if (pieceHasher) { const multihashDigest = await pieceHasher.digest(bytes) @@ -201,12 +201,7 @@ async function uploadBlockStream( /* c8 ignore next */ if (!root) throw new Error('missing root CID') - const index = ShardedDAGIndex.create(root) - for (const [i, shard] of shards.entries()) { - const slices = shardIndexes[i] - index.shards.set(shard.multihash, slices) - } - const indexBytes = await index.archive() + const indexBytes = await indexShardedDAG(root, shards, shardIndexes) /* c8 ignore next 3 */ if (!indexBytes.ok) { throw new Error('failed to archive DAG index', { cause: indexBytes.error }) @@ -215,7 +210,7 @@ async function uploadBlockStream( // Store the index in the space const commitment = await Blob.add(conf, indexBytes.ok, options) const indexLink = Link.create( - carCodec.code, + CAR.code, // @ts-ignore Element commitment.capabilities[0].nb.content.multihash ) @@ -227,3 +222,19 @@ async function uploadBlockStream( return root } + +/** + * Indexes a sharded DAG + * + * @param {import('multiformats').Link} root + * @param {import('./types.js').CARLink[]} shards + * @param {Array>} shardIndexes + */ +export async function indexShardedDAG(root, shards, shardIndexes) { + const index = ShardedDAGIndex.create(root) + for (const [i, shard] of shards.entries()) { + const slices = shardIndexes[i] + index.shards.set(shard.multihash, slices) + } + return await index.archive() +} diff --git a/packages/upload-client/test/helpers/utils.js b/packages/upload-client/test/helpers/utils.js index 47dd836a6..1ca3c5370 100644 --- a/packages/upload-client/test/helpers/utils.js +++ b/packages/upload-client/test/helpers/utils.js @@ -25,13 +25,11 @@ export const setupGetReceipt = (contentGenFn) => { ) { return await fetch(url, options) } - console.log(options?.method, url) const taskID = url.pathname.replace('/receipt/', '') const issuer = await Signer.generate() - const { value: content } = contentGen.next() - console.log(content) + const { value: content } = await contentGen.next() const locationClaim = await Assert.location.delegate({ issuer, audience: issuer, diff --git a/packages/w3up-client/test/capability/blob.test.js b/packages/w3up-client/test/capability/blob.test.js index fbedeba6a..d88a0737c 100644 --- a/packages/w3up-client/test/capability/blob.test.js +++ b/packages/w3up-client/test/capability/blob.test.js @@ -37,7 +37,9 @@ export const BlobClient = Test.withContext({ const bytesHash = await sha256.digest(bytes) const link = createLink(car.code, bytesHash) const commitment = await alice.capability.blob.add(new Blob([bytes]), { - fetch: setupGetReceipt(link), + fetch: setupGetReceipt(function* () { + yield link + }), }) // TODO we should check blobsStorage as well @@ -87,7 +89,9 @@ export const BlobClient = Test.withContext({ const bytesHash = await sha256.digest(bytes) const link = createLink(car.code, bytesHash) const commitment = await alice.capability.blob.add(new Blob([bytes]), { - fetch: setupGetReceipt(link), + fetch: setupGetReceipt(function* () { + yield link + }), }) assert.deepEqual( // @ts-ignore Element @@ -131,7 +135,9 @@ export const BlobClient = Test.withContext({ const bytesHash = await sha256.digest(bytes) const link = createLink(car.code, bytesHash) const commitment = await alice.capability.blob.add(new Blob([bytes]), { - fetch: setupGetReceipt(link), + fetch: setupGetReceipt(function* () { + yield link + }), }) const result = await alice.capability.blob.remove( diff --git a/packages/w3up-client/test/capability/index.test.js b/packages/w3up-client/test/capability/index.test.js index 4559ecf5f..f0bd2e1c7 100644 --- a/packages/w3up-client/test/capability/index.test.js +++ b/packages/w3up-client/test/capability/index.test.js @@ -1,9 +1,11 @@ import { ShardedDAGIndex } from '@web3-storage/blob-index' +import { codec as CAR } from '@ucanto/transport/car' import * as Link from 'multiformats/link' import * as Result from '../../src/result.js' import { randomCAR } from '../helpers/random.js' import * as Test from '../test.js' import { setupGetReceipt } from '../helpers/utils.js' +import { sha256 } from '@ucanto/server' export const IndexClient = Test.withContext({ add: { @@ -28,17 +30,21 @@ export const IndexClient = Test.withContext({ const index = ShardedDAGIndex.create(car.cid) const indexBytes = Result.unwrap(await index.archive()) + const bytesHash = await sha256.digest(indexBytes) + const link = Link.create(CAR.code, bytesHash) const commitment = await alice.capability.blob.add( new Blob([indexBytes]), { - fetch: setupGetReceipt(car.cid), + fetch: setupGetReceipt(function* () { + yield link + }), } ) assert.ok( await alice.capability.index.add( Link.create( - 0x0202, + CAR.code, // @ts-ignore Element commitment.capabilities[0].nb.content.multihash ) diff --git a/packages/w3up-client/test/capability/usage.test.js b/packages/w3up-client/test/capability/usage.test.js index 7040fb70e..0381c50b7 100644 --- a/packages/w3up-client/test/capability/usage.test.js +++ b/packages/w3up-client/test/capability/usage.test.js @@ -1,16 +1,17 @@ -import { create as createLink } from 'multiformats/link' +import * as Link from 'multiformats/link' import * as car from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' import { AgentData } from '@web3-storage/access/agent' import { Client } from '../../src/client.js' import * as Test from '../test.js' import { setupGetReceipt } from '../helpers/utils.js' +import { CID } from 'multiformats' export const UsageClient = Test.withContext({ report: { 'should fetch usage report': async ( assert, - { connection, provisionsStorage } + { connection, provisionsStorage, allocationsStorage } ) => { const alice = new Client(await AgentData.create(), { // @ts-ignore @@ -32,13 +33,35 @@ export const UsageClient = Test.withContext({ consumer: space.did(), }) - const content = new Blob(['hello world']) - const bytesHash = await sha256.digest( - new Uint8Array(await content.arrayBuffer()) - ) - const link = createLink(car.code, bytesHash) + const bytes = Buffer.from('hello world') + const bytesHash = await sha256.digest(bytes) + const link = Link.create(car.code, bytesHash) + + // hardcoded the index shards so as not to reimplement sharding logic + // TODO there's probably a better way to do this + const digest = new Uint8Array([ + 18, 32, 185, 77, 39, 185, 147, 77, 62, 8, 165, 46, 82, 215, 218, 125, + 171, 250, 196, 132, 239, 227, 122, 83, 128, 238, 144, 136, 247, 172, + 226, 239, 205, 233, + ]) + // @ts-ignore Argument + await allocationsStorage.insert({ + space: space.did(), + blob: { + digest: digest, + size: digest.length, + }, + }) + + const content = new Blob([bytes]) await alice.uploadFile(content, { - fetch: setupGetReceipt(link), + fetch: setupGetReceipt(async function* () { + yield link + // hardcoded the CID so as not to reimplement index logic + yield CID.parse( + 'bagbaiera34t5e64wf7evi3fagz5kspk7p5fnjjvfauyrziy6npvos2mpzdzq' + ) + }), }) const period = { from: new Date(0), to: new Date() } diff --git a/packages/w3up-client/test/client.test.js b/packages/w3up-client/test/client.test.js index 929c5b31f..4cf0af853 100644 --- a/packages/w3up-client/test/client.test.js +++ b/packages/w3up-client/test/client.test.js @@ -1,4 +1,6 @@ import assert from 'assert' +import { codec as CAR } from '@ucanto/transport/car' +import * as Link from 'multiformats/link' import { create as createLink } from 'multiformats/link' import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' @@ -10,6 +12,7 @@ import { File } from './helpers/shims.js' import { Client } from '../src/client.js' import * as Test from './test.js' import { setupGetReceipt } from './helpers/utils.js' +import { indexShardedDAG } from '@web3-storage/upload-client' /** @type {Test.Suite} */ export const testClient = { @@ -49,7 +52,10 @@ export const testClient = { onShardStored: (meta) => { carCID = meta.cid }, - fetch: setupGetReceipt(expectedCar.cid), + fetch: setupGetReceipt(function* () { + yield expectedCar.cid + yield expectedCar.cid + }), }) assert.deepEqual(await uploadTable.exists(space.did(), dataCID), { @@ -108,6 +114,12 @@ export const testClient = { /** @type {import('@web3-storage/upload-client/types').CARLink|undefined} */ let carCID + /** @type {Array>} */ + const shardIndexes = [] + /** @type {import('@web3-storage/capabilities/types').CARLink[]} */ + const shards = [] + /** @type {import('@web3-storage/upload-client/types').AnyLink?} */ + let root = null const alice = new Client(await AgentData.create(), { // @ts-ignore @@ -134,8 +146,20 @@ export const testClient = { const dataCID = await alice.uploadDirectory(files, { onShardStored: (meta) => { carCID = meta.cid + + root = meta.roots[0] + shards.push(meta.cid) + shardIndexes.push(meta.slices) }, - fetch: setupGetReceipt(links[0]), + fetch: setupGetReceipt(async function* () { + yield links[0] + // @ts-ignore Argument + const index = await indexShardedDAG(root, shards, shardIndexes) + // @ts-ignore Argument + yield Link.create(CAR.code, await sha256.digest(index.ok)) + // @ts-ignore Argument + yield Link.create(CAR.code, await sha256.digest(index.ok)) + }), }) assert.deepEqual(await uploadTable.exists(space.did(), dataCID), { @@ -178,7 +202,10 @@ export const testClient = { onShardStored: (meta) => { carCID = meta.cid }, - fetch: setupGetReceipt(car.cid), + fetch: setupGetReceipt(function* () { + yield car.cid + yield car.cid + }), }) assert.deepEqual(await uploadTable.exists(space.did(), root), { @@ -419,7 +446,10 @@ export const testClient = { }) const root = await alice.uploadFile(new Blob([bytes]), { - fetch: setupGetReceipt(link), + fetch: setupGetReceipt(function* () { + yield link + yield link + }), }) assert.deepEqual(await uploadTable.exists(space.did(), root), { @@ -470,7 +500,10 @@ export const testClient = { }) const root = await alice.uploadFile(new Blob([bytes]), { - fetch: setupGetReceipt(link), + fetch: setupGetReceipt(function* () { + yield link + yield link + }), }) assert.deepEqual(await uploadTable.exists(space.did(), root), { @@ -552,7 +585,10 @@ export const testClient = { }) const root = await alice.uploadFile(new Blob(bytesArray), { - fetch: setupGetReceipt(links[0]), + fetch: setupGetReceipt(function* () { + yield links[0] + yield links[1] + }), }) const upload = await uploadTable.get(space.did(), root) diff --git a/packages/w3up-client/test/helpers/utils.js b/packages/w3up-client/test/helpers/utils.js index b3fbe544e..986369d33 100644 --- a/packages/w3up-client/test/helpers/utils.js +++ b/packages/w3up-client/test/helpers/utils.js @@ -13,14 +13,15 @@ import { Assert } from '@web3-storage/content-claims/capability' export const validateAuthorization = () => ({ ok: {} }) -/** - * @param {import('multiformats').Link} content - */ -export const setupGetReceipt = (content) => { +// @ts-ignore Parameter +export const setupGetReceipt = (contentGenFn) => { + // @type {Generator} contentGenFn + const contentGen = contentGenFn() // @ts-ignore Parameter return async (url, options) => { // need to handle using regular fetch when not actually getting a receipt if ( + options || !url.pathname || (url.pathname.contains && !url.pathname.contains('/receipt/')) ) { @@ -30,6 +31,8 @@ export const setupGetReceipt = (content) => { const taskID = url.pathname.replace('/receipt/', '') const issuer = await Signer.generate() + const { value: content } = await contentGen.next() + console.log('get receipt content', content.multihash.bytes) const locationClaim = await Assert.location.delegate({ issuer, audience: issuer,