diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 5729c051f..d39002949 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -5,7 +5,6 @@ import * as Blob from '@web3-storage/capabilities/blob' import * as W3sBlob from '@web3-storage/capabilities/web3.storage/blob' import * as HTTP from '@web3-storage/capabilities/http' import * as API from '../types.js' - import { createConcludeInvocation } from '../ucan/conclude.js' import { AwaitError } from './lib.js' diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index 35269e42b..e9c285236 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -10,7 +10,6 @@ import * as Upload from './upload.js' import * as UnixFS from './unixfs.js' import * as CAR from './car.js' import { ShardingStream, defaultFileComparator } from './sharding.js' -import { sha256 } from '@ucanto/core' export { Blob, Index, Store, Upload, UnixFS, CAR } export * from './sharding.js' diff --git a/packages/w3up-client/package.json b/packages/w3up-client/package.json index 416541fb8..aefa188df 100644 --- a/packages/w3up-client/package.json +++ b/packages/w3up-client/package.json @@ -133,6 +133,7 @@ }, "devDependencies": { "@ipld/car": "^5.1.1", + "@ipld/unixfs": "^2.1.1", "@types/assert": "^1.5.6", "@types/mocha": "^10.0.1", "@types/node": "^20.8.4", diff --git a/packages/w3up-client/test/capability/usage.test.js b/packages/w3up-client/test/capability/usage.test.js index 0381c50b7..ac026ca15 100644 --- a/packages/w3up-client/test/capability/usage.test.js +++ b/packages/w3up-client/test/capability/usage.test.js @@ -1,17 +1,19 @@ import * as Link from 'multiformats/link' -import * as car from 'multiformats/codecs/raw' +import * as UnixFS from '@web3-storage/upload-client/unixfs' +import { codec as CAR } from '@ucanto/transport/car' import { sha256 } from 'multiformats/hashes/sha2' import { AgentData } from '@web3-storage/access/agent' import { Client } from '../../src/client.js' import * as Test from '../test.js' import { setupGetReceipt } from '../helpers/utils.js' -import { CID } from 'multiformats' +import { indexShardedDAG } from '@web3-storage/upload-client' +import { encode } from '@web3-storage/upload-client/car' export const UsageClient = Test.withContext({ report: { 'should fetch usage report': async ( assert, - { connection, provisionsStorage, allocationsStorage } + { connection, provisionsStorage } ) => { const alice = new Client(await AgentData.create(), { // @ts-ignore @@ -33,34 +35,31 @@ export const UsageClient = Test.withContext({ consumer: space.did(), }) - const bytes = Buffer.from('hello world') - const bytesHash = await sha256.digest(bytes) - const link = Link.create(car.code, bytesHash) + /** @type {Array>} */ + const shardIndexes = [] + /** @type {import('@web3-storage/capabilities/types').CARLink[]} */ + const shards = [] + /** @type {import('@web3-storage/upload-client/types').AnyLink?} */ + let root = null - // hardcoded the index shards so as not to reimplement sharding logic - // TODO there's probably a better way to do this - const digest = new Uint8Array([ - 18, 32, 185, 77, 39, 185, 147, 77, 62, 8, 165, 46, 82, 215, 218, 125, - 171, 250, 196, 132, 239, 227, 122, 83, 128, 238, 144, 136, 247, 172, - 226, 239, 205, 233, - ]) - // @ts-ignore Argument - await allocationsStorage.insert({ - space: space.did(), - blob: { - digest: digest, - size: digest.length, - }, - }) - - const content = new Blob([bytes]) + const content = new Blob(['hello world']) await alice.uploadFile(content, { + onShardStored: (meta) => { + root = meta.roots[0] + shards.push(meta.cid) + shardIndexes.push(meta.slices) + }, fetch: setupGetReceipt(async function* () { - yield link - // hardcoded the CID so as not to reimplement index logic - yield CID.parse( - 'bagbaiera34t5e64wf7evi3fagz5kspk7p5fnjjvfauyrziy6npvos2mpzdzq' + const { cid, blocks } = await UnixFS.encodeFile(content) + const car = await encode(blocks, cid) + yield Link.create( + CAR.code, + await sha256.digest(new Uint8Array(await car.arrayBuffer())) ) + // @ts-ignore Argument + const index = await indexShardedDAG(root, shards, shardIndexes) + // @ts-ignore Argument + yield Link.create(CAR.code, await sha256.digest(index.ok)) }), }) diff --git a/packages/w3up-client/test/client.test.js b/packages/w3up-client/test/client.test.js index 4cf0af853..791fbf08e 100644 --- a/packages/w3up-client/test/client.test.js +++ b/packages/w3up-client/test/client.test.js @@ -1,8 +1,7 @@ import assert from 'assert' +import * as UnixFS from '@web3-storage/upload-client/unixfs' import { codec as CAR } from '@ucanto/transport/car' import * as Link from 'multiformats/link' -import { create as createLink } from 'multiformats/link' -import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' import { parseLink } from '@ucanto/server' import { AgentData } from '@web3-storage/access/agent' @@ -13,6 +12,7 @@ import { Client } from '../src/client.js' import * as Test from './test.js' import { setupGetReceipt } from './helpers/utils.js' import { indexShardedDAG } from '@web3-storage/upload-client' +import { encode } from '@web3-storage/upload-client/car' /** @type {Test.Suite} */ export const testClient = { @@ -102,12 +102,6 @@ export const testClient = { { connection, provisionsStorage, uploadTable } ) => { const bytesList = [await randomBytes(128), await randomBytes(32)] - const bytesHash = await Promise.all( - bytesList.map(async (bytes) => await sha256.digest(bytes)) - ) - const links = await Promise.all( - bytesHash.map((hash) => createLink(raw.code, hash)) - ) const files = bytesList.map( (bytes, index) => new File([bytes], `${index}.txt`) ) @@ -152,13 +146,16 @@ export const testClient = { shardIndexes.push(meta.slices) }, fetch: setupGetReceipt(async function* () { - yield links[0] + const { cid, blocks } = await UnixFS.encodeDirectory(files) + const car = await encode(blocks, cid) + yield Link.create( + CAR.code, + await sha256.digest(new Uint8Array(await car.arrayBuffer())) + ) // @ts-ignore Argument const index = await indexShardedDAG(root, shards, shardIndexes) // @ts-ignore Argument yield Link.create(CAR.code, await sha256.digest(index.ok)) - // @ts-ignore Argument - yield Link.create(CAR.code, await sha256.digest(index.ok)) }), }) @@ -420,8 +417,13 @@ export const testClient = { { connection, provisionsStorage, uploadTable } ) => { const bytes = await randomBytes(128) - const bytesHash = await sha256.digest(bytes) - const link = createLink(raw.code, bytesHash) + + /** @type {Array>} */ + const shardIndexes = [] + /** @type {import('@web3-storage/capabilities/types').CARLink[]} */ + const shards = [] + /** @type {import('@web3-storage/upload-client/types').AnyLink?} */ + let root = null const alice = new Client(await AgentData.create(), { // @ts-ignore @@ -445,20 +447,34 @@ export const testClient = { consumer: space.did(), }) - const root = await alice.uploadFile(new Blob([bytes]), { - fetch: setupGetReceipt(function* () { - yield link - yield link + const content = new Blob([bytes]) + const fileLink = await alice.uploadFile(content, { + onShardStored: (meta) => { + root = meta.roots[0] + shards.push(meta.cid) + shardIndexes.push(meta.slices) + }, + fetch: setupGetReceipt(async function* () { + const { cid, blocks } = await UnixFS.encodeFile(content) + const car = await encode(blocks, cid) + yield Link.create( + CAR.code, + await sha256.digest(new Uint8Array(await car.arrayBuffer())) + ) + // @ts-ignore Argument + const index = await indexShardedDAG(root, shards, shardIndexes) + // @ts-ignore Argument + yield Link.create(CAR.code, await sha256.digest(index.ok)) }), }) - assert.deepEqual(await uploadTable.exists(space.did(), root), { + assert.deepEqual(await uploadTable.exists(space.did(), fileLink), { ok: true, }) assert.deepEqual( await alice - .remove(root, { shards: true }) + .remove(fileLink, { shards: true }) .then((ok) => ({ ok: {} })) .catch((error) => { error @@ -466,7 +482,7 @@ export const testClient = { { ok: {} } ) - assert.deepEqual(await uploadTable.exists(space.did(), root), { + assert.deepEqual(await uploadTable.exists(space.did(), fileLink), { ok: false, }) }, @@ -474,8 +490,13 @@ export const testClient = { 'should remove an uploaded file from the service without its shards by default': async (assert, { connection, provisionsStorage, uploadTable }) => { const bytes = await randomBytes(128) - const bytesHash = await sha256.digest(bytes) - const link = createLink(raw.code, bytesHash) + + /** @type {Array>} */ + const shardIndexes = [] + /** @type {import('@web3-storage/capabilities/types').CARLink[]} */ + const shards = [] + /** @type {import('@web3-storage/upload-client/types').AnyLink?} */ + let root = null const alice = new Client(await AgentData.create(), { // @ts-ignore @@ -499,20 +520,34 @@ export const testClient = { consumer: space.did(), }) - const root = await alice.uploadFile(new Blob([bytes]), { - fetch: setupGetReceipt(function* () { - yield link - yield link + const content = new Blob([bytes]) + const fileLink = await alice.uploadFile(content, { + onShardStored: (meta) => { + root = meta.roots[0] + shards.push(meta.cid) + shardIndexes.push(meta.slices) + }, + fetch: setupGetReceipt(async function* () { + const { cid, blocks } = await UnixFS.encodeFile(content) + const car = await encode(blocks, cid) + yield Link.create( + CAR.code, + await sha256.digest(new Uint8Array(await car.arrayBuffer())) + ) + // @ts-ignore Argument + const index = await indexShardedDAG(root, shards, shardIndexes) + // @ts-ignore Argument + yield Link.create(CAR.code, await sha256.digest(index.ok)) }), }) - assert.deepEqual(await uploadTable.exists(space.did(), root), { + assert.deepEqual(await uploadTable.exists(space.did(), fileLink), { ok: true, }) assert.deepEqual( await alice - .remove(root) + .remove(fileLink) .then((ok) => ({ ok: {} })) .catch((error) => { error @@ -520,7 +555,7 @@ export const testClient = { { ok: {} } ) - assert.deepEqual(await uploadTable.exists(space.did(), root), { + assert.deepEqual(await uploadTable.exists(space.did(), fileLink), { ok: false, }) }, @@ -555,12 +590,13 @@ export const testClient = { { connection, provisionsStorage, uploadTable } ) => { const bytesArray = [await randomBytes(128), await randomBytes(128)] - const bytesHash = await Promise.all( - bytesArray.map(async (bytes) => await sha256.digest(bytes)) - ) - const links = await Promise.all( - bytesHash.map(async (hash) => createLink(raw.code, hash)) - ) + + /** @type {Array>} */ + const shardIndexes = [] + /** @type {import('@web3-storage/capabilities/types').CARLink[]} */ + const shards = [] + /** @type {import('@web3-storage/upload-client/types').AnyLink?} */ + let root = null const alice = new Client(await AgentData.create(), { // @ts-ignore @@ -584,14 +620,28 @@ export const testClient = { consumer: space.did(), }) - const root = await alice.uploadFile(new Blob(bytesArray), { - fetch: setupGetReceipt(function* () { - yield links[0] - yield links[1] + const content = new Blob(bytesArray) + const fileLink = await alice.uploadFile(content, { + onShardStored: (meta) => { + root = meta.roots[0] + shards.push(meta.cid) + shardIndexes.push(meta.slices) + }, + fetch: setupGetReceipt(async function* () { + const { cid, blocks } = await UnixFS.encodeFile(content) + const car = await encode(blocks, cid) + yield Link.create( + CAR.code, + await sha256.digest(new Uint8Array(await car.arrayBuffer())) + ) + // @ts-ignore Argument + const index = await indexShardedDAG(root, shards, shardIndexes) + // @ts-ignore Argument + yield Link.create(CAR.code, await sha256.digest(index.ok)) }), }) - const upload = await uploadTable.get(space.did(), root) + const upload = await uploadTable.get(space.did(), fileLink) const shard = upload.ok?.shards?.[0] if (!shard) { @@ -603,7 +653,7 @@ export const testClient = { assert.deepEqual( await alice - .remove(root, { shards: true }) + .remove(fileLink, { shards: true }) .then(() => ({ ok: {} })) .catch((error) => ({ error })), { ok: {} } diff --git a/packages/w3up-client/test/helpers/utils.js b/packages/w3up-client/test/helpers/utils.js index 986369d33..9a476b344 100644 --- a/packages/w3up-client/test/helpers/utils.js +++ b/packages/w3up-client/test/helpers/utils.js @@ -32,7 +32,6 @@ export const setupGetReceipt = (contentGenFn) => { const issuer = await Signer.generate() const { value: content } = await contentGen.next() - console.log('get receipt content', content.multihash.bytes) const locationClaim = await Assert.location.delegate({ issuer, audience: issuer, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2cd050028..13b95462d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -641,6 +641,9 @@ importers: '@ipld/car': specifier: ^5.1.1 version: 5.3.0 + '@ipld/unixfs': + specifier: ^2.1.1 + version: 2.2.0 '@types/assert': specifier: ^1.5.6 version: 1.5.10 @@ -3411,7 +3414,6 @@ packages: multiformats: 11.0.2 protobufjs: 7.2.6 rabin-rs: 2.1.0 - dev: false /@istanbuljs/schema@0.1.3: resolution: {integrity: sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==} @@ -3559,7 +3561,6 @@ packages: dependencies: '@multiformats/murmur3': 2.1.8 murmurhash3js-revisited: 3.0.0 - dev: false /@pnpm/config.env-replace@1.1.0: resolution: {integrity: sha512-htyl8TWnKL7K/ESFa1oW2UB5lVDxuF5DpM7tBi6Hu2LNL3mWkIzNLG6N4zoCUP1lCKNxWy/3iu8mS8MvToGd6w==} @@ -4648,7 +4649,6 @@ packages: /actor@2.3.1: resolution: {integrity: sha512-ST/3wnvcP2tKDXnum7nLCLXm+/rsf8vPocXH2Fre6D8FQwNkGDd4JEitBlXj007VQJfiGYRQvXqwOBZVi+JtRg==} - dev: false /address@1.2.2: resolution: {integrity: sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA==} @@ -10977,7 +10977,6 @@ packages: /rabin-rs@2.1.0: resolution: {integrity: sha512-5y72gAXPzIBsAMHcpxZP8eMDuDT98qMP1BqSDHRbHkJJXEgWIN1lA47LxUqzsK6jknOJtgfkQr9v+7qMlFDm6g==} - dev: false /randombytes@2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==}