From 1cdc74e650d7b5fbc69266100fdeed20f8263fda Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 2 Aug 2023 15:31:24 +0200 Subject: [PATCH] feat: w3filecoin new client and api --- .github/release-please-config.json | 4 +- .github/release-please-manifest.json | 4 +- .../{aggregate-api.yml => filecoin-api.yml} | 14 +- ...gregate-client.yml => filecoin-client.yml} | 14 +- packages/aggregate-api/CHANGELOG.md | 18 - packages/aggregate-api/src/aggregate.js | 13 - packages/aggregate-api/src/aggregate/get.js | 38 -- packages/aggregate-api/src/aggregate/offer.js | 131 ------ packages/aggregate-api/src/lib.js | 46 --- packages/aggregate-api/src/offer.js | 11 - packages/aggregate-api/src/offer/arrange.js | 40 -- packages/aggregate-api/src/types.ts | 89 ---- packages/aggregate-api/test/aggregate.js | 386 ------------------ .../test/context/aggregate-store.js | 36 -- .../aggregate-api/test/context/offer-store.js | 24 -- packages/aggregate-api/test/lib.js | 8 - packages/aggregate-client/CHANGELOG.md | 17 - packages/aggregate-client/README.md | 75 ---- packages/aggregate-client/src/aggregate.js | 70 ---- packages/aggregate-client/src/index.js | 1 - packages/aggregate-client/src/service.js | 16 - packages/aggregate-client/src/types.ts | 69 ---- .../aggregate-client/test/aggregate.test.js | 155 ------- packages/capabilities/src/aggregate.js | 102 ----- packages/capabilities/src/filecoin.js | 187 +++++++++ packages/capabilities/src/index.js | 13 +- packages/capabilities/src/offer.js | 37 -- packages/capabilities/src/types.ts | 68 +-- .../package.json | 41 +- packages/filecoin-api/src/aggregator.js | 128 ++++++ packages/filecoin-api/src/broker.js | 162 ++++++++ packages/filecoin-api/src/errors.js | 61 +++ packages/filecoin-api/src/lib.js | 3 + packages/filecoin-api/src/storefront.js | 127 ++++++ .../src/types.js | 0 packages/filecoin-api/src/types.ts | 170 ++++++++ packages/filecoin-api/test/aggregator.spec.js | 48 +++ packages/filecoin-api/test/broker.spec.js | 44 ++ packages/filecoin-api/test/context/queue.js | 27 ++ packages/filecoin-api/test/context/store.js | 27 ++ packages/filecoin-api/test/lib.js | 12 + .../filecoin-api/test/services/aggregator.js | 149 +++++++ packages/filecoin-api/test/services/broker.js | 172 ++++++++ .../filecoin-api/test/services/storefront.js | 157 +++++++ .../test/storefront.spec.js} | 25 +- .../test/utils.js | 1 + .../tsconfig.json | 0 packages/filecoin-client/README.md | 37 ++ .../package.json | 28 +- packages/filecoin-client/src/aggregator.js | 52 +++ packages/filecoin-client/src/broker.js | 58 +++ packages/filecoin-client/src/chain.js | 49 +++ packages/filecoin-client/src/index.js | 4 + packages/filecoin-client/src/service.js | 28 ++ packages/filecoin-client/src/storefront.js | 52 +++ packages/filecoin-client/src/types.js | 1 + packages/filecoin-client/src/types.ts | 82 ++++ .../filecoin-client/test/aggregator.test.js | 210 ++++++++++ packages/filecoin-client/test/broker.test.js | 247 +++++++++++ packages/filecoin-client/test/chain.test.js | 88 ++++ .../test/fixtures.js | 0 .../test/helpers/block.js | 0 .../test/helpers/car.js | 0 .../filecoin-client/test/helpers/errors.js | 21 + .../test/helpers/mocks.js | 23 +- .../test/helpers/random.js | 1 + .../filecoin-client/test/storefront.test.js | 209 ++++++++++ .../tsconfig.json | 0 pnpm-lock.yaml | 119 ++++++ tsconfig.json | 1 + 70 files changed, 2877 insertions(+), 1473 deletions(-) rename .github/workflows/{aggregate-api.yml => filecoin-api.yml} (70%) rename .github/workflows/{aggregate-client.yml => filecoin-client.yml} (63%) delete mode 100644 packages/aggregate-api/CHANGELOG.md delete mode 100644 packages/aggregate-api/src/aggregate.js delete mode 100644 packages/aggregate-api/src/aggregate/get.js delete mode 100644 packages/aggregate-api/src/aggregate/offer.js delete mode 100644 packages/aggregate-api/src/lib.js delete mode 100644 packages/aggregate-api/src/offer.js delete mode 100644 packages/aggregate-api/src/offer/arrange.js delete mode 100644 packages/aggregate-api/src/types.ts delete mode 100644 packages/aggregate-api/test/aggregate.js delete mode 100644 packages/aggregate-api/test/context/aggregate-store.js delete mode 100644 packages/aggregate-api/test/context/offer-store.js delete mode 100644 packages/aggregate-api/test/lib.js delete mode 100644 packages/aggregate-client/CHANGELOG.md delete mode 100644 packages/aggregate-client/README.md delete mode 100644 packages/aggregate-client/src/aggregate.js delete mode 100644 packages/aggregate-client/src/index.js delete mode 100644 packages/aggregate-client/src/service.js delete mode 100644 packages/aggregate-client/src/types.ts delete mode 100644 packages/aggregate-client/test/aggregate.test.js delete mode 100644 packages/capabilities/src/aggregate.js create mode 100644 packages/capabilities/src/filecoin.js delete mode 100644 packages/capabilities/src/offer.js rename packages/{aggregate-api => filecoin-api}/package.json (73%) create mode 100644 packages/filecoin-api/src/aggregator.js create mode 100644 packages/filecoin-api/src/broker.js create mode 100644 packages/filecoin-api/src/errors.js create mode 100644 packages/filecoin-api/src/lib.js create mode 100644 packages/filecoin-api/src/storefront.js rename packages/{aggregate-api => filecoin-api}/src/types.js (100%) create mode 100644 packages/filecoin-api/src/types.ts create mode 100644 packages/filecoin-api/test/aggregator.spec.js create mode 100644 packages/filecoin-api/test/broker.spec.js create mode 100644 packages/filecoin-api/test/context/queue.js create mode 100644 packages/filecoin-api/test/context/store.js create mode 100644 packages/filecoin-api/test/lib.js create mode 100644 packages/filecoin-api/test/services/aggregator.js create mode 100644 packages/filecoin-api/test/services/broker.js create mode 100644 packages/filecoin-api/test/services/storefront.js rename packages/{aggregate-api/test/aggregate.spec.js => filecoin-api/test/storefront.spec.js} (59%) rename packages/{aggregate-api => filecoin-api}/test/utils.js (98%) rename packages/{aggregate-api => filecoin-api}/tsconfig.json (100%) create mode 100644 packages/filecoin-client/README.md rename packages/{aggregate-client => filecoin-client}/package.json (80%) create mode 100644 packages/filecoin-client/src/aggregator.js create mode 100644 packages/filecoin-client/src/broker.js create mode 100644 packages/filecoin-client/src/chain.js create mode 100644 packages/filecoin-client/src/index.js create mode 100644 packages/filecoin-client/src/service.js create mode 100644 packages/filecoin-client/src/storefront.js create mode 100644 packages/filecoin-client/src/types.js create mode 100644 packages/filecoin-client/src/types.ts create mode 100644 packages/filecoin-client/test/aggregator.test.js create mode 100644 packages/filecoin-client/test/broker.test.js create mode 100644 packages/filecoin-client/test/chain.test.js rename packages/{aggregate-client => filecoin-client}/test/fixtures.js (100%) rename packages/{aggregate-client => filecoin-client}/test/helpers/block.js (100%) rename packages/{aggregate-client => filecoin-client}/test/helpers/car.js (100%) create mode 100644 packages/filecoin-client/test/helpers/errors.js rename packages/{aggregate-client => filecoin-client}/test/helpers/mocks.js (51%) rename packages/{aggregate-client => filecoin-client}/test/helpers/random.js (98%) create mode 100644 packages/filecoin-client/test/storefront.test.js rename packages/{aggregate-client => filecoin-client}/tsconfig.json (100%) diff --git a/.github/release-please-config.json b/.github/release-please-config.json index 748c101c9..0e20078dd 100644 --- a/.github/release-please-config.json +++ b/.github/release-please-config.json @@ -3,8 +3,8 @@ "bootstrap-sha": "c918ffc59eafa01fbc63d5df11ba621cc1888c64", "packages": { "packages/access-client": {}, - "packages/aggregate-api": {}, - "packages/aggregate-client": {}, + "packages/filecoin-api": {}, + "packages/filecoin-client": {}, "packages/capabilities": {}, "packages/did-mailto": {}, "packages/upload-api": {}, diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index 4bb411cfc..66bf6f183 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1,7 +1,7 @@ { "packages/access-client": "15.0.0", - "packages/aggregate-api": "1.0.0", - "packages/aggregate-client": "1.0.0", + "packages/filecoin-api": "1.0.0", + "packages/filecoin-client": "1.0.0", "packages/capabilities": "7.0.0", "packages/upload-api": "4.1.0", "packages/upload-client": "9.1.1", diff --git a/.github/workflows/aggregate-api.yml b/.github/workflows/filecoin-api.yml similarity index 70% rename from .github/workflows/aggregate-api.yml rename to .github/workflows/filecoin-api.yml index ec83b63d6..e80d6ba38 100644 --- a/.github/workflows/aggregate-api.yml +++ b/.github/workflows/filecoin-api.yml @@ -1,4 +1,4 @@ -name: Aggregate API +name: Filecoin API env: CI: true FORCE_COLOR: 1 @@ -8,14 +8,14 @@ on: branches: - main paths: - - 'packages/aggregate-api/**' - - '.github/workflows/aggregate-api.yml' + - 'packages/filecoin-api/**' + - '.github/workflows/filecoin-api.yml' - 'pnpm-lock.yaml' - '.env.tpl' pull_request: paths: - - 'packages/aggregate-api/**' - - '.github/workflows/aggregate-api.yml' + - 'packages/filecoin-api/**' + - '.github/workflows/filecoin-api.yml' - 'pnpm-lock.yaml' - '.env.tpl' jobs: @@ -44,6 +44,6 @@ jobs: pnpm run --if-present build - name: Lint - run: pnpm -r --filter @web3-storage/aggregate-api run lint + run: pnpm -r --filter @web3-storage/filecoin-api run lint - name: Test - run: pnpm -r --filter @web3-storage/aggregate-api run test + run: pnpm -r --filter @web3-storage/filecoin-api run test diff --git a/.github/workflows/aggregate-client.yml b/.github/workflows/filecoin-client.yml similarity index 63% rename from .github/workflows/aggregate-client.yml rename to .github/workflows/filecoin-client.yml index 60df7c6c8..23bdf05d0 100644 --- a/.github/workflows/aggregate-client.yml +++ b/.github/workflows/filecoin-client.yml @@ -1,4 +1,4 @@ -name: Aggregate Client +name: Filecoin Client env: CI: true FORCE_COLOR: 1 @@ -7,13 +7,13 @@ on: branches: - main paths: - - 'packages/aggregate-client/**' - - '.github/workflows/aggregate-client.yml' + - 'packages/filecoin-client/**' + - '.github/workflows/filecoin-client.yml' - 'pnpm-lock.yaml' pull_request: paths: - - 'packages/aggregate-client/**' - - '.github/workflows/aggregate-client.yml' + - 'packages/filecoin-client/**' + - '.github/workflows/filecoin-client.yml' - 'pnpm-lock.yaml' jobs: test: @@ -34,5 +34,5 @@ jobs: cache: 'pnpm' - run: pnpm install - run: pnpm run build - - run: pnpm -r --filter @web3-storage/aggregate-client run lint - - run: pnpm -r --filter @web3-storage/aggregate-client run test + - run: pnpm -r --filter @web3-storage/filecoin-client run lint + - run: pnpm -r --filter @web3-storage/filecoin-client run test diff --git a/packages/aggregate-api/CHANGELOG.md b/packages/aggregate-api/CHANGELOG.md deleted file mode 100644 index 9d0d4046e..000000000 --- a/packages/aggregate-api/CHANGELOG.md +++ /dev/null @@ -1,18 +0,0 @@ -# Changelog - -## 1.0.0 (2023-07-11) - - -### ⚠ BREAKING CHANGES - -* aggregate capabilities now have different nb properties and aggregate client api was simplified - -### Features - -* w3 aggregate protocol client and api implementation ([#787](https://github.com/web3-storage/w3up/issues/787)) ([b58069d](https://github.com/web3-storage/w3up/commit/b58069d7960efe09283f3b23fed77515b62d4639)) - - -### Bug Fixes - -* aggregate api test link comparison type ([#816](https://github.com/web3-storage/w3up/issues/816)) ([81bdf1c](https://github.com/web3-storage/w3up/commit/81bdf1c08f7a99b55a8ff2d79af78bf161322737)) -* update aggregate spec in client and api ([#824](https://github.com/web3-storage/w3up/issues/824)) ([ebefd88](https://github.com/web3-storage/w3up/commit/ebefd889a028f325690370db8043c7b9e9fdf7bb)) diff --git a/packages/aggregate-api/src/aggregate.js b/packages/aggregate-api/src/aggregate.js deleted file mode 100644 index 43e7c733d..000000000 --- a/packages/aggregate-api/src/aggregate.js +++ /dev/null @@ -1,13 +0,0 @@ -import { provide as aggregateOfferProvider } from './aggregate/offer.js' -import { provide as aggregateGetProvider } from './aggregate/get.js' -import * as API from './types.js' - -/** - * @param {API.AggregateServiceContext} context - */ -export function createService(context) { - return { - offer: aggregateOfferProvider(context), - get: aggregateGetProvider(context), - } -} diff --git a/packages/aggregate-api/src/aggregate/get.js b/packages/aggregate-api/src/aggregate/get.js deleted file mode 100644 index 1804dd71b..000000000 --- a/packages/aggregate-api/src/aggregate/get.js +++ /dev/null @@ -1,38 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Aggregate from '@web3-storage/capabilities/aggregate' -import * as API from '../types.js' - -/** - * @param {API.AggregateServiceContext} context - */ -export const provide = (context) => - Server.provide(Aggregate.get, (input) => claim(input, context)) - -/** - * @param {API.Input} input - * @param {API.AggregateServiceContext} context - * @returns {Promise>} - */ -export const claim = async ({ capability }, { aggregateStore }) => { - const subject = capability.nb.subject - - const aggregateArrangedResult = await aggregateStore.get(subject) - if (!aggregateArrangedResult) { - return { - error: new AggregateNotFound( - `aggregate not found for subject: ${subject}` - ), - } - } - return { - ok: { - deals: aggregateArrangedResult, - }, - } -} - -class AggregateNotFound extends Server.Failure { - get name() { - return /** @type {const} */ ('AggregateNotFound') - } -} diff --git a/packages/aggregate-api/src/aggregate/offer.js b/packages/aggregate-api/src/aggregate/offer.js deleted file mode 100644 index 340f7750c..000000000 --- a/packages/aggregate-api/src/aggregate/offer.js +++ /dev/null @@ -1,131 +0,0 @@ -import * as Server from '@ucanto/server' -import { CBOR } from '@ucanto/core' -import { Node, Piece, Aggregate as AggregateBuilder } from '@web3-storage/data-segment' -import * as Aggregate from '@web3-storage/capabilities/aggregate' -import * as Offer from '@web3-storage/capabilities/offer' -import * as API from '../types.js' - -// 16 GiB -export const MIN_SIZE = Piece.PaddedSize.from(2n ** 34n) -// 32 GiB -export const MAX_SIZE = Piece.PaddedSize.from(2n ** 35n) - -/** - * @param {API.AggregateServiceContext} context - */ -export const provide = (context) => - Server.provideAdvanced({ - capability: Aggregate.offer, - handler: (input) => claim(input, context), - }) - -/** - * @param {API.Input} input - * @param {API.AggregateServiceContext} context - * @returns {Promise | API.UcantoInterface.JoinBuilder>} - */ -export const claim = async ( - { capability, invocation, context }, - { offerStore } -) => { - // Get offer block - const offerCid = capability.nb.offer - const piece = capability.nb.piece - const offers = getOfferBlock(offerCid, invocation.iterateIPLDBlocks()) - - if (!offers) { - return { - error: new AggregateOfferBlockNotFoundError( - `missing offer block in invocation: ${offerCid.toString()}` - ), - } - } - - // Validate offer content - const aggregateLeafs = 2n ** BigInt(piece.height) - const aggregateSize = aggregateLeafs * BigInt(Node.Size) - - if (aggregateSize < MIN_SIZE) { - return { - error: new AggregateOfferInvalidSizeError( - `offer under size, offered: ${aggregateSize}, minimum: ${MIN_SIZE}` - ), - } - } else if (aggregateSize > MAX_SIZE) { - return { - error: new AggregateOfferInvalidSizeError( - `offer over size, offered: ${aggregateSize}, maximum: ${MAX_SIZE}` - ), - } - } - - // Validate commP of commPs - const aggregateBuild = AggregateBuilder.build({ - size: aggregateSize, - pieces: offers.map(offer => Piece.fromJSON({ - height: offer.height, - link: { '/': offer.link.toString() } - })) - }) - if (!aggregateBuild.link.equals(piece.link)) { - return { - error: new AggregateOfferInvalidSizeError( - `aggregate piece CID mismatch, specified: ${piece.link}, computed: ${aggregateBuild.link}` - ), - } - } else if (aggregateBuild.height !== piece.height) { - return { - error: new AggregateOfferInvalidSizeError( - `aggregate height mismatch, specified: ${piece.height}, computed: ${aggregateBuild.height}` - ), - } - } - - // Create effect for receipt - const fx = await Offer.arrange - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: piece.link, - }, - }) - .delegate() - - // Write offer to store - await offerStore.queue({ piece, offers }) - - return Server.ok({ - status: 'queued', - }).join(fx.link()) -} - -/** - * @param {Server.API.Link} offerCid - * @param {IterableIterator>} blockIterator - */ -function getOfferBlock(offerCid, blockIterator) { - for (const block of blockIterator) { - if (block.cid.equals(offerCid)) { - const decoded = - /** @type {import('@web3-storage/data-segment').PieceView[]} */ ( - CBOR.decode(block.bytes) - ) - return decoded - // TODO: Validate with schema - } - } -} - -class AggregateOfferInvalidSizeError extends Server.Failure { - get name() { - return /** @type {const} */ ('AggregateOfferInvalidSize') - } -} - -class AggregateOfferBlockNotFoundError extends Server.Failure { - get name() { - return /** @type {const} */ ('AggregateOfferBlockNotFound') - } -} diff --git a/packages/aggregate-api/src/lib.js b/packages/aggregate-api/src/lib.js deleted file mode 100644 index 16cfef454..000000000 --- a/packages/aggregate-api/src/lib.js +++ /dev/null @@ -1,46 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Client from '@ucanto/client' -import * as Types from './types.js' -import * as CAR from '@ucanto/transport/car' -import { createService as createAggregateService } from './aggregate.js' -import { createService as createOfferService } from './offer.js' -export * from './types.js' - -/** - * @param {Types.UcantoServerContext} options - */ -export const createServer = ({ id, codec = CAR.inbound, ...context }) => - Server.create({ - id, - codec: CAR.inbound, - service: createService(context), - catch: (error) => context.errorReporter.catch(error), - }) - -/** - * @param {Types.ServiceContext} context - * @returns {Types.Service} - */ -export const createService = (context) => ({ - aggregate: createAggregateService(context), - offer: createOfferService(context), -}) - -/** - * @param {object} options - * @param {Types.UcantoInterface.Principal} options.id - * @param {Types.UcantoInterface.Transport.Channel} options.channel - * @param {Types.UcantoInterface.OutboundCodec} [options.codec] - */ -export const connect = ({ id, channel, codec = CAR.outbound }) => - Client.connect({ - id, - channel, - codec, - }) - -export { - createService as createUploadService, - createServer as createUploadServer, - connect as createUploadClient, -} diff --git a/packages/aggregate-api/src/offer.js b/packages/aggregate-api/src/offer.js deleted file mode 100644 index 07da0c321..000000000 --- a/packages/aggregate-api/src/offer.js +++ /dev/null @@ -1,11 +0,0 @@ -import { provide as offerArrangeProvider } from './offer/arrange.js' -import * as API from './types.js' - -/** - * @param {API.OfferServiceContext} context - */ -export function createService(context) { - return { - arrange: offerArrangeProvider(context), - } -} diff --git a/packages/aggregate-api/src/offer/arrange.js b/packages/aggregate-api/src/offer/arrange.js deleted file mode 100644 index dd552cf21..000000000 --- a/packages/aggregate-api/src/offer/arrange.js +++ /dev/null @@ -1,40 +0,0 @@ -import * as Server from '@ucanto/server' -import * as Offer from '@web3-storage/capabilities/offer' -import * as API from '../types.js' - -/** - * @param {API.OfferServiceContext} context - */ -export const provide = (context) => - Server.provide(Offer.arrange, (input) => claim(input, context)) - -/** - * @param {API.Input} input - * @param {API.OfferServiceContext} context - * @returns {Promise>} - */ -export const claim = async ({ capability }, { arrangedOfferStore }) => { - const pieceLink = capability.nb.pieceLink - - const status = await arrangedOfferStore.get(pieceLink) - - if (!status) { - return { - error: new OfferArrangeNotFound( - `arranged offer not found for piece: ${pieceLink}` - ), - } - } - - return { - ok: { - status, - }, - } -} - -class OfferArrangeNotFound extends Server.Failure { - get name() { - return /** @type {const} */ ('OfferArrangeNotFound') - } -} diff --git a/packages/aggregate-api/src/types.ts b/packages/aggregate-api/src/types.ts deleted file mode 100644 index 5dca45e4c..000000000 --- a/packages/aggregate-api/src/types.ts +++ /dev/null @@ -1,89 +0,0 @@ -import type { - HandlerExecutionError, - Signer, - InboundCodec, - CapabilityParser, - ParsedCapability, - InferInvokedCapability, - Match, - Link, -} from '@ucanto/interface' -import type { ProviderInput } from '@ucanto/server' - -import type { PieceLink, PieceView } from '@web3-storage/data-segment' -export * from '@web3-storage/aggregate-client/types' - -export * from '@web3-storage/capabilities/types' -export * as UcantoInterface from '@ucanto/interface' - -export interface AggregateServiceContext { - aggregateStore: AggregateStore - offerStore: OfferStore -} - -export interface OfferServiceContext { - arrangedOfferStore: ArrangedOfferStore -} - -export interface ServiceContext - extends AggregateServiceContext, - OfferServiceContext {} - -export interface ArrangedOfferStore { - get: (pieceLink: PieceLink) => Promise -} - -export interface OfferStore { - queue: (aggregateOffer: OfferToQueue) => Promise -} - -export interface OfferToQueue { - piece: PieceView - offers: PieceView[] -} - -export interface AggregateStore { - get: (pieceLink: PieceLink) => Promise -} - -export interface UcantoServerContext extends ServiceContext { - id: Signer - codec?: InboundCodec - errorReporter: ErrorReporter -} - -export interface ErrorReporter { - catch: (error: HandlerExecutionError) => void -} - -export interface Assert { - equal: ( - actual: Actual, - expected: Expected, - message?: string - ) => unknown - deepEqual: ( - actual: Actual, - expected: Expected, - message?: string - ) => unknown - ok: (actual: Actual, message?: string) => unknown -} - -export interface AggregateStoreBackend { - put: ( - pieceLink: Link, - aggregateInfo: unknown - ) => Promise -} - -export interface UcantoServerContextTest extends UcantoServerContext { - // to enable tests to insert data in aggregateStore memory db - aggregateStoreBackend: AggregateStoreBackend -} - -export type Test = (assert: Assert, context: UcantoServerContextTest) => unknown -export type Tests = Record - -export type Input>> = - ProviderInput & ParsedCapability> diff --git a/packages/aggregate-api/test/aggregate.js b/packages/aggregate-api/test/aggregate.js deleted file mode 100644 index d7c1936b1..000000000 --- a/packages/aggregate-api/test/aggregate.js +++ /dev/null @@ -1,386 +0,0 @@ -import { Aggregate, Offer } from '@web3-storage/capabilities' -import { Piece } from '@web3-storage/data-segment' - -import { CBOR, parseLink } from '@ucanto/core' -import * as Signer from '@ucanto/principal/ed25519' - -import { MIN_SIZE, MAX_SIZE } from '../src/aggregate/offer.js' -import * as API from '../src/types.js' -import { randomAggregate } from './utils.js' -import { createServer, connect } from '../src/lib.js' - -/** - * @type {API.Tests} - */ -export const test = { - // aggregate/offer tests - 'aggregate/offer inserts valid offer into bucket': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: aggregate, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - if (aggregateOffer.out.error) { - throw new Error('invocation failed', { cause: aggregateOffer.out.error }) - } - assert.ok(aggregateOffer.out.ok) - assert.deepEqual(aggregateOffer.out.ok.status, 'queued') - - // Validate effect in receipt - const fx = await Offer.arrange - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: aggregate.link, - }, - }) - .delegate() - - assert.ok(aggregateOffer.fx.join) - assert.ok(fx.link().equals(aggregateOffer.fx.join?.link())) - }, - 'aggregate/offer fails when offer block is not attached': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: aggregate, - }, - }) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - assert.deepEqual( - aggregateOffer.out.error?.message, - `missing offer block in invocation: ${block.cid.toString()}` - ) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - 'aggregate/offer fails when size is not enough for offer': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const badHeight = 3 - const size = Piece.PaddedSize.fromHeight(badHeight) - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: { - ...aggregate, - height: badHeight, - }, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - // TODO: compute size - assert.deepEqual( - aggregateOffer.out.error?.message, - `offer under size, offered: ${Number(size)}, minimum: ${MIN_SIZE}` - ) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - 'aggregate/offer fails when size is above limit for offer': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const badHeight = 31 - const size = Piece.PaddedSize.fromHeight(badHeight) - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: { - ...aggregate, - height: badHeight, - }, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - assert.deepEqual( - aggregateOffer.out.error?.message, - `offer over size, offered: ${size}, maximum: ${MAX_SIZE}` - ) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - 'aggregate/offer fails when provided height is different than computed': - async (assert, context) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const badHeight = 29 - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: { - link: aggregate.link, - height: badHeight, - }, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - 'aggregate/offer fails when provided piece CID is different than computed': - async (assert, context) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - const badLink = - /** @type {import('@web3-storage/data-segment').PieceLink} */ ( - parseLink( - 'baga6ea4seaqm2u43527zehkqqcpyyopgsw2c4mapyy2vbqzqouqtzhxtacueeki' - ) - ) - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: { - link: badLink, - height: aggregate.height, - }, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - assert.ok(aggregateOffer.out.error) - assert.deepEqual( - aggregateOffer.out.error?.message, - `aggregate piece CID mismatch, specified: ${badLink}, computed: ${aggregate.link}` - ) - - // Validate effect in receipt does not exist - assert.ok(!aggregateOffer.fx.join) - }, - // offer/arrange tests - 'aggregate/arrange can be invoked after aggregate/offer': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 128) - // TODO: Inflate size for testing - - const block = await CBOR.write(pieces) - const aggregateOfferInvocation = Aggregate.offer.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - offer: block.cid, - piece: aggregate, - }, - }) - aggregateOfferInvocation.attach(block) - - const aggregateOffer = await aggregateOfferInvocation.execute(connection) - if (aggregateOffer.out.error) { - throw new Error('invocation failed', { cause: aggregateOffer.out.error }) - } - assert.ok(aggregateOffer.out.ok) - - // Validate effect in receipt - const fx = await Offer.arrange - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: aggregate.link, - }, - }) - .delegate() - - const offerArrangeInvocation = Offer.arrange.invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: aggregate.link, - }, - }) - - const offerArrange = await offerArrangeInvocation.execute(connection) - if (offerArrange.out.error) { - throw new Error('invocation failed', { cause: offerArrange.out.error }) - } - assert.ok(offerArrange.out.ok) - assert.ok(offerArrange.ran.link().equals(fx.link())) - }, - // aggregate/get tests - 'aggregate/get fails when requested aggregate does not exist': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const subject = - /** @type {import('@web3-storage/data-segment').PieceLink} */ ( - parseLink( - 'baga6ea4seaqm2u43527zehkqqcpyyopgsw2c4mapyy2vbqzqouqtzhxtacueeki' - ) - ) - const aggregateGetInvocation = Aggregate.get.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - subject, - }, - }) - - const aggregateGet = await aggregateGetInvocation.execute(connection) - assert.ok(aggregateGet.out.error) - }, - // aggregate/get tests - 'aggregate/get returns known deals for given commitment proof': async ( - assert, - context - ) => { - const { storeFront } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const subject = - /** @type {import('@web3-storage/data-segment').PieceLink} */ ( - parseLink( - 'baga6ea4seaqm2u43527zehkqqcpyyopgsw2c4mapyy2vbqzqouqtzhxtacueeki' - ) - ) - const deal = { - status: 'done', - } - await context.aggregateStoreBackend.put(subject, deal) - - const aggregateGetInvocation = Aggregate.get.invoke({ - issuer: storeFront, - audience: connection.id, - with: storeFront.did(), - nb: { - subject, - }, - }) - - const aggregateGet = await aggregateGetInvocation.execute(connection) - if (aggregateGet.out.error) { - throw new Error('invocation failed', { cause: aggregateGet.out.error }) - } - assert.equal(aggregateGet.out.ok.deals.length, 1) - assert.deepEqual(aggregateGet.out.ok.deals[0], deal) - }, -} - -async function getServiceContext() { - const storeFront = await Signer.generate() - - return { storeFront } -} diff --git a/packages/aggregate-api/test/context/aggregate-store.js b/packages/aggregate-api/test/context/aggregate-store.js deleted file mode 100644 index 939270612..000000000 --- a/packages/aggregate-api/test/context/aggregate-store.js +++ /dev/null @@ -1,36 +0,0 @@ -import * as API from '../../src/types.js' - -/** - * @implements {API.AggregateStore} - */ -export class AggregateStore { - constructor() { - /** @type {Map} */ - this.items = new Map() - } - - /** - * @param {import('@ucanto/interface').Link} pieceLink - * @param {unknown} deal - */ - put(pieceLink, deal) { - const dealEntries = this.items.get(pieceLink.toString()) - let newEntries - if (dealEntries) { - newEntries = [...dealEntries, deal] - this.items.set(pieceLink.toString(), newEntries) - } else { - newEntries = [deal] - this.items.set(pieceLink.toString(), newEntries) - } - - return Promise.resolve() - } - - /** - * @param {import('@ucanto/interface').Link} pieceLink - */ - get(pieceLink) { - return Promise.resolve(this.items.get(pieceLink.toString())) - } -} diff --git a/packages/aggregate-api/test/context/offer-store.js b/packages/aggregate-api/test/context/offer-store.js deleted file mode 100644 index 16724785b..000000000 --- a/packages/aggregate-api/test/context/offer-store.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * @typedef {import('@web3-storage/data-segment').PieceView[]} Offers - */ - -export class OfferStore { - constructor() { - /** @type {Map} */ - this.offers = new Map() - } - /** - * @param {import('../../src/types').OfferToQueue} offerToQueue - */ - async queue(offerToQueue) { - this.offers.set(offerToQueue.piece.link.toString(), offerToQueue.offers) - } - - /** - * @param {import('@ucanto/interface').Link} pieceLink - * @returns {Promise} - */ - async get(pieceLink) { - return Promise.resolve(`todo:${pieceLink.toString()}`) - } -} diff --git a/packages/aggregate-api/test/lib.js b/packages/aggregate-api/test/lib.js deleted file mode 100644 index 98b21c0f1..000000000 --- a/packages/aggregate-api/test/lib.js +++ /dev/null @@ -1,8 +0,0 @@ -import * as Aggregate from './aggregate.js' -export * from './utils.js' - -export const test = { - ...Aggregate.test, -} - -export { Aggregate } diff --git a/packages/aggregate-client/CHANGELOG.md b/packages/aggregate-client/CHANGELOG.md deleted file mode 100644 index 49f857792..000000000 --- a/packages/aggregate-client/CHANGELOG.md +++ /dev/null @@ -1,17 +0,0 @@ -# Changelog - -## 1.0.0 (2023-07-11) - - -### ⚠ BREAKING CHANGES - -* aggregate capabilities now have different nb properties and aggregate client api was simplified - -### Features - -* w3 aggregate protocol client and api implementation ([#787](https://github.com/web3-storage/w3up/issues/787)) ([b58069d](https://github.com/web3-storage/w3up/commit/b58069d7960efe09283f3b23fed77515b62d4639)) - - -### Bug Fixes - -* update aggregate spec in client and api ([#824](https://github.com/web3-storage/w3up/issues/824)) ([ebefd88](https://github.com/web3-storage/w3up/commit/ebefd889a028f325690370db8043c7b9e9fdf7bb)) diff --git a/packages/aggregate-client/README.md b/packages/aggregate-client/README.md deleted file mode 100644 index 3f637f391..000000000 --- a/packages/aggregate-client/README.md +++ /dev/null @@ -1,75 +0,0 @@ -

⁂
web3.storage

-

The aggregate client for https://web3.storage

- -## About - -The `@web3-storage/aggregate-client` package provides the "low level" client API for aggregating data uploaded with the w3up platform. It is based on [web3-storage/specs/w3-aggregation.md])https://github.com/web3-storage/specs/blob/feat/filecoin-spec/w3-aggregation.md) and is not intended for web3.storage end users. - -## Install - -Install the package using npm: - -```bash -npm install @web3-storage/aggregate-client -``` - -## Usage - -### `aggregateOffer` - -```ts -function aggregateOffer( - conf: InvocationConfig, - piece: Piece, - offer: Piece[], -): Promise<{ status: string }> -``` - -Ask the service to create an aggregate offer and put it available for Storage Providers. - -More information: [`InvocationConfig`](#invocationconfig) - -### `aggregateGet` - -```ts -function aggregateGet( - conf: InvocationConfig, - subject: PieceCID, -): Promise -``` - -Ask the service to get deal details of an aggregate. - -More information: [`InvocationConfig`](#invocationconfig) - -## Types - -### `Piece` - -An offered CAR to be part of an Aggregate. - -```ts -export interface Piece { - link: PieceCID - size: number -} - -export type PieceCID = ReturnType -``` - -### `InvocationConfig` - -This is the configuration for the UCAN invocation. It is an object with `issuer`, `audience`, `resource` and `proofs`: - -- The `issuer` is the signing authority that is issuing the UCAN invocation(s). -- The `audience` is the principal authority that the UCAN is delegated to. -- The `resource` (`with` field) points to a storage space. -- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. These might not be required. - -## Contributing - -Feel free to join in. All welcome. Please [open an issue](https://github.com/web3-storage/w3protocol/issues)! - -## License - -Dual-licensed under [MIT + Apache 2.0](https://github.com/web3-storage/w3protocol/blob/main/license.md) diff --git a/packages/aggregate-client/src/aggregate.js b/packages/aggregate-client/src/aggregate.js deleted file mode 100644 index 3164a9e26..000000000 --- a/packages/aggregate-client/src/aggregate.js +++ /dev/null @@ -1,70 +0,0 @@ -import * as AggregateCapabilities from '@web3-storage/capabilities/aggregate' -import { CBOR } from '@ucanto/core' - -import { servicePrincipal, connection } from './service.js' - -export const MIN_SIZE = 1 + 127 * (1 << 27) -export const MAX_SIZE = 127 * (1 << 28) - -/** - * Offer an aggregate to be assembled and stored. - * - * @param {import('./types').InvocationConfig} conf - Configuration - * @param {import('@web3-storage/data-segment').PieceView} piece - * @param {import('@web3-storage/data-segment').PieceView[]} offer - * @param {import('./types').RequestOptions} [options] - */ -export async function aggregateOffer( - { issuer, with: resource, proofs, audience }, - piece, - offer, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - - const block = await CBOR.write(offer) - const invocation = AggregateCapabilities.offer.invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: resource, - nb: { - offer: block.cid, - piece, - }, - proofs, - }) - invocation.attach(block) - - return await invocation.execute(conn) -} - -/** - * Get details of an aggregate. - * - * @param {import('./types').InvocationConfig} conf - Configuration - * @param {import('@web3-storage/data-segment').PieceLink} subject - * @param {import('./types').RequestOptions} [options] - */ -export async function aggregateGet( - { issuer, with: resource, proofs, audience }, - subject, - options = {} -) { - /* c8 ignore next */ - const conn = options.connection ?? connection - - return await AggregateCapabilities.get - .invoke({ - issuer, - /* c8 ignore next */ - audience: audience ?? servicePrincipal, - with: resource, - nb: { - subject, - }, - proofs, - }) - .execute(conn) -} diff --git a/packages/aggregate-client/src/index.js b/packages/aggregate-client/src/index.js deleted file mode 100644 index a8aca01bf..000000000 --- a/packages/aggregate-client/src/index.js +++ /dev/null @@ -1 +0,0 @@ -export * as Aggregate from './aggregate.js' diff --git a/packages/aggregate-client/src/service.js b/packages/aggregate-client/src/service.js deleted file mode 100644 index 587ec5160..000000000 --- a/packages/aggregate-client/src/service.js +++ /dev/null @@ -1,16 +0,0 @@ -import { connect } from '@ucanto/client' -import { CAR, HTTP } from '@ucanto/transport' -import * as DID from '@ipld/dag-ucan/did' - -export const serviceURL = new URL('https://spade-proxy.web3.storage') -export const servicePrincipal = DID.parse('did:web:web3.storage') - -/** @type {import('@ucanto/interface').ConnectionView} */ -export const connection = connect({ - id: servicePrincipal, - codec: CAR.outbound, - channel: HTTP.open({ - url: serviceURL, - method: 'POST', - }), -}) diff --git a/packages/aggregate-client/src/types.ts b/packages/aggregate-client/src/types.ts deleted file mode 100644 index 976768445..000000000 --- a/packages/aggregate-client/src/types.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Link } from 'multiformats/link' -import { CAR } from '@ucanto/transport' -import { - ConnectionView, - ServiceMethod, - Signer, - Proof, - DID, - Principal, -} from '@ucanto/interface' -import { - AggregateOffer, - AggregateOfferSuccess, - AggregateOfferFailure, - AggregateGet, - AggregateGetSuccess, - AggregateGetFailure, - OfferArrange, - OfferArrangeSuccess, - OfferArrangeFailure, -} from '@web3-storage/capabilities/types' - -export interface InvocationConfig { - /** - * Signing authority that is issuing the UCAN invocation(s). - */ - issuer: Signer - /** - * The principal delegated to in the current UCAN. - */ - audience?: Principal - /** - * The resource the invocation applies to. - */ - with: DID - /** - * Proof(s) the issuer has the capability to perform the action. - */ - proofs?: Proof[] -} - -export interface Service { - aggregate: { - offer: ServiceMethod< - AggregateOffer, - AggregateOfferSuccess, - AggregateOfferFailure - > - get: ServiceMethod - } - offer: { - arrange: ServiceMethod< - OfferArrange, - OfferArrangeSuccess, - OfferArrangeFailure - > - } -} - -export interface RequestOptions extends Connectable {} - -export interface Connectable { - connection?: ConnectionView -} - -/** - * An IPLD Link that has the CAR codec code. - */ -export type CARLink = Link diff --git a/packages/aggregate-client/test/aggregate.test.js b/packages/aggregate-client/test/aggregate.test.js deleted file mode 100644 index c90684c69..000000000 --- a/packages/aggregate-client/test/aggregate.test.js +++ /dev/null @@ -1,155 +0,0 @@ -import assert from 'assert' -import * as Client from '@ucanto/client' -import * as Server from '@ucanto/server' -import * as Signer from '@ucanto/principal/ed25519' -import * as CAR from '@ucanto/transport/car' -import { CBOR, parseLink } from '@ucanto/core' -import * as AggregateCapabilities from '@web3-storage/capabilities/aggregate' -import * as OfferCapabilities from '@web3-storage/capabilities/offer' - -import * as Aggregate from '../src/aggregate.js' - -import { serviceProvider } from './fixtures.js' -import { mockService } from './helpers/mocks.js' -import { randomAggregate } from './helpers/random.js' - -describe('aggregate.offer', () => { - it('places a valid offer with the service', async () => { - const { storeFront } = await getContext() - - // Generate Pieces for offer - const { pieces, aggregate } = await randomAggregate(100, 100) - - const offerBlock = await CBOR.write(pieces) - /** @type {import('@web3-storage/capabilities/types').AggregateOfferSuccess} */ - const aggregateOfferResponse = { - status: 'queued', - } - - // Create Ucanto service - const service = mockService({ - aggregate: { - offer: Server.provideAdvanced({ - capability: AggregateCapabilities.offer, - // @ts-expect-error not failure type expected because of assert throw - handler: async ({ invocation, context }) => { - assert.strictEqual(invocation.issuer.did(), storeFront.did()) - assert.strictEqual(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.strictEqual(invCap.can, AggregateCapabilities.offer.can) - assert.equal(invCap.with, invocation.issuer.did()) - // size - assert.strictEqual(invCap.nb?.piece.height, aggregate.height) - assert.ok(invCap.nb?.piece.link) - // TODO: Validate commitmemnt proof - assert.ok(invCap.nb?.offer) - // Validate block inline exists - const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) - assert.ok( - invocationBlocks.find((b) => b.cid.equals(offerBlock.cid)) - ) - - // Create effect for receipt - const fx = await OfferCapabilities.arrange - .invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - pieceLink: invCap.nb?.piece.link, - }, - }) - .delegate() - - return Server.ok(aggregateOfferResponse).join(fx.link()) - }, - }), - }, - }) - const res = await Aggregate.aggregateOffer( - { - issuer: storeFront, - with: storeFront.did(), - audience: serviceProvider, - }, - aggregate, - pieces, - // @ts-expect-error no full service implemented - { connection: getConnection(service).connection } - ) - assert.ok(res.out.ok) - assert.deepEqual(res.out.ok, aggregateOfferResponse) - // includes effect fx in receipt - assert.ok(res.fx.join) - }) -}) - -describe('aggregate.get', () => { - it('places a valid offer with the service', async () => { - const { storeFront } = await getContext() - const subject = - /** @type {import('@web3-storage/data-segment').PieceLink} */ ( - parseLink( - 'baga6ea4seaqm2u43527zehkqqcpyyopgsw2c4mapyy2vbqzqouqtzhxtacueeki' - ) - ) - /** @type {unknown[]} */ - const deals = [] - - // Create Ucanto service - const service = mockService({ - aggregate: { - get: Server.provide(AggregateCapabilities.get, ({ invocation }) => { - assert.strictEqual(invocation.issuer.did(), storeFront.did()) - assert.strictEqual(invocation.capabilities.length, 1) - const invCap = invocation.capabilities[0] - assert.strictEqual(invCap.can, AggregateCapabilities.get.can) - assert.equal(invCap.with, invocation.issuer.did()) - assert.ok(invCap.nb?.subject) - return { ok: { deals } } - }), - }, - }) - - const res = await Aggregate.aggregateGet( - { - issuer: storeFront, - with: storeFront.did(), - audience: serviceProvider, - }, - subject, - // @ts-expect-error no full service implemented - { connection: getConnection(service).connection } - ) - - assert.ok(res.out.ok) - assert.deepEqual(res.out.ok.deals, deals) - }) -}) - -async function getContext() { - const storeFront = await Signer.generate() - - return { storeFront } -} - -/** - * @param {Partial<{ - * aggregate: Partial - * offer: Partial - * }>} service - */ -function getConnection(service) { - const server = Server.create({ - id: serviceProvider, - service, - codec: CAR.inbound, - }) - const connection = Client.connect({ - id: serviceProvider, - codec: CAR.outbound, - channel: server, - }) - - return { connection } -} diff --git a/packages/capabilities/src/aggregate.js b/packages/capabilities/src/aggregate.js deleted file mode 100644 index fabb9bf31..000000000 --- a/packages/capabilities/src/aggregate.js +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Aggregate Capabilities - * - * These can be imported directly with: - * ```js - * import * as Aggregate from '@web3-storage/capabilities/aggregate' - * ``` - * - * @module - */ -import { capability, Schema, ok } from '@ucanto/validator' -import { checkLink, equalWith, equal, and } from './utils.js' - -/** - * @see https://github.com/multiformats/go-multihash/blob/dc3bd6897fcd17f6acd8d4d6ffd2cea3d4d3ebeb/multihash.go#L73 - */ -const SHA2_256_TRUNC254_PADDED = 0x1012 -/** - * @see https://github.com/ipfs/go-cid/blob/829c826f6be23320846f4b7318aee4d17bf8e094/cid.go#L104 - */ -const FilCommitmentUnsealed = 0xf101 - -/** - * `aggregate/offer` capability allows agent to create an offer to get an aggregate - * of CARs files in the market to be fetched and stored by a Storage provider. - */ -export const offer = capability({ - can: 'aggregate/offer', - /** - * did:key identifier of the broker authority where offer is made available. - */ - with: Schema.did(), - nb: Schema.struct({ - /** - * CID of the DAG-CBOR encoded block with offer details. - * Service will queue given offer to be validated and handled. - */ - offer: Schema.link(), - /** - * Commitment proof for the aggregate being offered. - * https://github.com/filecoin-project/go-state-types/blob/1e6cf0d47cdda75383ef036fc2725d1cf51dbde8/abi/piece.go#L47-L50 - */ - piece: Schema.struct({ - /** - * CID of the aggregate piece. - */ - link: /** @type {import('./types').PieceLinkSchema} */ ( - Schema.link({ - code: FilCommitmentUnsealed, - version: 1, - multihash: { - code: SHA2_256_TRUNC254_PADDED, - }, - }) - ), - /** - * Height of the perfect binary tree for the piece. - * It can be used to derive leafCount and consequently `size` of the piece. - */ - height: Schema.integer(), - }), - }), - derives: (claim, from) => { - return ( - and(equalWith(claim, from)) || - and(checkLink(claim.nb.offer, from.nb.offer, 'nb.offer')) || - and( - checkLink(claim.nb.piece.link, from.nb.piece.link, 'nb.piece.link') - ) || - and( - equal(claim.nb.piece.height, from.nb.piece.height, 'nb.piece.height') - ) || - ok({}) - ) - }, -}) - -/** - * Capability can be used to get information about previously stored aggregates. - * space identified by `with` field. - */ -export const get = capability({ - can: 'aggregate/get', - with: Schema.did(), - nb: Schema.struct({ - /** - * Commitment proof for the aggregate being requested. - */ - subject: /** @type {import('./types').PieceLinkSchema} */ (Schema.link()), - }), - derives: (claim, from) => { - return ( - and(equalWith(claim, from)) || - and(checkLink(claim.nb.subject, from.nb.subject, 'nb.subject')) || - ok({}) - ) - }, -}) - -// ⚠️ We export imports here so they are not omitted in generated typedes -// @see https://github.com/microsoft/TypeScript/issues/51548 -export { Schema } diff --git a/packages/capabilities/src/filecoin.js b/packages/capabilities/src/filecoin.js new file mode 100644 index 000000000..2c4293d5d --- /dev/null +++ b/packages/capabilities/src/filecoin.js @@ -0,0 +1,187 @@ +/** + * Filecoin Capabilities + * + * These can be imported directly with: + * ```js + * import * as Filecoin from '@web3-storage/capabilities/filecoin' + * ``` + * + * @module + */ + +import { capability, Schema, ok } from '@ucanto/validator' +import { equal, equalWith, checkLink, and } from './utils.js' + +/** + * @see https://github.com/multiformats/go-multihash/blob/dc3bd6897fcd17f6acd8d4d6ffd2cea3d4d3ebeb/multihash.go#L73 + */ +const SHA2_256_TRUNC254_PADDED = 0x1012 +/** + * @see https://github.com/ipfs/go-cid/blob/829c826f6be23320846f4b7318aee4d17bf8e094/cid.go#L104 + */ +const FilCommitmentUnsealed = 0xf101 + +/** + * `filecoin/add` capability allows agent to add a filecoin piece to be aggregated + * so that it can be stored by a Storage provider on a future time. + */ +export const filecoinAdd = capability({ + can: 'filecoin/add', + /** + * did:key identifier of the broker authority where offer is made available. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the content that resulted in Filecoin piece. + */ + content: Schema.link(), + /** + * CID of the piece. + */ + piece: /** @type {import('./types').PieceLinkSchema} */ ( + Schema.link({ + code: FilCommitmentUnsealed, + version: 1, + multihash: { + code: SHA2_256_TRUNC254_PADDED, + }, + }) + ), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.content, from.nb.content, 'nb.content')) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || + ok({}) + ) + }, +}) + +/** + * `piece/add` capability allows agent to add a piece piece to be aggregated + * so that it can be stored by a Storage provider on a future time. + */ +export const pieceAdd = capability({ + can: 'piece/add', + /** + * did:key identifier of the broker authority where offer is made available. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the piece. + */ + piece: /** @type {import('./types').PieceLinkSchema} */ ( + Schema.link({ + code: FilCommitmentUnsealed, + version: 1, + multihash: { + code: SHA2_256_TRUNC254_PADDED, + }, + }) + ), + /** + * CID of the content that resulted in Filecoin piece. + */ + group: Schema.text(), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || + and(equal(claim.nb.group, from.nb.group, 'nb.group')) || + ok({}) + ) + }, +}) + +/** + * `aggregate/add` capability allows agent to create an offer to get an aggregate + * of CARs files in the market to be fetched and stored by a Storage provider. + */ +export const aggregateAdd = capability({ + can: 'aggregate/add', + /** + * did:key identifier of the broker authority where offer is made available. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the DAG-CBOR encoded block with offer details. + * Service will queue given offer to be validated and handled. + */ + offer: Schema.link(), + /** + * Commitment proof for the aggregate being offered. + * https://github.com/filecoin-project/go-state-types/blob/1e6cf0d47cdda75383ef036fc2725d1cf51dbde8/abi/piece.go#L47-L50 + */ + piece: /** @type {import('./types').PieceLinkSchema} */ ( + Schema.link({ + code: FilCommitmentUnsealed, + version: 1, + multihash: { + code: SHA2_256_TRUNC254_PADDED, + }, + }) + ), + /** + * Necessary fields for a Filecoin Deal proposal. + */ + deal: Schema.struct({ + /** + * with tenantId broker can select one of their configured wallets + */ + tenantId: Schema.text(), + /** + * arbitrary label to be added to the deal on chain + */ + label: Schema.text().optional(), + }), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.offer, from.nb.offer, 'nb.offer')) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || + and( + equal(claim.nb.deal.tenantId, from.nb.deal.tenantId, 'nb.deal.tenantId') + ) || + and(equal(claim.nb.deal.label, from.nb.deal.label, 'nb.deal.label')) || + ok({}) + ) + }, +}) + +/** + * `chain/info` capability allows agent to get chain info of a given piece. + */ +export const chainInfo = capability({ + can: 'chain/info', + /** + * did:key identifier of the broker authority where offer is made available. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the piece. + */ + piece: /** @type {import('./types').PieceLinkSchema} */ ( + Schema.link({ + code: FilCommitmentUnsealed, + version: 1, + multihash: { + code: SHA2_256_TRUNC254_PADDED, + }, + }) + ), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || + ok({}) + ) + }, +}) diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index 7f9119439..28791e91e 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -9,8 +9,7 @@ import * as Utils from './utils.js' import * as Consumer from './consumer.js' import * as Customer from './customer.js' import * as Console from './console.js' -import * as Offer from './offer.js' -import * as Aggregate from './aggregate.js' +import * as Filecoin from './filecoin.js' export { Access, @@ -24,8 +23,7 @@ export { Customer, Console, Utils, - Aggregate, - Offer, + Filecoin, } /** @type {import('./types.js').AbilitiesArray} */ @@ -49,7 +47,8 @@ export const abilitiesAsStrings = [ Access.access.can, Access.authorize.can, Access.session.can, - Aggregate.offer.can, - Aggregate.get.can, - Offer.arrange.can, + Filecoin.filecoinAdd.can, + Filecoin.pieceAdd.can, + Filecoin.aggregateAdd.can, + Filecoin.chainInfo.can, ] diff --git a/packages/capabilities/src/offer.js b/packages/capabilities/src/offer.js deleted file mode 100644 index 5546fabf7..000000000 --- a/packages/capabilities/src/offer.js +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Offer Capabilities - * - * These can be imported directly with: - * ```js - * import * as Offer from '@web3-storage/capabilities/offer' - * ``` - * - * @module - */ -import { capability, Schema, ok } from '@ucanto/validator' -import { equalWith, checkLink, and } from './utils.js' - -/** - * Capability can be used to arrange an offer with an aggregate of CARs. - */ -export const arrange = capability({ - can: 'offer/arrange', - with: Schema.did(), - nb: Schema.struct({ - /** - * Commitment proof for the aggregate being requested. - */ - pieceLink: /** @type {import('./types').PieceLinkSchema} */ (Schema.link()), - }), - derives: (claim, from) => { - return ( - and(equalWith(claim, from)) || - and(checkLink(claim.nb.pieceLink, from.nb.pieceLink, 'nb.pieceLink')) || - ok({}) - ) - }, -}) - -// ⚠️ We export imports here so they are not omitted in generated typedes -// @see https://github.com/microsoft/TypeScript/issues/51548 -export { Schema } diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 42dbbf1d9..31b47fa69 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -10,8 +10,7 @@ import { add, list, remove, store } from './store.js' import * as UploadCaps from './upload.js' import { claim, redeem } from './voucher.js' import * as AccessCaps from './access.js' -import * as AggregateCaps from './aggregate.js' -import * as OfferCaps from './offer.js' +import * as FilecoinCaps from './filecoin.js' export type { Unit } /** @@ -76,29 +75,40 @@ export type SpaceRecoverValidation = InferInvokedCapability< > export type SpaceRecover = InferInvokedCapability -// Aggregate -export interface AggregateGetSuccess { - deals: unknown[] +// filecoin +export type QUEUE_STATUS = 'queued' | 'accepted' | 'rejected' +export interface FilecoinAddSuccess { + status: QUEUE_STATUS + piece: PieceLink } -export interface AggregateGetFailure extends Ucanto.Failure { - name: 'AggregateNotFound' +export interface FilecoinAddFailure extends Ucanto.Failure { + reason: string + piece: PieceLink } -export interface AggregateOfferSuccess { - status: string +export interface PieceAddSuccess { + status: QUEUE_STATUS + piece: PieceLink + aggregate?: PieceLink } -export interface AggregateOfferFailure extends Ucanto.Failure { - name: - | 'AggregateOfferInvalidSize' - | 'AggregateOfferBlockNotFound' - | 'AggregateOfferInvalidUrl' +export interface PieceAddFailure extends Ucanto.Failure { + reason: string + piece: PieceLink } -export interface OfferArrangeSuccess { - status: string +export interface AggregateAddSuccess { + status: QUEUE_STATUS + piece?: PieceLink } -export interface OfferArrangeFailure extends Ucanto.Failure { - name: 'OfferArrangeNotFound' + +export interface AggregateAddFailure extends Ucanto.Failure {} + +export interface ChainInfoSuccess { + // TODO +} + +export interface ChainInfoFailure extends Ucanto.Failure { + // TODO } // Voucher Protocol @@ -114,12 +124,15 @@ export type Store = InferInvokedCapability export type StoreAdd = InferInvokedCapability export type StoreRemove = InferInvokedCapability export type StoreList = InferInvokedCapability -// Aggregate -export type AggregateOffer = InferInvokedCapability -export type AggregateGet = InferInvokedCapability -// Offer -export type OfferArrange = InferInvokedCapability - +// Filecoin +export type FilecoinAdd = InferInvokedCapability< + typeof FilecoinCaps.filecoinAdd +> +export type PieceAdd = InferInvokedCapability +export type AggregateAdd = InferInvokedCapability< + typeof FilecoinCaps.aggregateAdd +> +export type ChainInfo = InferInvokedCapability // Top export type Top = InferInvokedCapability @@ -145,7 +158,8 @@ export type AbilitiesArray = [ Access['can'], AccessAuthorize['can'], AccessSession['can'], - AggregateOffer['can'], - AggregateGet['can'], - OfferArrange['can'] + FilecoinAdd['can'], + PieceAdd['can'], + AggregateAdd['can'], + ChainInfo['can'] ] diff --git a/packages/aggregate-api/package.json b/packages/filecoin-api/package.json similarity index 73% rename from packages/aggregate-api/package.json rename to packages/filecoin-api/package.json index 64be82874..5e0d0a211 100644 --- a/packages/aggregate-api/package.json +++ b/packages/filecoin-api/package.json @@ -1,6 +1,6 @@ { - "name": "@web3-storage/aggregate-api", - "version": "1.0.0", + "name": "@web3-storage/filecoin-api", + "version": "0.0.0", "type": "module", "main": "./src/lib.js", "files": [ @@ -14,11 +14,17 @@ "src/lib.js": [ "dist/src/lib.d.ts" ], - "aggregate": [ - "dist/src/aggregate.d.ts" + "aggregator": [ + "dist/src/aggregator.d.ts" ], - "offer": [ - "dist/src/offer.d.ts" + "broker": [ + "dist/src/broker.d.ts" + ], + "chain": [ + "dist/src/chain.d.ts" + ], + "storefront": [ + "dist/src/storefront.d.ts" ], "types": [ "dist/src/types.d.ts" @@ -37,13 +43,21 @@ "types": "./dist/src/types.d.ts", "import": "./src/types.js" }, - "./aggregate": { - "types": "./dist/src/aggregate.d.ts", - "import": "./src/aggregate.js" + "./aggregator": { + "types": "./dist/src/aggregator.d.ts", + "import": "./src/aggregator.js" + }, + "./broker": { + "types": "./dist/src/broker.d.ts", + "import": "./src/broker.js" }, - "./offer": { - "types": "./dist/src/offer.d.ts", - "import": "./src/offer.js" + "./chain": { + "types": "./dist/src/chain.d.ts", + "import": "./src/chain.js" + }, + "./storefront": { + "types": "./dist/src/storefront.d.ts", + "import": "./src/storefront.js" }, "./test": { "types": "./dist/test/lib.d.ts", @@ -69,9 +83,10 @@ "devDependencies": { "@ipld/car": "^5.1.1", "@types/mocha": "^10.0.1", + "@ucanto/client": "^8.0.0", "@ucanto/principal": "^8.0.0", "@web-std/blob": "^3.0.4", - "@web3-storage/aggregate-client": "workspace:^", + "@web3-storage/filecoin-client": "workspace:^", "hd-scripts": "^4.1.0", "mocha": "^10.2.0", "multiformats": "^11.0.2" diff --git a/packages/filecoin-api/src/aggregator.js b/packages/filecoin-api/src/aggregator.js new file mode 100644 index 000000000..45af50fd5 --- /dev/null +++ b/packages/filecoin-api/src/aggregator.js @@ -0,0 +1,128 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import * as API from './types.js' +import { QueueOperationFailed, StoreOperationFailed } from './errors.js' + +/** + * @param {API.Input} input + * @param {API.AggregatorServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const claim = async ({ capability }, context) => { + const { piece, group } = capability.nb + // Check if self signed to call queue handler + if (context.id.did() === capability.with) { + return queueHandler(piece, group, context) + } + + return queueAdd(piece, group, context) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {string} group + * @param {API.AggregatorServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function queueAdd(piece, group, context) { + const queued = await context.addQueue.add( + { + piece, + }, + { + group, + } + ) + if (queued.error) { + return { + error: new QueueOperationFailed(queued.error.message, piece), + } + } + + // Create effect for receipt + const fx = await FilecoinCapabilities.pieceAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + group, + }, + }) + .delegate() + + return Server.ok({ + status: /** @type {API.QUEUE_STATUS} */ ('queued'), + piece, + }).join(fx.link()) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {string} group + * @param {API.AggregatorServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function queueHandler(piece, group, context) { + const put = await context.pieceStore.put({ + piece, + // TODO + }) + + if (put.error) { + return { + error: new StoreOperationFailed(put.error.message, piece), + } + } + + // TODO: send to broker? + + return { + ok: { + status: 'accepted', + piece, + }, + } +} + +/** + * @param {API.AggregatorServiceContext} context + */ +export function createService(context) { + return { + piece: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.pieceAdd, + handler: (input) => claim(input, context), + }), + }, + } +} + +/** + * @param {API.UcantoServerContext & API.AggregatorServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/filecoin-api/src/broker.js b/packages/filecoin-api/src/broker.js new file mode 100644 index 000000000..977c99ad3 --- /dev/null +++ b/packages/filecoin-api/src/broker.js @@ -0,0 +1,162 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import { CBOR } from '@ucanto/core' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import * as API from './types.js' +import { + QueueOperationFailed, + StoreOperationFailed, + DecodeBlockOperationFailed, +} from './errors.js' + +/** + * @param {API.Input} input + * @param {API.BrokerServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const claim = async ({ capability, invocation }, context) => { + const { piece, offer: offerCid, deal } = capability.nb + const offer = getOfferBlock(offerCid, invocation.iterateIPLDBlocks()) + + if (!offer) { + return { + error: new DecodeBlockOperationFailed( + `missing offer block in invocation: ${offerCid.toString()}`, + piece + ), + } + } + + // Check if self signed to call queue handler + if (context.id.did() === capability.with) { + return queueHandler(piece, offer, deal, context) + } + + return queueAdd(piece, offerCid, deal, offer, context) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {Server.API.Link} offerCid + * @param {import('@web3-storage/data-segment').PieceLink[]} offer + * @param {import('@web3-storage/filecoin-client/types').DealConfig} deal + * @param {API.BrokerServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function queueAdd(piece, offerCid, deal, offer, context) { + const queued = await context.addQueue.add({ + piece, + offer, // TODO: not store in queue but proper data structure + deal, + }) + if (queued.error) { + return { + error: new QueueOperationFailed(queued.error.message, piece), + } + } + + // Create effect for receipt + const fx = await FilecoinCapabilities.aggregateAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + offer: offerCid, + deal, + }, + }) + .delegate() + + return Server.ok({ + status: /** @type {API.QUEUE_STATUS} */ ('queued'), + piece, + }).join(fx.link()) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('@web3-storage/data-segment').PieceLink[]} offer + * @param {import('@web3-storage/filecoin-client/types').DealConfig} deal + * @param {API.BrokerServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function queueHandler(piece, offer, deal, context) { + const put = await context.offerStore.put({ + offer, + piece, + deal, + }) + if (put.error) { + return { + error: new StoreOperationFailed(put.error.message, piece), + } + } + + // TODO: how to failure? + + return { + ok: { + status: 'accepted', + piece, + }, + } +} + +/** + * @param {Server.API.Link} offerCid + * @param {IterableIterator>} blockIterator + */ +function getOfferBlock(offerCid, blockIterator) { + for (const block of blockIterator) { + if (block.cid.equals(offerCid)) { + const decoded = + /** @type {import('@web3-storage/data-segment').PieceLink[]} */ ( + CBOR.decode(block.bytes) + ) + return decoded + // TODO: Validate with schema + } + } +} + +/** + * @param {API.BrokerServiceContext} context + */ +export function createService(context) { + return { + aggregate: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.aggregateAdd, + handler: (input) => claim(input, context), + }), + }, + } +} + +/** + * @param {API.UcantoServerContext & API.BrokerServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/filecoin-api/src/errors.js b/packages/filecoin-api/src/errors.js new file mode 100644 index 000000000..9f616801c --- /dev/null +++ b/packages/filecoin-api/src/errors.js @@ -0,0 +1,61 @@ +import * as Server from '@ucanto/server' + +export const QueueOperationErrorName = /** @type {const} */ ( + 'QueueOperationFailed' +) +export class QueueOperationFailed extends Server.Failure { + /** + * @param {string} message + * @param {import('@web3-storage/data-segment').PieceLink} piece + */ + constructor(message, piece) { + super(message) + this.piece = piece + } + get reason() { + return this.message + } + get name() { + return QueueOperationErrorName + } +} + +export const StoreOperationErrorName = /** @type {const} */ ( + 'StoreOperationFailed' +) +export class StoreOperationFailed extends Server.Failure { + /** + * @param {string} message + * @param {import('@web3-storage/data-segment').PieceLink} piece + */ + constructor(message, piece) { + super(message) + this.piece = piece + } + get reason() { + return this.message + } + get name() { + return StoreOperationErrorName + } +} + +export const DecodeBlockOperationErrorName = /** @type {const} */ ( + 'DecodeBlockOperationFailed' +) +export class DecodeBlockOperationFailed extends Server.Failure { + /** + * @param {string} message + * @param {import('@web3-storage/data-segment').PieceLink} piece + */ + constructor(message, piece) { + super(message) + this.piece = piece + } + get reason() { + return this.message + } + get name() { + return DecodeBlockOperationErrorName + } +} diff --git a/packages/filecoin-api/src/lib.js b/packages/filecoin-api/src/lib.js new file mode 100644 index 000000000..83d95767d --- /dev/null +++ b/packages/filecoin-api/src/lib.js @@ -0,0 +1,3 @@ +export * as Storefront from './storefront.js' +export * as Aggregator from './aggregator.js' +export * as Broker from './broker.js' diff --git a/packages/filecoin-api/src/storefront.js b/packages/filecoin-api/src/storefront.js new file mode 100644 index 000000000..2b7143c11 --- /dev/null +++ b/packages/filecoin-api/src/storefront.js @@ -0,0 +1,127 @@ +import * as Server from '@ucanto/server' +import * as Client from '@ucanto/client' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import * as API from './types.js' +import { QueueOperationFailed, StoreOperationFailed } from './errors.js' + +/** + * @param {API.Input} input + * @param {API.StorefrontServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const claim = async ({ capability }, context) => { + // TODO: source + const { piece, content } = capability.nb + + // Check if self signed to call queue handler + if (context.id.did() === capability.with) { + return queueHandler(piece, content, context) + } + + // TODO: queue verify + + return queueAdd(piece, content, context) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('multiformats').UnknownLink} content + * @param {API.StorefrontServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function queueAdd(piece, content, context) { + const queued = await context.addQueue.add({ + piece, + content, + }) + if (queued.error) { + return { + error: new QueueOperationFailed(queued.error.message, piece), + } + } + + // Create effect for receipt + const fx = await FilecoinCapabilities.filecoinAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece, + content, + }, + }) + .delegate() + + return Server.ok({ + status: /** @type {API.QUEUE_STATUS} */ ('queued'), + piece, + }).join(fx.link()) +} + +/** + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('multiformats').UnknownLink} content + * @param {API.StorefrontServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +async function queueHandler(piece, content, context) { + const put = await context.pieceStore.put({ + content, + piece, + }) + if (put.error) { + return { + error: new StoreOperationFailed(put.error.message, piece), + } + } + + // TODO: call piece/add to aggregator + + return { + ok: { + status: 'accepted', + piece, + }, + } +} + +/** + * @param {API.StorefrontServiceContext} context + */ +export function createService(context) { + return { + filecoin: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.filecoinAdd, + handler: (input) => claim(input, context), + }), + }, + } +} + +/** + * @param {API.UcantoServerContext & API.StorefrontServiceContext} context + */ +export const createServer = (context) => + Server.create({ + id: context.id, + codec: context.codec || CAR.inbound, + service: createService(context), + catch: (error) => context.errorReporter.catch(error), + }) + +/** + * @param {object} options + * @param {API.UcantoInterface.Principal} options.id + * @param {API.UcantoInterface.Transport.Channel} options.channel + * @param {API.UcantoInterface.OutboundCodec} [options.codec] + */ +export const connect = ({ id, channel, codec = CAR.outbound }) => + Client.connect({ + id, + channel, + codec, + }) diff --git a/packages/aggregate-api/src/types.js b/packages/filecoin-api/src/types.js similarity index 100% rename from packages/aggregate-api/src/types.js rename to packages/filecoin-api/src/types.js diff --git a/packages/filecoin-api/src/types.ts b/packages/filecoin-api/src/types.ts new file mode 100644 index 000000000..6a841715d --- /dev/null +++ b/packages/filecoin-api/src/types.ts @@ -0,0 +1,170 @@ +import type { + HandlerExecutionError, + Signer, + InboundCodec, + CapabilityParser, + ParsedCapability, + InferInvokedCapability, + Match, +} from '@ucanto/interface' +import type { ProviderInput } from '@ucanto/server' +import { PieceLink } from '@web3-storage/data-segment' +import { UnknownLink } from '@ucanto/interface' + +export * as UcantoInterface from '@ucanto/interface' +export * from '@web3-storage/filecoin-client/types' +export * from '@web3-storage/capabilities/types' + +// Resources +export interface Queue { + add: (record: Record, options?: any) => Promise> +} + +export interface Store { + put: (key: Record) => Promise> +} + +// Services +export interface StorefrontServiceContext { + id: Signer + addQueue: Queue + pieceStore: Store + aggregatorDid: string + aggregatorUrl: string +} + +export interface AggregatorServiceContext { + id: Signer + addQueue: Queue + pieceStore: Store + brokerDid: string + brokerUrl: string +} + +export interface BrokerServiceContext { + id: Signer + addQueue: Queue + offerStore: Store +} + +// Service Types + +export interface StorefrontQueueRecord { + piece: PieceLink + content: UnknownLink + // TODO: Source +} + +export interface StorefrontRecord { + piece: PieceLink + content: UnknownLink +} + +export interface AggregatorQueueRecord { + piece: PieceLink +} + +// Errors + +export type QueueAddError = QueueOperationError +export type StorePutError = StoreOperationError + +export interface QueueOperationError extends Error { + name: 'QueueOperationFailed' +} + +export interface StoreOperationError extends Error { + name: 'StoreOperationFailed' +} + +// Service utils + +export interface UcantoServerContext { + id: Signer + codec?: InboundCodec + errorReporter: ErrorReporter +} + +export interface ErrorReporter { + catch: (error: HandlerExecutionError) => void +} + +export type Result = Variant<{ + ok: T + error: X +}> + +/** + * Utility type for defining a [keyed union] type as in IPLD Schema. In practice + * this just works around typescript limitation that requires discriminant field + * on all variants. + * + * ```ts + * type Result = + * | { ok: T } + * | { error: X } + * + * const demo = (result: Result) => { + * if (result.ok) { + * // ^^^^^^^^^ Property 'ok' does not exist on type '{ error: Error; }` + * } + * } + * ``` + * + * Using `Variant` type we can define same union type that works as expected: + * + * ```ts + * type Result = Variant<{ + * ok: T + * error: X + * }> + * + * const demo = (result: Result) => { + * if (result.ok) { + * result.ok.toUpperCase() + * } + * } + * ``` + * + * [keyed union]:https://ipld.io/docs/schemas/features/representation-strategies/#union-keyed-representation + */ +export type Variant> = { + [Key in keyof U]: { [K in Exclude]?: never } & { + [K in Key]: U[Key] + } +}[keyof U] + +// test + +export type Test = ( + assert: Assert, + context: UcantoServerContext & S +) => unknown +export type Tests = Record> + +export type Input>> = + ProviderInput & ParsedCapability> + +export interface Assert { + equal: ( + actual: Actual, + expected: Expected, + message?: string + ) => unknown + deepEqual: ( + actual: Actual, + expected: Expected, + message?: string + ) => unknown + ok: (actual: Actual, message?: string) => unknown +} + +export interface TestQueue extends Queue { + add: (record: Record, options?: any) => Promise> + all: () => Record[] +} + +export interface TestStore extends Store { + put: (key: Record) => Promise> + all: () => Record[] +} diff --git a/packages/filecoin-api/test/aggregator.spec.js b/packages/filecoin-api/test/aggregator.spec.js new file mode 100644 index 000000000..66b979b24 --- /dev/null +++ b/packages/filecoin-api/test/aggregator.spec.js @@ -0,0 +1,48 @@ +/* eslint-disable no-only-tests/no-only-tests */ +import * as assert from 'assert' +import * as Aggregator from './services/aggregator.js' +import * as Signer from '@ucanto/principal/ed25519' + +import { Store } from './context/store.js' +import { Queue } from './context/queue.js' + +describe('piece/*', () => { + for (const [name, test] of Object.entries(Aggregator.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const signer = await Signer.generate() + const id = signer.withDID('did:web:test.aggregator.web3.storage') + + // resources + const addQueue = new Queue() + const pieceStore = new Store() + const brokerDid = '' + const brokerUrl = '' + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id, + errorReporter: { + catch(error) { + assert.fail(error) + }, + }, + addQueue, + pieceStore, + brokerDid, + brokerUrl, + } + ) + }) + } +}) diff --git a/packages/filecoin-api/test/broker.spec.js b/packages/filecoin-api/test/broker.spec.js new file mode 100644 index 000000000..d97d7da64 --- /dev/null +++ b/packages/filecoin-api/test/broker.spec.js @@ -0,0 +1,44 @@ +/* eslint-disable no-only-tests/no-only-tests */ +import * as assert from 'assert' +import * as Broker from './services/broker.js' +import * as Signer from '@ucanto/principal/ed25519' + +import { Store } from './context/store.js' +import { Queue } from './context/queue.js' + +describe('aggregate/*', () => { + for (const [name, test] of Object.entries(Broker.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const signer = await Signer.generate() + const id = signer.withDID('did:web:test.spade-proxy.web3.storage') + + // resources + const addQueue = new Queue() + const offerStore = new Store() + + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + { + id, + errorReporter: { + catch(error) { + assert.fail(error) + }, + }, + addQueue, + offerStore, + } + ) + }) + } +}) diff --git a/packages/filecoin-api/test/context/queue.js b/packages/filecoin-api/test/context/queue.js new file mode 100644 index 000000000..a0682e30f --- /dev/null +++ b/packages/filecoin-api/test/context/queue.js @@ -0,0 +1,27 @@ +import * as API from '../../src/types.js' + +/** + * @template T + * @implements {API.TestQueue} + */ +export class Queue { + constructor() { + /** @type {Set} */ + this.items = new Set() + } + + /** + * @param {T} record + */ + async add(record) { + this.items.add(record) + + return Promise.resolve({ + ok: {}, + }) + } + + all() { + return Array.from(this.items) + } +} diff --git a/packages/filecoin-api/test/context/store.js b/packages/filecoin-api/test/context/store.js new file mode 100644 index 000000000..b2aa44b24 --- /dev/null +++ b/packages/filecoin-api/test/context/store.js @@ -0,0 +1,27 @@ +import * as API from '../../src/types.js' + +/** + * @template T + * @implements {API.TestStore} + */ +export class Store { + constructor() { + /** @type {Set} */ + this.items = new Set() + } + + /** + * @param {T} record + */ + async put(record) { + this.items.add(record) + + return Promise.resolve({ + ok: {}, + }) + } + + all() { + return Array.from(this.items) + } +} diff --git a/packages/filecoin-api/test/lib.js b/packages/filecoin-api/test/lib.js new file mode 100644 index 000000000..ae05a7776 --- /dev/null +++ b/packages/filecoin-api/test/lib.js @@ -0,0 +1,12 @@ +import * as Aggregator from './services/aggregator.js' +import * as Broker from './services/broker.js' +import * as Storefront from './services/storefront.js' +export * from './utils.js' + +export const test = { + ...Aggregator.test, + ...Broker.test, + ...Storefront.test, +} + +export { Aggregator, Broker, Storefront } diff --git a/packages/filecoin-api/test/services/aggregator.js b/packages/filecoin-api/test/services/aggregator.js new file mode 100644 index 000000000..d2833ba6b --- /dev/null +++ b/packages/filecoin-api/test/services/aggregator.js @@ -0,0 +1,149 @@ +import { Filecoin } from '@web3-storage/capabilities' +import * as Signer from '@ucanto/principal/ed25519' + +import * as API from '../../src/types.js' + +import { randomCargo } from '../utils.js' +import { createServer, connect } from '../../src/aggregator.js' + +/** + * @type {API.Tests + * pieceStore: API.TestStore + * }>} + */ +export const test = { + 'piece/add inserts piece into processing queue': async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = storefront.did() + + // storefront invocation + const pieceAddInv = Filecoin.pieceAdd.invoke({ + issuer: storefront, + audience: connection.id, + with: storefront.did(), + nb: { + piece: cargo.link.link(), + group, + }, + }) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.status, 'queued') + + // Validate effect in receipt + const fx = await Filecoin.pieceAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + group, + }, + }) + .delegate() + + assert.ok(response.fx.join) + assert.ok(fx.link().equals(response.fx.join?.link())) + + const queuedItems = context.addQueue.all() + assert.equal(queuedItems.length, 1) + assert.ok(queuedItems.find((item) => item.piece.equals(cargo.link.link()))) + + const storedItems = context.pieceStore.all() + assert.equal(storedItems.length, 0) + }, + 'piece/add from signer inserts piece into store and returns accepted': async ( + assert, + context + ) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = storefront.did() + + // aggregator invocation + const pieceAddInv = Filecoin.pieceAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + group, + }, + }) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.status, 'accepted') + + const queuedItems = context.addQueue.all() + assert.equal(queuedItems.length, 0) + + const storedItems = context.pieceStore.all() + assert.equal(storedItems.length, 1) + assert.ok(storedItems.find((item) => item.piece.equals(cargo.link.link()))) + }, + 'skip piece/add from signer inserts piece into store and returns rejected': + async (assert, context) => { + const { storefront } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + const group = storefront.did() + + // aggregator invocation + const pieceAddInv = Filecoin.pieceAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + group, + }, + }) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.status, 'rejected') + + const queuedItems = context.addQueue.all() + assert.equal(queuedItems.length, 0) + + const storedItems = context.pieceStore.all() + assert.equal(storedItems.length, 0) + }, +} + +async function getServiceContext() { + const storefront = await Signer.generate() + + return { storefront } +} diff --git a/packages/filecoin-api/test/services/broker.js b/packages/filecoin-api/test/services/broker.js new file mode 100644 index 000000000..1dfdb1ca1 --- /dev/null +++ b/packages/filecoin-api/test/services/broker.js @@ -0,0 +1,172 @@ +import { Filecoin } from '@web3-storage/capabilities' +import * as Signer from '@ucanto/principal/ed25519' +import { CBOR } from '@ucanto/core' + +import * as API from '../../src/types.js' + +import { randomAggregate } from '../utils.js' +import { createServer, connect } from '../../src/broker.js' + +/** + * @type {API.Tests + * offerStore: API.TestStore + * }>} + */ +export const test = { + 'aggregate/add inserts piece into processing queue': async ( + assert, + context + ) => { + const { aggregator } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const offerBlock = await CBOR.write(offer) + const dealConfig = { + tenantId: 'web3.storage', + } + + // aggregator invocation + const pieceAddInv = Filecoin.aggregateAdd.invoke({ + issuer: aggregator, + audience: connection.id, + with: aggregator.did(), + nb: { + piece: aggregate.link, + offer: offerBlock.cid, + deal: dealConfig, + }, + }) + pieceAddInv.attach(offerBlock) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.status, 'queued') + + // Validate effect in receipt + const fx = await Filecoin.aggregateAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: aggregate.link, + offer: offerBlock.cid, + deal: dealConfig, + }, + }) + .delegate() + + assert.ok(response.fx.join) + assert.ok(fx.link().equals(response.fx.join?.link())) + + const queuedItems = context.addQueue.all() + assert.equal(queuedItems.length, 1) + assert.ok( + queuedItems.find((item) => item.piece.equals(aggregate.link.link())) + ) + + const storedItems = context.offerStore.all() + assert.equal(storedItems.length, 0) + }, + 'aggregate/add from signer inserts piece into store and returns accepted': + async (assert, context) => { + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const offerBlock = await CBOR.write(offer) + const dealConfig = { + tenantId: 'web3.storage', + } + + // aggregator invocation + const pieceAddInv = Filecoin.aggregateAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: aggregate.link, + offer: offerBlock.cid, + deal: dealConfig, + }, + }) + pieceAddInv.attach(offerBlock) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.status, 'accepted') + + const queuedItems = context.addQueue.all() + assert.equal(queuedItems.length, 0) + + const storedItems = context.offerStore.all() + assert.equal(storedItems.length, 1) + assert.ok( + storedItems.find((item) => item.piece.equals(aggregate.link.link())) + ) + }, + 'skip aggregate/add from signer inserts piece into store and returns rejected': + async (assert, context) => { + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + const offerBlock = await CBOR.write(offer) + const dealConfig = { + tenantId: 'web3.storage', + } + + // aggregator invocation + const pieceAddInv = Filecoin.aggregateAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: aggregate.link, + offer: offerBlock.cid, + deal: dealConfig, + }, + }) + pieceAddInv.attach(offerBlock) + + const response = await pieceAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.status, 'rejected') + + const queuedItems = context.addQueue.all() + assert.equal(queuedItems.length, 0) + + const storedItems = context.offerStore.all() + assert.equal(storedItems.length, 0) + }, +} + +async function getServiceContext() { + const aggregator = await Signer.generate() + + return { aggregator } +} diff --git a/packages/filecoin-api/test/services/storefront.js b/packages/filecoin-api/test/services/storefront.js new file mode 100644 index 000000000..0d443c535 --- /dev/null +++ b/packages/filecoin-api/test/services/storefront.js @@ -0,0 +1,157 @@ +import { Filecoin } from '@web3-storage/capabilities' +import * as Signer from '@ucanto/principal/ed25519' + +import * as API from '../../src/types.js' + +import { randomCargo } from '../utils.js' +import { createServer, connect } from '../../src/storefront.js' + +/** + * @type {API.Tests + * pieceStore: API.TestStore + * }>} + */ +export const test = { + 'filecoin/add inserts piece into verification queue': async ( + assert, + context + ) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // agent invocation + const filecoinAddInv = Filecoin.filecoinAdd.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.status, 'queued') + + // Validate effect in receipt + const fx = await Filecoin.filecoinAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + .delegate() + + assert.ok(response.fx.join) + assert.ok(fx.link().equals(response.fx.join?.link())) + + const queuedItems = context.addQueue.all() + assert.equal(queuedItems.length, 1) + assert.ok( + queuedItems.find( + (item) => + item.piece.equals(cargo.link.link()) && + item.content.equals(cargo.content.link()) + ) + ) + + const storedItems = context.pieceStore.all() + assert.equal(storedItems.length, 0) + }, + 'filecoin/add from signer inserts piece into store and returns accepted': + async (assert, context) => { + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // storefront invocation + const filecoinAddInv = Filecoin.filecoinAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.status, 'accepted') + + const queuedItems = context.addQueue.all() + assert.equal(queuedItems.length, 0) + + const storedItems = context.pieceStore.all() + assert.equal(storedItems.length, 1) + assert.ok( + storedItems.find( + (item) => + item.piece.equals(cargo.link.link()) && + item.content.equals(cargo.content.link()) + ) + ) + }, + 'skip filecoin/add from signer inserts piece into store and returns rejected': + async (assert, context) => { + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Generate piece for test + const [cargo] = await randomCargo(1, 128) + + // storefront invocation + const filecoinAddInv = Filecoin.filecoinAdd.invoke({ + issuer: context.id, + audience: connection.id, + with: context.id.did(), + nb: { + piece: cargo.link.link(), + content: cargo.content.link(), + }, + }) + + const response = await filecoinAddInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.deepEqual(response.out.ok.status, 'rejected') + + const queuedItems = context.addQueue.all() + assert.equal(queuedItems.length, 0) + + const storedItems = context.pieceStore.all() + assert.equal(storedItems.length, 0) + }, +} + +async function getServiceContext() { + const agent = await Signer.generate() + + return { agent } +} diff --git a/packages/aggregate-api/test/aggregate.spec.js b/packages/filecoin-api/test/storefront.spec.js similarity index 59% rename from packages/aggregate-api/test/aggregate.spec.js rename to packages/filecoin-api/test/storefront.spec.js index 1e7ae0551..dae5361eb 100644 --- a/packages/aggregate-api/test/aggregate.spec.js +++ b/packages/filecoin-api/test/storefront.spec.js @@ -1,13 +1,13 @@ /* eslint-disable no-only-tests/no-only-tests */ import * as assert from 'assert' -import * as Aggregate from './aggregate.js' +import * as Storefront from './services/storefront.js' import * as Signer from '@ucanto/principal/ed25519' -import { OfferStore } from './context/offer-store.js' -import { AggregateStore } from './context/aggregate-store.js' +import { Store } from './context/store.js' +import { Queue } from './context/queue.js' -describe('aggregate/*', () => { - for (const [name, test] of Object.entries(Aggregate.test)) { +describe('filecoin/*', () => { + for (const [name, test] of Object.entries(Storefront.test)) { const define = name.startsWith('only ') ? it.only : name.startsWith('skip ') @@ -17,7 +17,12 @@ describe('aggregate/*', () => { define(name, async () => { const signer = await Signer.generate() const id = signer.withDID('did:web:test.web3.storage') - const aggregateStore = new AggregateStore() + + // resources + const addQueue = new Queue() + const pieceStore = new Store() + const aggregatorDid = '' + const aggregatorUrl = '' await test( { @@ -32,10 +37,10 @@ describe('aggregate/*', () => { assert.fail(error) }, }, - offerStore: new OfferStore(), - arrangedOfferStore: new OfferStore(), - aggregateStore, - aggregateStoreBackend: aggregateStore, + addQueue, + pieceStore, + aggregatorDid, + aggregatorUrl, } ) }) diff --git a/packages/aggregate-api/test/utils.js b/packages/filecoin-api/test/utils.js similarity index 98% rename from packages/aggregate-api/test/utils.js rename to packages/filecoin-api/test/utils.js index 663d7685b..47452a6cf 100644 --- a/packages/aggregate-api/test/utils.js +++ b/packages/filecoin-api/test/utils.js @@ -67,6 +67,7 @@ export async function randomCargo(length, size) { return { link: piece.link, + content: car.cid, height: piece.height, size: piece.size, } diff --git a/packages/aggregate-api/tsconfig.json b/packages/filecoin-api/tsconfig.json similarity index 100% rename from packages/aggregate-api/tsconfig.json rename to packages/filecoin-api/tsconfig.json diff --git a/packages/filecoin-client/README.md b/packages/filecoin-client/README.md new file mode 100644 index 000000000..0fb6b9b35 --- /dev/null +++ b/packages/filecoin-client/README.md @@ -0,0 +1,37 @@ +

⁂
web3.storage

+

The w3filecoin client for https://web3.storage

+ +## About + +The `@web3-storage/w3filecoin-client` package provides the "low level" client API to make data uploaded with the w3up platform available in Filecoin Storage providers. It is based on [web3-storage/specs/w3-filecoin.md])https://github.com/web3-storage/specs/blob/feat/filecoin-spec/w3-filecoin.md) and is not intended for web3.storage end users. + +## Install + +Install the package using npm: + +```bash +npm install @web3-storage/w3filecoin-client +``` + +## Usage + +TODO + +## Types + +### `InvocationConfig` + +This is the configuration for the UCAN invocation. It is an object with `issuer`, `audience`, `resource` and `proofs`: + +- The `issuer` is the signing authority that is issuing the UCAN invocation(s). +- The `audience` is the principal authority that the UCAN is delegated to. +- The `resource` (`with` field) points to a storage space. +- The `proofs` are a set of capability delegations that prove the issuer has the capability to perform the action. These might not be required. + +## Contributing + +Feel free to join in. All welcome. Please [open an issue](https://github.com/web3-storage/w3protocol/issues)! + +## License + +Dual-licensed under [MIT + Apache 2.0](https://github.com/web3-storage/w3protocol/blob/main/license.md) \ No newline at end of file diff --git a/packages/aggregate-client/package.json b/packages/filecoin-client/package.json similarity index 80% rename from packages/aggregate-client/package.json rename to packages/filecoin-client/package.json index 451974cf8..46c3dff9d 100644 --- a/packages/aggregate-client/package.json +++ b/packages/filecoin-client/package.json @@ -1,12 +1,12 @@ { - "name": "@web3-storage/aggregate-client", - "version": "1.0.0", - "description": "The web3.storage aggregate client", - "homepage": "https://github.com/web3-storage/w3up/tree/main/packages/aggregate-client", + "name": "@web3-storage/filecoin-client", + "version": "0.0.0", + "description": "The w3filecoin client for web3.storage", + "homepage": "https://github.com/web3-storage/w3up/tree/main/packages/filecoin-client", "repository": { "type": "git", "url": "https://github.com/web3-storage/w3up.git", - "directory": "packages/aggregate-client" + "directory": "packages/w3filecoin-client" }, "author": "Vasco Santos", "license": "Apache-2.0 OR MIT", @@ -24,7 +24,10 @@ }, "exports": { ".": "./src/index.js", - "./aggregate": "./src/aggregate.js", + "./aggregator": "./src/aggregator.js", + "./broker": "./src/broker.js", + "./chain": "./src/chain.js", + "./storefront": "./src/storefront.js", "./types": "./src/types.js" }, "typesVersions": { @@ -32,8 +35,17 @@ "types": [ "dist/src/types.d.ts" ], - "aggregate": [ - "dist/src/aggregate.d.ts" + "aggregator": [ + "dist/src/aggregator.d.ts" + ], + "broker": [ + "dist/src/broker.d.ts" + ], + "chain": [ + "dist/src/chain.d.ts" + ], + "storefront": [ + "dist/src/storefront.d.ts" ] } }, diff --git a/packages/filecoin-client/src/aggregator.js b/packages/filecoin-client/src/aggregator.js new file mode 100644 index 000000000..dd36ae14a --- /dev/null +++ b/packages/filecoin-client/src/aggregator.js @@ -0,0 +1,52 @@ +import { connect } from '@ucanto/client' +import { CAR, HTTP } from '@ucanto/transport' + +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { services } from './service.js' + +/** + * @typedef {import('@ucanto/interface').ConnectionView} ConnectionView + */ + +/** @type {ConnectionView} */ +export const connection = connect({ + id: services.AGGREGATOR.principal, + codec: CAR.outbound, + channel: HTTP.open({ + url: services.AGGREGATOR.url, + method: 'POST', + }), +}) + +/** + * Add a piece to the aggregator system of the filecoin pipeline. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {string} group + * @param {import('./types.js').RequestOptions} [options] + */ +export async function pieceAdd( + { issuer, with: resource, proofs, audience }, + piece, + group, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const invocation = FilecoinCapabilities.pieceAdd.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.STORE_FRONT.principal, + with: resource, + nb: { + group, + piece, + }, + proofs, + }) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/broker.js b/packages/filecoin-client/src/broker.js new file mode 100644 index 000000000..f5afa92b4 --- /dev/null +++ b/packages/filecoin-client/src/broker.js @@ -0,0 +1,58 @@ +import { connect } from '@ucanto/client' +import { CAR, HTTP } from '@ucanto/transport' +import { CBOR } from '@ucanto/core' + +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { services } from './service.js' + +/** + * @typedef {import('@ucanto/interface').ConnectionView} ConnectionView + */ + +/** @type {ConnectionView} */ +export const connection = connect({ + id: services.BROKER.principal, + codec: CAR.outbound, + channel: HTTP.open({ + url: services.BROKER.url, + method: 'POST', + }), +}) + +/** + * Add a piece (aggregate) to the broker system of the filecoin pipeline to offer to SPs. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('@web3-storage/data-segment').PieceLink[]} offer + * @param {import('./types.js').DealConfig} deal + * @param {import('./types.js').RequestOptions} [options] + */ +export async function aggregateAdd( + { issuer, with: resource, proofs, audience }, + piece, + offer, + deal, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const block = await CBOR.write(offer) + const invocation = FilecoinCapabilities.aggregateAdd.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.AGGREGATOR.principal, + with: resource, + nb: { + piece, + offer: block.cid, + deal, + }, + proofs, + }) + invocation.attach(block) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/chain.js b/packages/filecoin-client/src/chain.js new file mode 100644 index 000000000..854572fcd --- /dev/null +++ b/packages/filecoin-client/src/chain.js @@ -0,0 +1,49 @@ +import { connect } from '@ucanto/client' +import { CAR, HTTP } from '@ucanto/transport' + +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { services } from './service.js' + +/** + * @typedef {import('@ucanto/interface').ConnectionView} ConnectionView + */ + +/** @type {ConnectionView} */ +export const connection = connect({ + id: services.CHAIN.principal, + codec: CAR.outbound, + channel: HTTP.open({ + url: services.CHAIN.url, + method: 'POST', + }), +}) + +/** + * Get chain information for a given a piece.. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('./types.js').RequestOptions} [options] + */ +export async function chainInfo( + { issuer, with: resource, proofs, audience }, + piece, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const invocation = FilecoinCapabilities.chainInfo.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.STORE_FRONT.principal, + with: resource, + nb: { + piece, + }, + proofs, + }) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/index.js b/packages/filecoin-client/src/index.js new file mode 100644 index 000000000..2794b952a --- /dev/null +++ b/packages/filecoin-client/src/index.js @@ -0,0 +1,4 @@ +export * as Storefront from './storefront.js' +export * as Aggregator from './aggregator.js' +export * as Broker from './broker.js' +export * as Chain from './chain.js' diff --git a/packages/filecoin-client/src/service.js b/packages/filecoin-client/src/service.js new file mode 100644 index 000000000..45697de2f --- /dev/null +++ b/packages/filecoin-client/src/service.js @@ -0,0 +1,28 @@ +import * as DID from '@ipld/dag-ucan/did' + +/** + * @typedef {import('./types').SERVICE} Service + * @typedef {import('./types').ServiceConfig} ServiceConfig + */ + +/** + * @type {Record} + */ +export const services = { + STORE_FRONT: { + url: new URL('https://up.web3.storage'), + principal: DID.parse('did:web:web3.storage'), + }, + AGGREGATOR: { + url: new URL('https://aggregator.web3.storage'), + principal: DID.parse('did:web:web3.storage'), + }, + BROKER: { + url: new URL('https://spade-proxy.web3.storage'), + principal: DID.parse('did:web:spade.web3.storage'), + }, + CHAIN: { + url: new URL('https://spade-proxy.web3.storage'), + principal: DID.parse('did:web:spade.web3.storage'), + }, +} diff --git a/packages/filecoin-client/src/storefront.js b/packages/filecoin-client/src/storefront.js new file mode 100644 index 000000000..20938e495 --- /dev/null +++ b/packages/filecoin-client/src/storefront.js @@ -0,0 +1,52 @@ +import { connect } from '@ucanto/client' +import { CAR, HTTP } from '@ucanto/transport' + +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { services } from './service.js' + +/** + * @typedef {import('@ucanto/interface').ConnectionView} ConnectionView + */ + +/** @type {ConnectionView} */ +export const connection = connect({ + id: services.STORE_FRONT.principal, + codec: CAR.outbound, + channel: HTTP.open({ + url: services.STORE_FRONT.url, + method: 'POST', + }), +}) + +/** + * Add a piece to the filecoin pipeline. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {import('multiformats').UnknownLink} content + * @param {import('./types.js').RequestOptions} [options] + */ +export async function filecoinAdd( + { issuer, with: resource, proofs, audience }, + piece, + content, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const invocation = FilecoinCapabilities.filecoinAdd.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.STORE_FRONT.principal, + with: resource, + nb: { + content: content, + piece, + }, + proofs, + }) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/types.js b/packages/filecoin-client/src/types.js new file mode 100644 index 000000000..336ce12bb --- /dev/null +++ b/packages/filecoin-client/src/types.js @@ -0,0 +1 @@ +export {} diff --git a/packages/filecoin-client/src/types.ts b/packages/filecoin-client/src/types.ts new file mode 100644 index 000000000..09564a93b --- /dev/null +++ b/packages/filecoin-client/src/types.ts @@ -0,0 +1,82 @@ +import { + ConnectionView, + ServiceMethod, + Signer, + Proof, + DID, + Principal, +} from '@ucanto/interface' +import { + FilecoinAdd, + FilecoinAddSuccess, + FilecoinAddFailure, + PieceAdd, + PieceAddSuccess, + PieceAddFailure, + AggregateAdd, + AggregateAddSuccess, + AggregateAddFailure, + ChainInfo, + ChainInfoSuccess, + ChainInfoFailure, +} from '@web3-storage/capabilities/types' + +export type SERVICE = 'STORE_FRONT' | 'AGGREGATOR' | 'BROKER' | 'CHAIN' +export interface ServiceConfig { + url: URL + principal: Principal +} + +export interface InvocationConfig { + /** + * Signing authority that is issuing the UCAN invocation(s). + */ + issuer: Signer + /** + * The principal delegated to in the current UCAN. + */ + audience?: Principal + /** + * The resource the invocation applies to. + */ + with: DID + /** + * Proof(s) the issuer has the capability to perform the action. + */ + proofs?: Proof[] +} + +export interface StorefrontService { + filecoin: { + add: ServiceMethod + } +} + +export interface AggregatorService { + piece: { + add: ServiceMethod + } +} + +export interface BrokerService { + aggregate: { + add: ServiceMethod + } +} + +export interface ChainService { + chain: { + info: ServiceMethod + } +} + +export interface DealConfig { + tenantId: string + label?: string +} + +export interface RequestOptions extends Connectable {} + +export interface Connectable> { + connection?: ConnectionView +} diff --git a/packages/filecoin-client/test/aggregator.test.js b/packages/filecoin-client/test/aggregator.test.js new file mode 100644 index 000000000..33b59fa5d --- /dev/null +++ b/packages/filecoin-client/test/aggregator.test.js @@ -0,0 +1,210 @@ +import assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { pieceAdd } from '../src/aggregator.js' + +import { randomCargo } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' +import { OperationFailed, OperationErrorName } from './helpers/errors.js' +import { serviceProvider as aggregatorService } from './fixtures.js' + +describe('piece.add', () => { + it('storefront adds a filecoin piece to aggregator, getting the piece queued', async () => { + const { storefront } = await getContext() + + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + const group = 'group' + + /** @type {import('@web3-storage/capabilities/types').PieceAddSuccess} */ + const pieceAddResponse = { + status: 'queued', + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + piece: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.pieceAdd, + // @ts-expect-error not failure type expected because of assert throw + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), storefront.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.pieceAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + // piece link + assert.ok(invCap.nb?.piece.equals(cargo.link.link())) + // group + assert.strictEqual(invCap.nb?.group, group) + + // Create effect for receipt with self signed queued operation + const fx = await FilecoinCapabilities.pieceAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + }) + .delegate() + + return Server.ok(pieceAddResponse).join(fx.link()) + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await pieceAdd( + { + issuer: storefront, + with: storefront.did(), + audience: aggregatorService, + }, + cargo.link.link(), + group, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, pieceAddResponse) + // includes effect fx in receipt + assert.ok(res.fx.join) + }) + + it('aggregator self invokes add a filecoin piece to accept the piece queued', async () => { + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + const group = 'group' + + /** @type {import('@web3-storage/capabilities/types').PieceAddSuccess} */ + const pieceAddResponse = { + status: 'accepted', + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + piece: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.pieceAdd, + handler: async ({ invocation }) => { + assert.strictEqual(invocation.issuer.did(), aggregatorService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.pieceAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + // piece link + assert.ok(invCap.nb?.piece.equals(cargo.link.link())) + // group + assert.strictEqual(invCap.nb?.group, group) + + return Server.ok(pieceAddResponse) + }, + }), + }, + }) + + // self invoke piece/add from aggregator + const res = await pieceAdd( + { + issuer: aggregatorService, + with: aggregatorService.did(), + audience: aggregatorService, + }, + cargo.link.link(), + group, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, pieceAddResponse) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) + + it('aggregator self invokes add a filecoin piece to reject the piece queued', async () => { + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + const group = 'group' + + /** @type {import('@web3-storage/capabilities/types').PieceAddFailure} */ + const pieceAddResponse = new OperationFailed( + 'failed to add to aggregate', + cargo.link + ) + + // Create Ucanto service + const service = mockService({ + piece: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.pieceAdd, + handler: async ({ invocation }) => { + assert.strictEqual(invocation.issuer.did(), aggregatorService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.pieceAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + // piece link + assert.ok(invCap.nb?.piece.equals(cargo.link.link())) + // group + assert.strictEqual(invCap.nb?.group, group) + + return { + error: pieceAddResponse, + } + }, + }), + }, + }) + + // self invoke piece add from aggregator + const res = await pieceAdd( + { + issuer: aggregatorService, + with: aggregatorService.did(), + audience: aggregatorService, + }, + cargo.link.link(), + group, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.error) + // @ts-expect-error no name inferred + assert.deepEqual(res.out.error.name, OperationErrorName) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) +}) + +async function getContext() { + const storefront = await Signer.generate() + + return { storefront } +} + +/** + * @param {Partial< + * import('../src/types').AggregatorService + * >} service + */ +function getConnection(service) { + const server = Server.create({ + id: aggregatorService, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: aggregatorService, + codec: CAR.outbound, + channel: server, + }) + + return { connection } +} diff --git a/packages/filecoin-client/test/broker.test.js b/packages/filecoin-client/test/broker.test.js new file mode 100644 index 000000000..e6cd2cb60 --- /dev/null +++ b/packages/filecoin-client/test/broker.test.js @@ -0,0 +1,247 @@ +import assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import { CBOR } from '@ucanto/core' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { aggregateAdd } from '../src/broker.js' + +import { randomAggregate } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' +import { OperationFailed, OperationErrorName } from './helpers/errors.js' +import { serviceProvider as brokerService } from './fixtures.js' + +describe('aggregate.add', () => { + it('aggregator adds an aggregate piece to the broker, getting the piece queued', async () => { + const { aggregator } = await getContext() + + // generate aggregate to add + const { pieces, aggregate } = await randomAggregate(100, 100) + const offer = pieces.map((p) => p.link) + const offerBlock = await CBOR.write(offer) + const dealConfig = { + tenantId: 'web3.storage', + } + /** @type {import('@web3-storage/capabilities/types').AggregateAddSuccess} */ + const aggregateAddResponse = { + status: 'queued', + } + + // Create Ucanto service + const service = mockService({ + aggregate: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.aggregateAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), aggregator.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual( + invCap.can, + FilecoinCapabilities.aggregateAdd.can + ) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + + // piece link + assert.ok(invCap.nb.piece.equals(aggregate.link.link())) + + // Validate block inline exists + const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) + assert.ok( + invocationBlocks.find((b) => b.cid.equals(offerBlock.cid)) + ) + + // Create effect for receipt with self signed queued operation + const fx = await FilecoinCapabilities.aggregateAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + }) + .delegate() + + return Server.ok(aggregateAddResponse).join(fx.link()) + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await aggregateAdd( + { + issuer: aggregator, + with: aggregator.did(), + audience: brokerService, + }, + aggregate.link.link(), + offer, + dealConfig, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, aggregateAddResponse) + // includes effect fx in receipt + assert.ok(res.fx.join) + }) + + it('broker self invokes add an aggregate piece to accept the piece queued', async () => { + // generate aggregate to add + const { pieces, aggregate } = await randomAggregate(100, 100) + const offer = pieces.map((p) => p.link) + const offerBlock = await CBOR.write(offer) + const dealConfig = { + tenantId: 'web3.storage', + } + /** @type {import('@web3-storage/capabilities/types').AggregateAddSuccess} */ + const aggregateAddResponse = { + status: 'accepted', + } + + // Create Ucanto service + const service = mockService({ + aggregate: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.aggregateAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), brokerService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual( + invCap.can, + FilecoinCapabilities.aggregateAdd.can + ) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + + // piece link + assert.ok(invCap.nb.piece.equals(aggregate.link.link())) + + // Validate block inline exists + const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) + assert.ok( + invocationBlocks.find((b) => b.cid.equals(offerBlock.cid)) + ) + + return Server.ok(aggregateAddResponse) + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await aggregateAdd( + { + issuer: brokerService, + with: brokerService.did(), + audience: brokerService, + }, + aggregate.link.link(), + offer, + dealConfig, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, aggregateAddResponse) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) + + it('broker self invokes add an aggregate piece to reject the piece queued', async () => { + // generate aggregate to add + const { pieces, aggregate } = await randomAggregate(100, 100) + const offer = pieces.map((p) => p.link) + const offerBlock = await CBOR.write(offer) + const dealConfig = { + tenantId: 'web3.storage', + } + /** @type {import('@web3-storage/capabilities/types').AggregateAddFailure} */ + const aggregateAddResponse = new OperationFailed( + 'failed to add to aggregate', + aggregate.link + ) + + // Create Ucanto service + const service = mockService({ + aggregate: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.aggregateAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), brokerService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual( + invCap.can, + FilecoinCapabilities.aggregateAdd.can + ) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + + // piece link + assert.ok(invCap.nb.piece.equals(aggregate.link.link())) + + // Validate block inline exists + const invocationBlocks = Array.from(invocation.iterateIPLDBlocks()) + assert.ok( + invocationBlocks.find((b) => b.cid.equals(offerBlock.cid)) + ) + + return { + error: aggregateAddResponse, + } + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await aggregateAdd( + { + issuer: brokerService, + with: brokerService.did(), + audience: brokerService, + }, + aggregate.link.link(), + offer, + dealConfig, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.error) + // @ts-expect-error no name inferred + assert.deepEqual(res.out.error.name, OperationErrorName) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) +}) + +async function getContext() { + const aggregator = await Signer.generate() + + return { aggregator } +} + +/** + * @param {Partial< + *import('../src/types').BrokerService + * >} service + */ +function getConnection(service) { + const server = Server.create({ + id: brokerService, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: brokerService, + codec: CAR.outbound, + channel: server, + }) + + return { connection } +} diff --git a/packages/filecoin-client/test/chain.test.js b/packages/filecoin-client/test/chain.test.js new file mode 100644 index 000000000..87e91f588 --- /dev/null +++ b/packages/filecoin-client/test/chain.test.js @@ -0,0 +1,88 @@ +import assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { chainInfo } from '../src/chain.js' + +import { randomCargo } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' +import { serviceProvider as chainService } from './fixtures.js' + +describe('chain.info', () => { + it('storefront gets info of a filecoin piece from chain', async () => { + const { storefront } = await getContext() + + // Generate cargo to get info + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').ChainInfoSuccess} */ + const chainInfoResponse = { + status: 'queued', + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + chain: { + info: Server.provideAdvanced({ + capability: FilecoinCapabilities.chainInfo, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), storefront.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.chainInfo.can) + assert.equal(invCap.with, invocation.issuer.did()) + + // piece link + assert.ok(invCap.nb?.piece.equals(cargo.link.link())) + + return Server.ok(chainInfoResponse) + }, + }), + }, + }) + + // invoke piece add from storefront + const res = await chainInfo( + { + issuer: storefront, + with: storefront.did(), + audience: chainService, + }, + cargo.link.link(), + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, chainInfoResponse) + }) +}) + +async function getContext() { + const storefront = await Signer.generate() + + return { storefront } +} + +/** + * @param {Partial< + * import('../src/types').ChainService + * >} service + */ +function getConnection(service) { + const server = Server.create({ + id: chainService, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: chainService, + codec: CAR.outbound, + channel: server, + }) + + return { connection } +} diff --git a/packages/aggregate-client/test/fixtures.js b/packages/filecoin-client/test/fixtures.js similarity index 100% rename from packages/aggregate-client/test/fixtures.js rename to packages/filecoin-client/test/fixtures.js diff --git a/packages/aggregate-client/test/helpers/block.js b/packages/filecoin-client/test/helpers/block.js similarity index 100% rename from packages/aggregate-client/test/helpers/block.js rename to packages/filecoin-client/test/helpers/block.js diff --git a/packages/aggregate-client/test/helpers/car.js b/packages/filecoin-client/test/helpers/car.js similarity index 100% rename from packages/aggregate-client/test/helpers/car.js rename to packages/filecoin-client/test/helpers/car.js diff --git a/packages/filecoin-client/test/helpers/errors.js b/packages/filecoin-client/test/helpers/errors.js new file mode 100644 index 000000000..eac436079 --- /dev/null +++ b/packages/filecoin-client/test/helpers/errors.js @@ -0,0 +1,21 @@ +import * as Server from '@ucanto/server' + +export const OperationErrorName = /** @type {const} */ ('OperationFailed') +export class OperationFailed extends Server.Failure { + /** + * @param {string} message + * @param {import('@web3-storage/data-segment').PieceLink} piece + */ + constructor(message, piece) { + super(message) + this.piece = piece + } + + get reason() { + return this.message + } + + get name() { + return OperationErrorName + } +} diff --git a/packages/aggregate-client/test/helpers/mocks.js b/packages/filecoin-client/test/helpers/mocks.js similarity index 51% rename from packages/aggregate-client/test/helpers/mocks.js rename to packages/filecoin-client/test/helpers/mocks.js index cfe92fe76..1fbc0552a 100644 --- a/packages/aggregate-client/test/helpers/mocks.js +++ b/packages/filecoin-client/test/helpers/mocks.js @@ -5,19 +5,26 @@ const notImplemented = () => { } /** - * @param {Partial<{ - * aggregate: Partial - * offer: Partial - * }>} impl + * @param {Partial< + * import('../../src/types').StorefrontService & + * import('../../src/types').AggregatorService & + * import('../../src/types').BrokerService & + * import('../../src/types').ChainService + * >} impl */ export function mockService(impl) { return { + filecoin: { + add: withCallCount(impl.filecoin?.add ?? notImplemented), + }, + piece: { + add: withCallCount(impl.piece?.add ?? notImplemented), + }, aggregate: { - offer: withCallCount(impl.aggregate?.offer ?? notImplemented), - get: withCallCount(impl.aggregate?.get ?? notImplemented), + add: withCallCount(impl.aggregate?.add ?? notImplemented), }, - offer: { - arrange: withCallCount(impl.offer?.arrange ?? notImplemented), + chain: { + info: withCallCount(impl.chain?.info ?? notImplemented), }, } } diff --git a/packages/aggregate-client/test/helpers/random.js b/packages/filecoin-client/test/helpers/random.js similarity index 98% rename from packages/aggregate-client/test/helpers/random.js rename to packages/filecoin-client/test/helpers/random.js index f9e9c6368..5694cbec5 100644 --- a/packages/aggregate-client/test/helpers/random.js +++ b/packages/filecoin-client/test/helpers/random.js @@ -61,6 +61,7 @@ export async function randomCargo(length, size) { return { link: piece.link, height: piece.height, + content: car.cid, size: piece.size, } }) diff --git a/packages/filecoin-client/test/storefront.test.js b/packages/filecoin-client/test/storefront.test.js new file mode 100644 index 000000000..35b8eef27 --- /dev/null +++ b/packages/filecoin-client/test/storefront.test.js @@ -0,0 +1,209 @@ +import assert from 'assert' +import * as Signer from '@ucanto/principal/ed25519' +import * as Client from '@ucanto/client' +import * as Server from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' + +import { filecoinAdd } from '../src/storefront.js' + +import { randomCargo } from './helpers/random.js' +import { mockService } from './helpers/mocks.js' +import { OperationFailed, OperationErrorName } from './helpers/errors.js' +import { serviceProvider as storefrontService } from './fixtures.js' + +describe('filecoin.add', () => { + it('agent adds a filecoin piece to a storefront, getting the piece queued', async () => { + const { agent } = await getContext() + + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').FilecoinAddSuccess} */ + const filecoinAddResponse = { + status: 'queued', + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + filecoin: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.filecoinAdd, + // @ts-expect-error not failure type expected because of assert throw + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), agent.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.filecoinAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + // piece link + assert.ok(invCap.nb.piece.equals(cargo.link.link())) + // content + assert.ok(invCap.nb.content.equals(cargo.content.link())) + + // Create effect for receipt with self signed queued operation + const fx = await FilecoinCapabilities.filecoinAdd + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + }) + .delegate() + + return Server.ok(filecoinAddResponse).join(fx.link()) + }, + }), + }, + }) + + const res = await filecoinAdd( + { + issuer: agent, + with: agent.did(), + audience: storefrontService, + }, + cargo.link.link(), + cargo.content.link(), + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, filecoinAddResponse) + // includes effect fx in receipt + assert.ok(res.fx.join) + }) + + it('storefront self invokes add a filecoin piece to accept the piece queued', async () => { + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').FilecoinAddSuccess} */ + const filecoinAddResponse = { + status: 'accepted', + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + filecoin: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.filecoinAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), storefrontService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.filecoinAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + // piece link + assert.ok(invCap.nb.piece.equals(cargo.link.link())) + // content + assert.ok(invCap.nb.content.equals(cargo.content.link())) + + return Server.ok(filecoinAddResponse) + }, + }), + }, + }) + + // self invoke filecoin/add from storefront + const res = await filecoinAdd( + { + issuer: storefrontService, + with: storefrontService.did(), + audience: storefrontService, + }, + cargo.link.link(), + cargo.content.link(), + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.deepEqual(res.out.ok, filecoinAddResponse) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) + + it('storefront self invokes add a filecoin piece to reject the piece queued', async () => { + // Generate cargo to add + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').FilecoinAddFailure} */ + const filecoinAddResponse = new OperationFailed( + 'failed to verify piece', + cargo.link + ) + + // Create Ucanto service + const service = mockService({ + filecoin: { + add: Server.provideAdvanced({ + capability: FilecoinCapabilities.filecoinAdd, + handler: async ({ invocation, context }) => { + assert.strictEqual(invocation.issuer.did(), storefrontService.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, FilecoinCapabilities.filecoinAdd.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + // piece link + assert.ok(invCap.nb.piece.equals(cargo.link.link())) + // content + assert.ok(invCap.nb.content.equals(cargo.content.link())) + + return { + error: filecoinAddResponse, + } + }, + }), + }, + }) + + // self invoke filecoin/add from storefront + const res = await filecoinAdd( + { + issuer: storefrontService, + with: storefrontService.did(), + audience: storefrontService, + }, + cargo.link.link(), + cargo.content.link(), + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.error) + // @ts-expect-error no name inferred + assert.deepEqual(res.out.error.name, OperationErrorName) + // does not include effect fx in receipt + assert.ok(!res.fx.join) + }) +}) + +async function getContext() { + const agent = await Signer.generate() + + return { agent } +} + +/** + * @param {Partial< + *import('../src/types').StorefrontService + * >} service + */ +function getConnection(service) { + const server = Server.create({ + id: storefrontService, + service, + codec: CAR.inbound, + }) + const connection = Client.connect({ + id: storefrontService, + codec: CAR.outbound, + channel: server, + }) + + return { connection } +} diff --git a/packages/aggregate-client/tsconfig.json b/packages/filecoin-client/tsconfig.json similarity index 100% rename from packages/aggregate-client/tsconfig.json rename to packages/filecoin-client/tsconfig.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a30efc80..ab89acaec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -347,6 +347,125 @@ importers: specifier: ^10.2.0 version: 10.2.0 + packages/filecoin-api: + dependencies: + '@ucanto/client': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/core': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/interface': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/server': + specifier: ^8.0.0 + version: 8.0.1 + '@ucanto/transport': + specifier: ^8.0.0 + version: 8.0.0 + '@web3-storage/capabilities': + specifier: workspace:^ + version: link:../capabilities + '@web3-storage/data-segment': + specifier: ^2.2.0 + version: 2.2.0 + devDependencies: + '@ipld/car': + specifier: ^5.1.1 + version: 5.1.1 + '@types/mocha': + specifier: ^10.0.1 + version: 10.0.1 + '@ucanto/principal': + specifier: ^8.0.0 + version: 8.0.0 + '@web-std/blob': + specifier: ^3.0.4 + version: 3.0.4 + '@web3-storage/filecoin-client': + specifier: workspace:^ + version: link:../filecoin-client + hd-scripts: + specifier: ^4.1.0 + version: 4.1.0 + mocha: + specifier: ^10.2.0 + version: 10.2.0 + multiformats: + specifier: ^11.0.2 + version: 11.0.2 + + packages/filecoin-client: + dependencies: + '@ipld/dag-cbor': + specifier: ^9.0.0 + version: 9.0.0 + '@ipld/dag-ucan': + specifier: ^3.3.2 + version: 3.3.2 + '@ucanto/client': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/core': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/interface': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/transport': + specifier: ^8.0.0 + version: 8.0.0 + '@web3-storage/capabilities': + specifier: workspace:^ + version: link:../capabilities + devDependencies: + '@ipld/car': + specifier: ^5.1.1 + version: 5.1.1 + '@types/assert': + specifier: ^1.5.6 + version: 1.5.6 + '@types/mocha': + specifier: ^10.0.1 + version: 10.0.1 + '@ucanto/principal': + specifier: ^8.0.0 + version: 8.0.0 + '@ucanto/server': + specifier: ^8.0.1 + version: 8.0.1 + '@web3-storage/data-segment': + specifier: ^2.2.0 + version: 2.2.0 + assert: + specifier: ^2.0.0 + version: 2.0.0 + c8: + specifier: ^7.13.0 + version: 7.13.0 + hd-scripts: + specifier: ^4.0.0 + version: 4.1.0 + hundreds: + specifier: ^0.0.9 + version: 0.0.9 + mocha: + specifier: ^10.2.0 + version: 10.2.0 + multiformats: + specifier: ^11.0.2 + version: 11.0.2 + npm-run-all: + specifier: ^4.1.5 + version: 4.1.5 + playwright-test: + specifier: ^8.1.2 + version: 8.1.2 + typescript: + specifier: 4.9.5 + version: 4.9.5 + packages/upload-api: dependencies: '@ucanto/client': diff --git a/tsconfig.json b/tsconfig.json index afabcba71..b848c7969 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -39,6 +39,7 @@ "packages/access-client", "packages/capabilities", "packages/upload-client", + "packages/filecoin-client", "packages/w3up-client" ], "excludeExternals": true,