From 51bf9aa9647a9e1cc4a197ced268a21ffb17ec74 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Mon, 17 Jul 2023 17:31:18 -0700 Subject: [PATCH 01/33] feat: introduce new administrative capabilities As described in https://hackmd.io/@tvpl/H1whErCI3, introduce a set of capabilities that will: 1) allow providers to rate limit (especially fully limit, ie block) users 2) allow providers to get information about subscriptions and consumers TODO -[] write tests for capabilities (need to pick @gozala's brain) -[] implement in-memory RateLimitsStorage -[] use it to write `upload-api` tests -[] move capabilities spec from hackmd to https://github.com/web3-storage/w3up/blob/main/spec/capabilities.md -[] port https://github.com/web3-storage/w3up/pull/801/commits/0eea3d1268893062264414b2bbbeb689ac25c6c1 and https://github.com/web3-storage/w3up/pull/801/commits/d230033c1569094738b2dcdd9c6acb5afa889c82 to this commit --- packages/capabilities/src/consumer.js | 18 ++ packages/capabilities/src/rate-limit.js | 71 ++++++++ packages/capabilities/src/subscription.js | 23 +++ .../test/capabilities/rate-limit.test.js | 155 ++++++++++++++++++ packages/upload-api/src/rate-limit/add.js | 17 ++ packages/upload-api/src/rate-limit/list.js | 17 ++ packages/upload-api/src/rate-limit/remove.js | 17 ++ packages/upload-api/src/types.ts | 10 ++ packages/upload-api/src/types/rate-limits.ts | 44 +++++ 9 files changed, 372 insertions(+) create mode 100644 packages/capabilities/src/rate-limit.js create mode 100644 packages/capabilities/src/subscription.js create mode 100644 packages/capabilities/test/capabilities/rate-limit.test.js create mode 100644 packages/upload-api/src/rate-limit/add.js create mode 100644 packages/upload-api/src/rate-limit/list.js create mode 100644 packages/upload-api/src/rate-limit/remove.js create mode 100644 packages/upload-api/src/types/rate-limits.ts diff --git a/packages/capabilities/src/consumer.js b/packages/capabilities/src/consumer.js index 98b4cf92c..01923afc2 100644 --- a/packages/capabilities/src/consumer.js +++ b/packages/capabilities/src/consumer.js @@ -24,3 +24,21 @@ export const has = capability({ ) }, }) + +/** + * Capability can be invoked by a provider to get information about a consumer. + */ +export const get = capability({ + can: 'consumer/get', + with: ProviderDID, + nb: struct({ + consumer: SpaceDID, + }), + derives: (child, parent) => { + return ( + and(equalWith(child, parent)) || + and(equal(child.nb.consumer, parent.nb.consumer, 'consumer')) || + ok({}) + ) + }, +}) diff --git a/packages/capabilities/src/rate-limit.js b/packages/capabilities/src/rate-limit.js new file mode 100644 index 000000000..5ed553a70 --- /dev/null +++ b/packages/capabilities/src/rate-limit.js @@ -0,0 +1,71 @@ +/** + * Rate Limit Capabilities + * + * These can be imported directly with: + * ```js + * import * as RateLimit from '@web3-storage/capabilities/rate-limit' + * ``` + * + * @module + */ +import { capability, DID, struct, Schema, ok } from '@ucanto/validator' +import { equalWith, and, equal } from './utils.js' + +// e.g. did:web:web3.storage or did:web:staging.web3.storage +export const Provider = DID + +/** + * Capability can be invoked by an agent to add a rate limit to a subject. + */ +export const add = capability({ + can: 'rate-limit/add', + with: Provider, + nb: struct({ + subject: Schema.string(), + rate: Schema.number() + }), + derives: (child, parent) => { + return ( + and(equalWith(child, parent)) || + and(equal(child.nb.subject, parent.nb.subject, 'subject')) || + and(equal(child.nb.rate, parent.nb.rate, 'rate')) || + ok({}) + ) + }, +}) + +/** + * Capability can be invoked by an agent to remove rate limits from a subject. + */ +export const remove = capability({ + can: 'rate-limit/remove', + with: Provider, + nb: struct({ + id: Schema.string() + }), + derives: (child, parent) => { + return ( + and(equalWith(child, parent)) || + and(equal(child.nb.id, parent.nb.id, 'id')) || + ok({}) + ) + }, +}) + +/** + * Capability can be invoked by an agent to list rate limits on a subject + */ +export const list = capability({ + can: 'rate-limit/list', + with: Provider, + nb: struct({ + subject: Schema.string() + }), + derives: (child, parent) => { + return ( + and(equalWith(child, parent)) || + and(equal(child.nb.subject, parent.nb.subject, 'subject')) || + ok({}) + ) + }, +}) diff --git a/packages/capabilities/src/subscription.js b/packages/capabilities/src/subscription.js new file mode 100644 index 000000000..9606353d2 --- /dev/null +++ b/packages/capabilities/src/subscription.js @@ -0,0 +1,23 @@ +import { capability, DID, struct, ok, Schema } from '@ucanto/validator' +import { equalWith, and, equal } from './utils.js' + +// e.g. did:web:web3.storage or did:web:staging.web3.storage +export const ProviderDID = DID.match({ method: 'web' }) + +/** + * Capability can be invoked by a provider to get information about a subscription. + */ +export const get = capability({ + can: 'subscription/get', + with: ProviderDID, + nb: struct({ + subscription: Schema.string(), + }), + derives: (child, parent) => { + return ( + and(equalWith(child, parent)) || + and(equal(child.nb.subscription, parent.nb.subscription, 'consumer')) || + ok({}) + ) + }, +}) diff --git a/packages/capabilities/test/capabilities/rate-limit.test.js b/packages/capabilities/test/capabilities/rate-limit.test.js new file mode 100644 index 000000000..c9c9aecb3 --- /dev/null +++ b/packages/capabilities/test/capabilities/rate-limit.test.js @@ -0,0 +1,155 @@ +import assert from 'assert' +import { access } from '@ucanto/validator' +import { Verifier } from '@ucanto/principal/ed25519' +import * as RateLimit from '../../src/rate-limit.js' +import { bob, service, alice } from '../helpers/fixtures.js' +import { createAuthorization } from '../helpers/utils.js' + +const provider = 'did:web:test.web3.storage' + +describe('rate-limit/add', function () { + it('can by invoked as account', async function () { + const agent = alice + const space = bob + const auth = RateLimit.add.invoke({ + issuer: agent, + audience: service, + with: provider, + nb: { + subject: space.did(), + rate: 0 + }, + // TODO: check in with @gozala about whether passing provider as account makes sense + proofs: await createAuthorization({ agent, service, account: provider }), + }) + const result = await access(await auth.delegate(), { + capability: RateLimit.add, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail('error in self issue') + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'rate-limit/add') + assert.deepEqual(result.ok.capability.nb, { + resource: space.did(), + rate: 0, + }) + } + }) + + it('fails without account delegation', async function () { + const agent = alice + const space = bob + const auth = RateLimit.add.invoke({ + issuer: agent, + audience: service, + with: provider, + nb: { + subject: space.did(), + rate: 0 + }, + }) + + const result = await access(await auth.delegate(), { + capability: RateLimit.add, + principal: Verifier, + authority: service, + }) + + assert.equal(result.error?.message.includes('not authorized'), true) + }) + + it('requires nb.resource', async function () { + assert.throws(() => { + RateLimit.add.invoke({ + issuer: alice, + audience: service, + with: provider, + // @ts-ignore + nb: { + rate: 0 + }, + }) + }, /Error: Invalid 'nb' - Object contains invalid field "resource"/) + }) + + it('requires nb.rate', async function () { + assert.throws(() => { + RateLimit.add.invoke({ + issuer: alice, + audience: service, + with: provider, + // @ts-ignore + nb: { + subject: alice.did() + }, + }) + }, /Error: Invalid 'nb' - Object contains invalid field "rate"/) + }) +}) + +describe('rate-limit/remove', function () { + it('can by invoked as account', async function () { + const agent = alice + const space = bob + const auth = RateLimit.remove.invoke({ + issuer: agent, + audience: service, + with: provider, + nb: { + id: '123' + }, + // TODO: check in with @gozala about whether passing provider as account makes sense + proofs: await createAuthorization({ agent, service, account: provider }), + }) + const result = await access(await auth.delegate(), { + capability: RateLimit.remove, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail('error in self issue') + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'rate-limit/remove') + assert.deepEqual(result.ok.capability.nb, { + resource: space.did() + }) + } + }) + + it('fails without account delegation', async function () { + const agent = alice + const auth = RateLimit.remove.invoke({ + issuer: agent, + audience: service, + with: provider, + nb: { + id: '123' + }, + }) + + const result = await access(await auth.delegate(), { + capability: RateLimit.remove, + principal: Verifier, + authority: service, + }) + + assert.equal(result.error?.message.includes('not authorized'), true) + }) + + it('requires nb.resource', async function () { + assert.throws(() => { + RateLimit.remove.invoke({ + issuer: alice, + audience: service, + with: provider, + // @ts-ignore + nb: { + }, + }) + }, /Error: Invalid 'nb' - Object contains invalid field "resource"/) + }) +}) \ No newline at end of file diff --git a/packages/upload-api/src/rate-limit/add.js b/packages/upload-api/src/rate-limit/add.js new file mode 100644 index 000000000..556d3a3cc --- /dev/null +++ b/packages/upload-api/src/rate-limit/add.js @@ -0,0 +1,17 @@ +import * as Server from '@ucanto/server' +import * as API from '../types.js' +import * as RateLimit from '@web3-storage/capabilities/rate-limit' + +/** + * @param {API.RateLimitsServiceContext} ctx + */ +export const provide = (ctx) => + Server.provide(RateLimit.add, (input) => add(input, ctx)) + +/** + * @param {API.Input} input + * @param {API.RateLimitsServiceContext} ctx + */ +export const add = async ({ capability }, ctx) => { + return ctx.rateLimitsStorage.add(capability.nb.subject, capability.nb.rate) +} diff --git a/packages/upload-api/src/rate-limit/list.js b/packages/upload-api/src/rate-limit/list.js new file mode 100644 index 000000000..7c93757a4 --- /dev/null +++ b/packages/upload-api/src/rate-limit/list.js @@ -0,0 +1,17 @@ +import * as Server from '@ucanto/server' +import * as API from '../types.js' +import * as RateLimit from '@web3-storage/capabilities/rate-limit' + +/** + * @param {API.RateLimitsServiceContext} ctx + */ +export const provide = (ctx) => + Server.provide(RateLimit.list, (input) => list(input, ctx)) + +/** + * @param {API.Input} input + * @param {API.RateLimitsServiceContext} ctx + */ +export const list = async ({ capability }, ctx) => { + return ctx.rateLimitsStorage.list(capability.nb.subject) +} diff --git a/packages/upload-api/src/rate-limit/remove.js b/packages/upload-api/src/rate-limit/remove.js new file mode 100644 index 000000000..4dac07e31 --- /dev/null +++ b/packages/upload-api/src/rate-limit/remove.js @@ -0,0 +1,17 @@ +import * as Server from '@ucanto/server' +import * as API from '../types.js' +import * as RateLimit from '@web3-storage/capabilities/rate-limit' + +/** + * @param {API.RateLimitsServiceContext} ctx + */ +export const provide = (ctx) => + Server.provide(RateLimit.remove, (input) => remove(input, ctx)) + +/** + * @param {API.Input} input + * @param {API.RateLimitsServiceContext} ctx + */ +export const remove = async ({ capability }, ctx) => { + return ctx.rateLimitsStorage.remove(capability.nb.id) +} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index a34b167c2..ad5a178dc 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -21,6 +21,8 @@ import { Signer as EdSigner } from '@ucanto/principal/ed25519' import { ToString, UnknownLink } from 'multiformats' import { DelegationsStorage as Delegations } from './types/delegations' import { ProvisionsStorage as Provisions } from './types/provisions' +import { RateLimitsStorage as RateLimits } from './types/rate-limits' + export type ValidationEmailSend = { to: string @@ -83,6 +85,9 @@ export type { DelegationsStorage, Query as DelegationsStorageQuery, } from './types/delegations' +export type { + RateLimitsStorage +} from './types/rate-limits' export interface Service { store: { @@ -189,6 +194,10 @@ export interface ProviderServiceContext { provisionsStorage: Provisions } +export interface RateLimitsServiceContext { + rateLimitsStorage: RateLimits +} + export interface ServiceContext extends AccessServiceContext, ConsoleServiceContext, @@ -197,6 +206,7 @@ export interface ServiceContext ProviderServiceContext, SpaceServiceContext, StoreServiceContext, + RateLimitsServiceContext, UploadServiceContext {} export interface UcantoServerContext extends ServiceContext { diff --git a/packages/upload-api/src/types/rate-limits.ts b/packages/upload-api/src/types/rate-limits.ts new file mode 100644 index 000000000..fe8673bd8 --- /dev/null +++ b/packages/upload-api/src/types/rate-limits.ts @@ -0,0 +1,44 @@ +import * as Ucanto from '@ucanto/interface' + +export type RateLimitID = string + +export interface RateLimit { + /** + * Identifier of rate limited subject - could be a DID, an email address, a URL or any other string. + */ + subject?: string + /** + * Identifier of this rate limit - can used to remove a limit. + */ + RateLimitID?: string + /** + * Rate limit applied to the subject - intentionally unitless, should be interpreted by consumer. + */ + rate: number +} + +/** + * stores instances of a storage provider being consumed by a consumer + */ +export interface RateLimitsStorage { + /** + * Add rate limit for subject. + * + * @param subject identifier for subject - could be a DID, a URI, or anything else + * @param rate a limit to be interpreted by the consuming system - intentionally unitless + */ + add: (subject: string, rate: number) => Promise> + + /** + * Returns rate limits on subject. + * + * @param subject identifier for subject - could be a DID, a URI, or anything else + * @returns a list of rate limits for the idenfied subject + */ + list: (subject: string) => Promise> + + /** + * Remove a rate limit with a given ID. + */ + remove: (id: RateLimitID) => Promise> +} From 56d1db1f78f00cfce203fa39853e0cf6dc7d1074 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Tue, 18 Jul 2023 16:24:58 -0700 Subject: [PATCH 02/33] feat: port blocking logic to this implementation originally implemented in https://github.com/web3-storage/w3up/pull/801, this allows us to block space allocation for a particular space or account creation for a particular email or domain --- packages/upload-api/src/access/authorize.js | 13 +++++++++++++ packages/upload-api/src/provider-add.js | 17 ++++++++++++++++- packages/upload-api/src/space-allocate.js | 9 +++++++++ packages/upload-api/src/types.ts | 3 +++ packages/upload-api/src/types/rate-limits.ts | 7 ++++++- packages/upload-api/src/utils/did-mailto.js | 10 ++++++++++ 6 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 packages/upload-api/src/utils/did-mailto.js diff --git a/packages/upload-api/src/access/authorize.js b/packages/upload-api/src/access/authorize.js index decf7893c..2b59d28a7 100644 --- a/packages/upload-api/src/access/authorize.js +++ b/packages/upload-api/src/access/authorize.js @@ -3,6 +3,7 @@ import * as API from '../types.js' import * as Access from '@web3-storage/capabilities/access' import * as DidMailto from '@web3-storage/did-mailto' import { delegationToString } from '@web3-storage/access/encoding' +import { emailAndDomainFromMailtoDid } from '../utils/did-mailto.js' /** * @param {API.AccessServiceContext} ctx @@ -15,6 +16,18 @@ export const provide = (ctx) => * @param {API.AccessServiceContext} ctx */ export const authorize = async ({ capability }, ctx) => { + const isBlocked = await ctx.rateLimitsStorage.areAnyBlocked( + emailAndDomainFromMailtoDid(/** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */(capability.nb.iss)) + ) + if (isBlocked.error || isBlocked.ok) { + return { + error: { + name: 'AccountBlocked', + message: `Account identified by {capability.nb.iss} is blocked` + } + } + } + /** * We issue `access/confirm` invocation which will * get embedded in the URL that we send to the user. When user clicks the diff --git a/packages/upload-api/src/provider-add.js b/packages/upload-api/src/provider-add.js index 955eee3be..6cb8932ad 100644 --- a/packages/upload-api/src/provider-add.js +++ b/packages/upload-api/src/provider-add.js @@ -2,6 +2,7 @@ import * as API from './types.js' import * as Server from '@ucanto/server' import { Provider } from '@web3-storage/capabilities' import * as validator from '@ucanto/validator' +import { emailAndDomainFromMailtoDid } from './utils/did-mailto.js' /** * @param {API.ProviderServiceContext} ctx @@ -15,7 +16,10 @@ export const provide = (ctx) => */ export const add = async ( { capability, invocation }, - { provisionsStorage: provisions } + { + provisionsStorage: provisions, + rateLimitsStorage: rateLimits + } ) => { const { nb: { consumer, provider }, @@ -29,6 +33,17 @@ export const add = async ( }, } } + const isBlocked = await rateLimits.areAnyBlocked( + emailAndDomainFromMailtoDid(/** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */(accountDID)) + ) + if (isBlocked.error || isBlocked.ok) { + return { + error: { + name: 'AccountBlocked', + message: `Account identified by {capability.nb.iss} is blocked` + } + } + } if (!provisions.services.includes(provider)) { return { error: { diff --git a/packages/upload-api/src/space-allocate.js b/packages/upload-api/src/space-allocate.js index 41a970b3f..3fc9f3acf 100644 --- a/packages/upload-api/src/space-allocate.js +++ b/packages/upload-api/src/space-allocate.js @@ -10,6 +10,15 @@ import * as Space from '@web3-storage/capabilities/space' */ export const allocate = async ({ capability }, context) => { const { with: space, nb } = capability + const isBlocked = await context.rateLimitsStorage.areAnyBlocked([space]) + if (isBlocked.error || isBlocked.ok) { + return { + error: { + name: 'InsufficientStorage', + message: `${space} is blocked` + } + } + } const { size } = nb const result = await context.provisionsStorage.hasStorageProvider(space) if (result.ok) { diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index ad5a178dc..c41ffc300 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -171,6 +171,7 @@ export type AccessServiceContext = AccessClaimContext & { email: Email url: URL provisionsStorage: Provisions + rateLimitsStorage: RateLimits } export interface ConsumerServiceContext { @@ -188,10 +189,12 @@ export interface ConsoleServiceContext {} export interface SpaceServiceContext { provisionsStorage: Provisions delegationsStorage: Delegations + rateLimitsStorage: RateLimits } export interface ProviderServiceContext { provisionsStorage: Provisions + rateLimitsStorage: RateLimits } export interface RateLimitsServiceContext { diff --git a/packages/upload-api/src/types/rate-limits.ts b/packages/upload-api/src/types/rate-limits.ts index fe8673bd8..0b6247e45 100644 --- a/packages/upload-api/src/types/rate-limits.ts +++ b/packages/upload-api/src/types/rate-limits.ts @@ -32,7 +32,7 @@ export interface RateLimitsStorage { /** * Returns rate limits on subject. * - * @param subject identifier for subject - could be a DID, a URI, or anything else + * @param subjects a subject identifier - could be a DID, a URI, or anything else * @returns a list of rate limits for the idenfied subject */ list: (subject: string) => Promise> @@ -41,4 +41,9 @@ export interface RateLimitsStorage { * Remove a rate limit with a given ID. */ remove: (id: RateLimitID) => Promise> + + /** + * Returns true if the given subject has a limit equal to 0. + */ + areAnyBlocked: (subjects: string[]) => Promise> } diff --git a/packages/upload-api/src/utils/did-mailto.js b/packages/upload-api/src/utils/did-mailto.js new file mode 100644 index 000000000..cdaa5a918 --- /dev/null +++ b/packages/upload-api/src/utils/did-mailto.js @@ -0,0 +1,10 @@ +import * as DidMailto from '@web3-storage/did-mailto' + +/** + * @param {import("@web3-storage/did-mailto/dist/src/types").DidMailto} mailtoDid + */ +export function emailAndDomainFromMailtoDid(mailtoDid){ + const accountEmail = DidMailto.email(DidMailto.fromString(mailtoDid)) + const accountDomain = accountEmail.split('@')[1] + return [accountEmail, accountDomain] +} \ No newline at end of file From ba3cfcfc842ad4b0f0685101da6c73a3ca193286 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Tue, 18 Jul 2023 16:49:20 -0700 Subject: [PATCH 03/33] feat: add in-memory implementation of rate limits storage --- packages/upload-api/src/types.ts | 3 +- packages/upload-api/src/types/rate-limits.ts | 2 +- packages/upload-api/test/helpers/context.js | 2 + .../upload-api/test/rate-limits-storage.js | 59 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 packages/upload-api/test/rate-limits-storage.js diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index c41ffc300..88af2e791 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -86,7 +86,8 @@ export type { Query as DelegationsStorageQuery, } from './types/delegations' export type { - RateLimitsStorage + RateLimitsStorage, + RateLimit } from './types/rate-limits' export interface Service { diff --git a/packages/upload-api/src/types/rate-limits.ts b/packages/upload-api/src/types/rate-limits.ts index 0b6247e45..600ab2754 100644 --- a/packages/upload-api/src/types/rate-limits.ts +++ b/packages/upload-api/src/types/rate-limits.ts @@ -10,7 +10,7 @@ export interface RateLimit { /** * Identifier of this rate limit - can used to remove a limit. */ - RateLimitID?: string + id?: string /** * Rate limit applied to the subject - intentionally unitless, should be interpreted by consumer. */ diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index d2270a800..896ee9fdb 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -7,6 +7,7 @@ import { DudewhereBucket } from '../dude-where-bucket.js' import * as AccessVerifier from '../access-verifier.js' import { ProvisionsStorage } from '../provisions-storage.js' import { DelegationsStorage } from '../delegations-storage.js' +import { RateLimitsStorage } from '../rate-limits-storage.js' import * as Email from '../../src/utils/email.js' import { createServer, connect } from '../../src/lib.js' import * as Types from '../../src/types.js' @@ -34,6 +35,7 @@ export const createContext = async (options = {}) => { url: new URL('http://localhost:8787'), provisionsStorage: new ProvisionsStorage(options.providers), delegationsStorage: new DelegationsStorage(), + rateLimitsStorage: new RateLimitsStorage(), errorReporter: { catch(error) { assert.fail(error) diff --git a/packages/upload-api/test/rate-limits-storage.js b/packages/upload-api/test/rate-limits-storage.js new file mode 100644 index 000000000..bba84afe4 --- /dev/null +++ b/packages/upload-api/test/rate-limits-storage.js @@ -0,0 +1,59 @@ +import * as Types from '../src/types.js' + +/** + * @implements {Types.RateLimitsStorage} + */ +export class RateLimitsStorage { + constructor() { + /** + * @type {Record}} + */ + this.rateLimits = {} + this.nextID = 0 + } + + /** + * + * @param {string} subject + * @param {number} rate + * @returns + */ + async add(subject, rate) { + const id = this.nextID.toString() + this.nextID += 1 + this.rateLimits[id] = { + id, + subject, + rate + } + return { ok: {} } + } + + /** + * + * @param {string} subject + * @returns + */ + async list(subject) { + return { ok: Object.values(this.rateLimits).filter((rl) => rl.subject === subject) || [] } + } + + /** + * + * @param {string} id + */ + async remove(id) { + delete this.rateLimits[id] + return { ok: {} } + } + + /** + * + * @param {string[]} subjects + */ + async areAnyBlocked(subjects) { + return { ok: Object.values(this.rateLimits).some(({subject, rate}) => (subject && subjects.includes(subject)) && (rate === 0))} + } + + +} From 340ccd129f6a9d6149f24c1b9a4862d59b49f1e3 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 26 Jul 2023 12:44:49 -0700 Subject: [PATCH 04/33] feat: add types for new capability service definitions mostly just cargo-culting the existing type setups, but basing these types on the designs proposed in https://hackmd.io/@tvpl/H1whErCI3 --- packages/capabilities/src/index.js | 11 ++++ packages/capabilities/src/types.ts | 80 +++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index 7f9119439..a8b66c28f 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -11,6 +11,8 @@ import * as Customer from './customer.js' import * as Console from './console.js' import * as Offer from './offer.js' import * as Aggregate from './aggregate.js' +import * as RateLimit from './rate-limit.js' +import * as Subscription from './subscription.js' export { Access, @@ -26,6 +28,8 @@ export { Utils, Aggregate, Offer, + RateLimit, + Subscription, } /** @type {import('./types.js').AbilitiesArray} */ @@ -52,4 +56,11 @@ export const abilitiesAsStrings = [ Aggregate.offer.can, Aggregate.get.can, Offer.arrange.can, + Customer.get.can, + Consumer.has.can, + Consumer.get.can, + Subscription.get.can, + RateLimit.add.can, + RateLimit.remove.can, + RateLimit.list.can, ] diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 07604eaa0..862729c6f 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -1,6 +1,6 @@ import type { TupleToUnion } from 'type-fest' import * as Ucanto from '@ucanto/interface' -import { InferInvokedCapability, Unit, DID } from '@ucanto/interface' +import { InferInvokedCapability, Unit, DID, DIDKey } from '@ucanto/interface' import { space, info, recover, recoverValidation } from './space.js' import * as provider from './provider.js' import { top } from './top.js' @@ -10,8 +10,15 @@ import { claim, redeem } from './voucher.js' import * as AccessCaps from './access.js' import * as AggregateCaps from './aggregate.js' import * as OfferCaps from './offer.js' +import * as CustomerCaps from './customer.js' +import * as ConsumerCaps from './consumer.js' +import * as SubscriptionCaps from './subscription.js' +import * as RateLimitCaps from './rate-limit.js' export type { Unit } + +export type AccountDID = DID<'mailto'> + /** * failure due to a resource not having enough storage capacity. */ @@ -64,6 +71,68 @@ export interface InvalidProvider extends Ucanto.Failure { name: 'InvalidProvider' } +// Customer +export type CustomerGet = InferInvokedCapability +export interface CustomerGetSuccess { + did: AccountDID +} +export interface CustomerNotFound extends Ucanto.Failure { + name: 'CustomerNotFound' +} +export type CustomerGetFailure = CustomerNotFound + +// Consumer +export type ConsumerHas = InferInvokedCapability +export type ConsumerHasSuccess = boolean +export type ConsumerHasFailure = Ucanto.Failure +export type ConsumerGet = InferInvokedCapability +export interface ConsumerGetSuccess { + did: DIDKey, + allocated: number, + total: number, + subscription: string +} +export interface ConsumerNotFound extends Ucanto.Failure { + name: 'ConsumerNotFound' +} +export type ConsumerGetFailure = ConsumerNotFound + +// Subscription +export type SubscriptionGet = InferInvokedCapability +export interface SubscriptionGetSuccess { + customer: AccountDID + consumer: DIDKey +} +export interface SubscriptionNotFound extends Ucanto.Failure { + name: 'SubscriptionNotFound' +} +export type SubscriptionGetFailure = SubscriptionNotFound + +// Rate Limit +export type RateLimitAdd = InferInvokedCapability +export interface RateLimitAddSuccess { + id: string +} +export type RateLimitAddFailure = Ucanto.Failure + +export type RateLimitRemove = InferInvokedCapability +export type RateLimitRemoveSuccess = {} +export interface RateLimitsNotFound extends Ucanto.Failure { + name: 'RateLimitsNotFound' +} +export type RateLimitRemoveFailure = RateLimitsNotFound + +export type RateLimitList = InferInvokedCapability +export interface RateLimit { + id: string, + limit: number +} +export interface RateLimitListSuccess { + limits: RateLimit[] +} +export type RateLimitListFailure = Ucanto.Failure + + // Space export type Space = InferInvokedCapability export type SpaceInfo = InferInvokedCapability @@ -143,5 +212,12 @@ export type AbilitiesArray = [ AccessSession['can'], AggregateOffer['can'], AggregateGet['can'], - OfferArrange['can'] + OfferArrange['can'], + CustomerGet['can'], + ConsumerHas['can'], + ConsumerGet['can'], + SubscriptionGet['can'], + RateLimitAdd['can'], + RateLimitRemove['can'], + RateLimitList['can'] ] From 2d1133a39ab94b3e2fec12b16353f80a64c4f99b Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 26 Jul 2023 12:47:04 -0700 Subject: [PATCH 05/33] feat: add new capability types to service definition also update existing Consumer.has types to match our normal patterns --- packages/upload-api/src/consumer/has.js | 2 +- packages/upload-api/src/types.ts | 77 +++++++++++++++---------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/packages/upload-api/src/consumer/has.js b/packages/upload-api/src/consumer/has.js index b781c5ce2..2bb2d24c5 100644 --- a/packages/upload-api/src/consumer/has.js +++ b/packages/upload-api/src/consumer/has.js @@ -11,7 +11,7 @@ export const provide = (context) => /** * @param {API.Input} input * @param {API.ConsumerServiceContext} context - * @returns {Promise>} + * @returns {Promise>} */ const has = async ({ capability }, context) => { if (capability.with !== context.signer.did()) { diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 88af2e791..b4f0cc69c 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -30,7 +30,6 @@ export type ValidationEmailSend = { } export type SpaceDID = DIDKey -export type AccountDID = DID<'mailto'> export type ServiceDID = DID<'web'> export type ServiceSigner = Signer export interface SpaceProviderRegistry { @@ -70,6 +69,27 @@ import { AccessConfirm, AccessConfirmSuccess, AccessConfirmFailure, + ConsumerHas, + ConsumerHasSuccess, + ConsumerHasFailure, + ConsumerGet, + ConsumerGetSuccess, + ConsumerGetFailure, + CustomerGet, + CustomerGetSuccess, + CustomerGetFailure, + SubscriptionGet, + SubscriptionGetSuccess, + SubscriptionGetFailure, + RateLimitAdd, + RateLimitAddSuccess, + RateLimitAddFailure, + RateLimitRemove, + RateLimitRemoveSuccess, + RateLimitRemoveFailure, + RateLimitList, + RateLimitListSuccess, + RateLimitListFailure, ProviderAdd, ProviderAddSuccess, ProviderAddFailure, @@ -128,18 +148,20 @@ export interface Service { > } consumer: { - has: ServiceMethod< - InferInvokedCapability, - boolean, - Failure + has: ServiceMethod + get: ServiceMethod } customer: { - get: ServiceMethod< - InferInvokedCapability, - CustomerGetOk, - CustomerGetError - > + get: ServiceMethod + } + subscription: { + get: ServiceMethod + }, + "rate-limits": { + add: ServiceMethod + remove: ServiceMethod + list: ServiceMethod } provider: { add: ServiceMethod @@ -204,14 +226,14 @@ export interface RateLimitsServiceContext { export interface ServiceContext extends AccessServiceContext, - ConsoleServiceContext, - ConsumerServiceContext, - CustomerServiceContext, - ProviderServiceContext, - SpaceServiceContext, - StoreServiceContext, - RateLimitsServiceContext, - UploadServiceContext {} + ConsoleServiceContext, + ConsumerServiceContext, + CustomerServiceContext, + ProviderServiceContext, + SpaceServiceContext, + StoreServiceContext, + RateLimitsServiceContext, + UploadServiceContext {} export interface UcantoServerContext extends ServiceContext { id: Signer @@ -221,8 +243,8 @@ export interface UcantoServerContext extends ServiceContext { export interface UcantoServerTestContext extends UcantoServerContext, - StoreTestContext, - UploadTestContext { + StoreTestContext, + UploadTestContext { connection: ConnectionView mail: DebugEmail service: Signer @@ -264,7 +286,7 @@ export interface CarStoreBucketOptions { } export interface CarStoreBucketService { - use(options?: CarStoreBucketOptions): Promise + use (options?: CarStoreBucketOptions): Promise } export interface DudewhereBucket { @@ -282,7 +304,7 @@ export interface StoreTable { } export interface TestStoreTable { - get( + get ( space: DID, link: UnknownLink ): Promise<(StoreAddInput & StoreListItem) | undefined> @@ -305,16 +327,7 @@ export type SpaceInfoResult = { export interface UnknownProvider extends Failure { name: 'UnknownProvider' } - -export type CustomerGetError = UnknownProvider - -export interface CustomerGetOk { - customer: null | { - did: AccountDID - } -} - -export type CustomerGetResult = Result +export type CustomerGetResult = Result export interface StoreAddInput { space: DID From d2766fa152ddb5de56ead36efda98b3a5d301be3 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 26 Jul 2023 12:53:14 -0700 Subject: [PATCH 06/33] feat: two more typing tweaks --- packages/capabilities/src/types.ts | 7 ++++++- packages/upload-api/src/types.ts | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 862729c6f..94dabc61a 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -27,6 +27,11 @@ export interface InsufficientStorage { message: string } +export interface UnknownProvider { + name: 'UnknownProvider', + did: DID +} + // Access export type Access = InferInvokedCapability export type AccessAuthorize = InferInvokedCapability< @@ -106,7 +111,7 @@ export interface SubscriptionGetSuccess { export interface SubscriptionNotFound extends Ucanto.Failure { name: 'SubscriptionNotFound' } -export type SubscriptionGetFailure = SubscriptionNotFound +export type SubscriptionGetFailure = SubscriptionNotFound | UnknownProvider // Rate Limit export type RateLimitAdd = InferInvokedCapability diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index b4f0cc69c..ba71e693d 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -33,7 +33,7 @@ export type SpaceDID = DIDKey export type ServiceDID = DID<'web'> export type ServiceSigner = Signer export interface SpaceProviderRegistry { - hasStorageProvider(space: SpaceDID): Promise> + hasStorageProvider (space: SpaceDID): Promise> } export interface InsufficientStorage extends Failure { @@ -220,6 +220,11 @@ export interface ProviderServiceContext { rateLimitsStorage: RateLimits } +export interface SubscriptionServiceContext { + signer: EdSigner.Signer + provisionsStorage: Provisions +} + export interface RateLimitsServiceContext { rateLimitsStorage: RateLimits } @@ -232,6 +237,7 @@ export interface ServiceContext ProviderServiceContext, SpaceServiceContext, StoreServiceContext, + SubscriptionServiceContext, RateLimitsServiceContext, UploadServiceContext {} @@ -328,6 +334,7 @@ export interface UnknownProvider extends Failure { name: 'UnknownProvider' } export type CustomerGetResult = Result +export type SubscriptionGetResult = Result export interface StoreAddInput { space: DID From 23793d5da0f0fa74253b3497a096106c631bb241 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 26 Jul 2023 12:58:26 -0700 Subject: [PATCH 07/33] feat: small naming tweak --- packages/capabilities/src/types.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 94dabc61a..d238e87b7 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -128,12 +128,12 @@ export interface RateLimitsNotFound extends Ucanto.Failure { export type RateLimitRemoveFailure = RateLimitsNotFound export type RateLimitList = InferInvokedCapability -export interface RateLimit { +export interface RateLimitSubject { id: string, limit: number } export interface RateLimitListSuccess { - limits: RateLimit[] + limits: RateLimitSubject[] } export type RateLimitListFailure = Ucanto.Failure From 15576fb918e84763cd39256016bfd5608d1beeb1 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 26 Jul 2023 17:23:19 -0700 Subject: [PATCH 08/33] fix: get types checking and a simple test implemented --- packages/capabilities/src/rate-limit.js | 8 ++-- packages/capabilities/src/types.ts | 38 +++++++++------ .../test/capabilities/rate-limit.test.js | 19 ++++---- packages/upload-api/src/consumer.js | 2 + packages/upload-api/src/consumer/get.js | 29 +++++++++++ packages/upload-api/src/lib.js | 4 ++ packages/upload-api/src/rate-limit.js | 13 +++++ packages/upload-api/src/rate-limit/add.js | 4 +- packages/upload-api/src/rate-limit/list.js | 11 +++-- packages/upload-api/src/rate-limit/remove.js | 6 +-- packages/upload-api/src/subscription.js | 9 ++++ packages/upload-api/src/subscription/get.js | 46 ++++++++++++++++++ packages/upload-api/src/types.ts | 40 ++++++++++------ packages/upload-api/src/types/provisions.ts | 48 +++++++++++++++++-- packages/upload-api/src/types/rate-limits.ts | 19 ++++---- packages/upload-api/test/rate-limit/add.js | 38 +++++++++++++++ .../upload-api/test/rate-limit/add.spec.js | 31 ++++++++++++ .../upload-api/test/rate-limits-storage.js | 47 ++++++++++-------- 18 files changed, 328 insertions(+), 84 deletions(-) create mode 100644 packages/upload-api/src/consumer/get.js create mode 100644 packages/upload-api/src/rate-limit.js create mode 100644 packages/upload-api/src/subscription.js create mode 100644 packages/upload-api/src/subscription/get.js create mode 100644 packages/upload-api/test/rate-limit/add.js create mode 100644 packages/upload-api/test/rate-limit/add.spec.js diff --git a/packages/capabilities/src/rate-limit.js b/packages/capabilities/src/rate-limit.js index 5ed553a70..7e36a1c8f 100644 --- a/packages/capabilities/src/rate-limit.js +++ b/packages/capabilities/src/rate-limit.js @@ -22,7 +22,7 @@ export const add = capability({ with: Provider, nb: struct({ subject: Schema.string(), - rate: Schema.number() + rate: Schema.number(), }), derives: (child, parent) => { return ( @@ -41,12 +41,12 @@ export const remove = capability({ can: 'rate-limit/remove', with: Provider, nb: struct({ - id: Schema.string() + ids: Schema.string().array(), }), derives: (child, parent) => { return ( and(equalWith(child, parent)) || - and(equal(child.nb.id, parent.nb.id, 'id')) || + and(equal(child.nb.ids, parent.nb.ids, 'ids')) || ok({}) ) }, @@ -59,7 +59,7 @@ export const list = capability({ can: 'rate-limit/list', with: Provider, nb: struct({ - subject: Schema.string() + subject: Schema.string(), }), derives: (child, parent) => { return ( diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index d238e87b7..dec593850 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -27,8 +27,8 @@ export interface InsufficientStorage { message: string } -export interface UnknownProvider { - name: 'UnknownProvider', +export interface UnknownProvider extends Ucanto.Failure { + name: 'UnknownProvider' did: DID } @@ -79,9 +79,9 @@ export interface InvalidProvider extends Ucanto.Failure { // Customer export type CustomerGet = InferInvokedCapability export interface CustomerGetSuccess { - did: AccountDID + did: AccountDID } -export interface CustomerNotFound extends Ucanto.Failure { +export interface CustomerNotFound extends Ucanto.Failure { name: 'CustomerNotFound' } export type CustomerGetFailure = CustomerNotFound @@ -92,18 +92,20 @@ export type ConsumerHasSuccess = boolean export type ConsumerHasFailure = Ucanto.Failure export type ConsumerGet = InferInvokedCapability export interface ConsumerGetSuccess { - did: DIDKey, - allocated: number, - total: number, + did: DIDKey + allocated: number + total: number subscription: string } export interface ConsumerNotFound extends Ucanto.Failure { name: 'ConsumerNotFound' } -export type ConsumerGetFailure = ConsumerNotFound +export type ConsumerGetFailure = ConsumerNotFound | Ucanto.Failure // Subscription -export type SubscriptionGet = InferInvokedCapability +export type SubscriptionGet = InferInvokedCapability< + typeof SubscriptionCaps.get +> export interface SubscriptionGetSuccess { customer: AccountDID consumer: DIDKey @@ -111,7 +113,10 @@ export interface SubscriptionGetSuccess { export interface SubscriptionNotFound extends Ucanto.Failure { name: 'SubscriptionNotFound' } -export type SubscriptionGetFailure = SubscriptionNotFound | UnknownProvider +export type SubscriptionGetFailure = + | SubscriptionNotFound + | UnknownProvider + | Ucanto.Failure // Rate Limit export type RateLimitAdd = InferInvokedCapability @@ -120,17 +125,20 @@ export interface RateLimitAddSuccess { } export type RateLimitAddFailure = Ucanto.Failure -export type RateLimitRemove = InferInvokedCapability -export type RateLimitRemoveSuccess = {} +export type RateLimitRemove = InferInvokedCapability< + typeof RateLimitCaps.remove +> +export type RateLimitRemoveSuccess = Unit + export interface RateLimitsNotFound extends Ucanto.Failure { name: 'RateLimitsNotFound' } -export type RateLimitRemoveFailure = RateLimitsNotFound +export type RateLimitRemoveFailure = RateLimitsNotFound | Ucanto.Failure export type RateLimitList = InferInvokedCapability export interface RateLimitSubject { - id: string, - limit: number + id: string + rate: number } export interface RateLimitListSuccess { limits: RateLimitSubject[] diff --git a/packages/capabilities/test/capabilities/rate-limit.test.js b/packages/capabilities/test/capabilities/rate-limit.test.js index c9c9aecb3..1e14434e0 100644 --- a/packages/capabilities/test/capabilities/rate-limit.test.js +++ b/packages/capabilities/test/capabilities/rate-limit.test.js @@ -17,7 +17,7 @@ describe('rate-limit/add', function () { with: provider, nb: { subject: space.did(), - rate: 0 + rate: 0, }, // TODO: check in with @gozala about whether passing provider as account makes sense proofs: await createAuthorization({ agent, service, account: provider }), @@ -48,7 +48,7 @@ describe('rate-limit/add', function () { with: provider, nb: { subject: space.did(), - rate: 0 + rate: 0, }, }) @@ -69,7 +69,7 @@ describe('rate-limit/add', function () { with: provider, // @ts-ignore nb: { - rate: 0 + rate: 0, }, }) }, /Error: Invalid 'nb' - Object contains invalid field "resource"/) @@ -83,7 +83,7 @@ describe('rate-limit/add', function () { with: provider, // @ts-ignore nb: { - subject: alice.did() + subject: alice.did(), }, }) }, /Error: Invalid 'nb' - Object contains invalid field "rate"/) @@ -99,7 +99,7 @@ describe('rate-limit/remove', function () { audience: service, with: provider, nb: { - id: '123' + ids: ['123'], }, // TODO: check in with @gozala about whether passing provider as account makes sense proofs: await createAuthorization({ agent, service, account: provider }), @@ -115,7 +115,7 @@ describe('rate-limit/remove', function () { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'rate-limit/remove') assert.deepEqual(result.ok.capability.nb, { - resource: space.did() + resource: space.did(), }) } }) @@ -127,7 +127,7 @@ describe('rate-limit/remove', function () { audience: service, with: provider, nb: { - id: '123' + ids: ['123'], }, }) @@ -147,9 +147,8 @@ describe('rate-limit/remove', function () { audience: service, with: provider, // @ts-ignore - nb: { - }, + nb: {}, }) }, /Error: Invalid 'nb' - Object contains invalid field "resource"/) }) -}) \ No newline at end of file +}) diff --git a/packages/upload-api/src/consumer.js b/packages/upload-api/src/consumer.js index 77165e8fe..f12369725 100644 --- a/packages/upload-api/src/consumer.js +++ b/packages/upload-api/src/consumer.js @@ -1,9 +1,11 @@ import * as Types from './types.js' import * as Has from './consumer/has.js' +import * as Get from './consumer/get.js' /** * @param {Types.ConsumerServiceContext} context */ export const createService = (context) => ({ has: Has.provide(context), + get: Get.provide(context), }) diff --git a/packages/upload-api/src/consumer/get.js b/packages/upload-api/src/consumer/get.js new file mode 100644 index 000000000..ed060366e --- /dev/null +++ b/packages/upload-api/src/consumer/get.js @@ -0,0 +1,29 @@ +import * as API from '../types.js' +import * as Provider from '@ucanto/server' +import { Consumer } from '@web3-storage/capabilities' + +/** + * @param {API.ConsumerServiceContext} context + */ +export const provide = (context) => + Provider.provide(Consumer.get, (input) => get(input, context)) + +/** + * @param {API.Input} input + * @param {API.ConsumerServiceContext} context + * @returns {Promise>} + */ +const get = async ({ capability }, context) => { + if (capability.with !== context.signer.did()) { + return Provider.fail( + `Expected with to be ${context.signer.did()}} instead got ${ + capability.with + }` + ) + } + + return context.provisionsStorage.getConsumer( + capability.with, + capability.nb.consumer + ) +} diff --git a/packages/upload-api/src/lib.js b/packages/upload-api/src/lib.js index 3293f845e..b6b1d3048 100644 --- a/packages/upload-api/src/lib.js +++ b/packages/upload-api/src/lib.js @@ -11,6 +11,8 @@ import { createService as createConsumerService } from './consumer.js' import { createService as createCustomerService } from './customer.js' import { createService as createSpaceService } from './space.js' import { createService as createProviderService } from './provider.js' +import { createService as createSubscriptionService } from './subscription.js' +import { createService as createRateLimitService } from './rate-limit.js' export * from './types.js' @@ -38,6 +40,8 @@ export const createService = (context) => ({ space: createSpaceService(context), store: createStoreService(context), upload: createUploadService(context), + subscription: createSubscriptionService(context), + 'rate-limit': createRateLimitService(context), }) /** diff --git a/packages/upload-api/src/rate-limit.js b/packages/upload-api/src/rate-limit.js new file mode 100644 index 000000000..6e7bca7e7 --- /dev/null +++ b/packages/upload-api/src/rate-limit.js @@ -0,0 +1,13 @@ +import * as Types from './types.js' +import * as Add from './rate-limit/add.js' +import * as Remove from './rate-limit/remove.js' +import * as List from './rate-limit/list.js' + +/** + * @param {Types.RateLimitServiceContext} context + */ +export const createService = (context) => ({ + add: Add.provide(context), + remove: Remove.provide(context), + list: List.provide(context), +}) diff --git a/packages/upload-api/src/rate-limit/add.js b/packages/upload-api/src/rate-limit/add.js index 556d3a3cc..4cf6906bd 100644 --- a/packages/upload-api/src/rate-limit/add.js +++ b/packages/upload-api/src/rate-limit/add.js @@ -3,14 +3,14 @@ import * as API from '../types.js' import * as RateLimit from '@web3-storage/capabilities/rate-limit' /** - * @param {API.RateLimitsServiceContext} ctx + * @param {API.RateLimitServiceContext} ctx */ export const provide = (ctx) => Server.provide(RateLimit.add, (input) => add(input, ctx)) /** * @param {API.Input} input - * @param {API.RateLimitsServiceContext} ctx + * @param {API.RateLimitServiceContext} ctx */ export const add = async ({ capability }, ctx) => { return ctx.rateLimitsStorage.add(capability.nb.subject, capability.nb.rate) diff --git a/packages/upload-api/src/rate-limit/list.js b/packages/upload-api/src/rate-limit/list.js index 7c93757a4..8e770160c 100644 --- a/packages/upload-api/src/rate-limit/list.js +++ b/packages/upload-api/src/rate-limit/list.js @@ -3,15 +3,20 @@ import * as API from '../types.js' import * as RateLimit from '@web3-storage/capabilities/rate-limit' /** - * @param {API.RateLimitsServiceContext} ctx + * @param {API.RateLimitServiceContext} ctx */ export const provide = (ctx) => Server.provide(RateLimit.list, (input) => list(input, ctx)) /** * @param {API.Input} input - * @param {API.RateLimitsServiceContext} ctx + * @param {API.RateLimitServiceContext} ctx */ export const list = async ({ capability }, ctx) => { - return ctx.rateLimitsStorage.list(capability.nb.subject) + const result = await ctx.rateLimitsStorage.list(capability.nb.subject) + if (result.ok) { + return Server.ok({ limits: result.ok }) + } else { + return result + } } diff --git a/packages/upload-api/src/rate-limit/remove.js b/packages/upload-api/src/rate-limit/remove.js index 4dac07e31..fd3fa6a0a 100644 --- a/packages/upload-api/src/rate-limit/remove.js +++ b/packages/upload-api/src/rate-limit/remove.js @@ -3,15 +3,15 @@ import * as API from '../types.js' import * as RateLimit from '@web3-storage/capabilities/rate-limit' /** - * @param {API.RateLimitsServiceContext} ctx + * @param {API.RateLimitServiceContext} ctx */ export const provide = (ctx) => Server.provide(RateLimit.remove, (input) => remove(input, ctx)) /** * @param {API.Input} input - * @param {API.RateLimitsServiceContext} ctx + * @param {API.RateLimitServiceContext} ctx */ export const remove = async ({ capability }, ctx) => { - return ctx.rateLimitsStorage.remove(capability.nb.id) + return ctx.rateLimitsStorage.remove(capability.nb.ids) } diff --git a/packages/upload-api/src/subscription.js b/packages/upload-api/src/subscription.js new file mode 100644 index 000000000..d612055e2 --- /dev/null +++ b/packages/upload-api/src/subscription.js @@ -0,0 +1,9 @@ +import * as Types from './types.js' +import * as Get from './subscription/get.js' + +/** + * @param {Types.SubscriptionServiceContext} context + */ +export const createService = (context) => ({ + get: Get.provide(context), +}) diff --git a/packages/upload-api/src/subscription/get.js b/packages/upload-api/src/subscription/get.js new file mode 100644 index 000000000..817357f2a --- /dev/null +++ b/packages/upload-api/src/subscription/get.js @@ -0,0 +1,46 @@ +import * as API from '../types.js' +import * as Server from '@ucanto/server' +import { Subscription } from '@web3-storage/capabilities' + +/** + * @param {API.SubscriptionServiceContext} context + */ +export const provide = (context) => + Server.provide(Subscription.get, (input) => get(input, context)) + +/** + * @param {API.Input} input + * @param {API.SubscriptionServiceContext} context + * @returns {Promise} + */ +const get = async ({ capability }, context) => { + /** + * Ensure that resource is the service DID, which implies it's either + * invoked by service itself or an authorized delegate (like admin). + * In other words no user will be able to invoke this unless service + * explicitly delegated capability to them to do so. + */ + if (capability.with !== context.signer.did()) { + return { error: new UnknownProvider(capability.with) } + } + + return await context.provisionsStorage.getSubscription( + capability.with, + capability.nb.subscription + ) +} + +class UnknownProvider extends Server.Failure { + /** + * @param {API.DID} did + */ + constructor(did) { + super() + this.did = did + this.name = /** @type {const} */ ('UnknownProvider') + } + + describe() { + return `Provider ${this.did} not found` + } +} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index ba71e693d..cccafff8e 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -23,7 +23,6 @@ import { DelegationsStorage as Delegations } from './types/delegations' import { ProvisionsStorage as Provisions } from './types/provisions' import { RateLimitsStorage as RateLimits } from './types/rate-limits' - export type ValidationEmailSend = { to: string url: string @@ -105,10 +104,7 @@ export type { DelegationsStorage, Query as DelegationsStorageQuery, } from './types/delegations' -export type { - RateLimitsStorage, - RateLimit -} from './types/rate-limits' +export type { RateLimitsStorage, RateLimit } from './types/rate-limits' export interface Service { store: { @@ -149,19 +145,30 @@ export interface Service { } consumer: { has: ServiceMethod - get: ServiceMethod + get: ServiceMethod } customer: { get: ServiceMethod } subscription: { - get: ServiceMethod - }, - "rate-limits": { + get: ServiceMethod< + SubscriptionGet, + SubscriptionGetSuccess, + SubscriptionGetFailure + > + } + 'rate-limit': { add: ServiceMethod - remove: ServiceMethod - list: ServiceMethod + remove: ServiceMethod< + RateLimitRemove, + RateLimitRemoveSuccess, + RateLimitRemoveFailure + > + list: ServiceMethod< + RateLimitList, + RateLimitListSuccess, + RateLimitListFailure + > } provider: { add: ServiceMethod @@ -225,7 +232,7 @@ export interface SubscriptionServiceContext { provisionsStorage: Provisions } -export interface RateLimitsServiceContext { +export interface RateLimitServiceContext { rateLimitsStorage: RateLimits } @@ -238,7 +245,7 @@ export interface ServiceContext SpaceServiceContext, StoreServiceContext, SubscriptionServiceContext, - RateLimitsServiceContext, + RateLimitServiceContext, UploadServiceContext {} export interface UcantoServerContext extends ServiceContext { @@ -334,7 +341,10 @@ export interface UnknownProvider extends Failure { name: 'UnknownProvider' } export type CustomerGetResult = Result -export type SubscriptionGetResult = Result +export type SubscriptionGetResult = Result< + SubscriptionGetSuccess, + SubscriptionGetFailure +> export interface StoreAddInput { space: DID diff --git a/packages/upload-api/src/types/provisions.ts b/packages/upload-api/src/types/provisions.ts index f394ba9a8..a015e31e1 100644 --- a/packages/upload-api/src/types/provisions.ts +++ b/packages/upload-api/src/types/provisions.ts @@ -1,4 +1,7 @@ -import type { ProviderDID } from '@web3-storage/capabilities/src/types' +import type { + AccountDID, + ProviderDID, +} from '@web3-storage/capabilities/src/types' import * as Ucanto from '@ucanto/interface' import { ProviderAdd } from '@web3-storage/capabilities/src/types' @@ -12,10 +15,22 @@ export interface Provision { provider: ProviderDID } +export interface Consumer { + did: Ucanto.DIDKey + allocated: number + total: number + subscription: string +} + export interface Customer { did: Ucanto.DID<'mailto'> } +export interface Subscription { + customer: AccountDID + consumer: Ucanto.DIDKey +} + /** * stores instances of a storage provider being consumed by a consumer */ @@ -34,8 +49,21 @@ export interface ProvisionsStorage> { /** * Returns information about a customer related to the given provider. * - * TODO: this should probably be moved to its own Storage interface, but - * I'd like to tackle that once we've finished consolidating the access and upload services. + * TODO: this should be moved out to a consumers store + * + * @param provider DID of the provider we care about + * @param consumer DID of the consumer + * @returns record for the specified customer, if it is in our system + */ + getConsumer: ( + provider: ProviderDID, + consumer: Ucanto.DIDKey + ) => Promise> + + /** + * Returns information about a customer related to the given provider. + * + * TODO: this should be moved out to a subscriptions store (and maybe eventually a "customers" store) * * @param provider DID of the provider we care about * @param customer DID of the customer @@ -43,9 +71,21 @@ export interface ProvisionsStorage> { */ getCustomer: ( provider: ProviderDID, - customer: Ucanto.DID<'mailto'> + customer: AccountDID ) => Promise> + /** + * Returns information about a subscription to a provider. + * + * TODO: this should be moved out to a subscriptions store + * + * @returns subscription information for a given subscription ID at the given provider + */ + getSubscription: ( + provider: ProviderDID, + subscription: string + ) => Promise> + /** * get number of stored items */ diff --git a/packages/upload-api/src/types/rate-limits.ts b/packages/upload-api/src/types/rate-limits.ts index 600ab2754..099043c25 100644 --- a/packages/upload-api/src/types/rate-limits.ts +++ b/packages/upload-api/src/types/rate-limits.ts @@ -3,14 +3,10 @@ import * as Ucanto from '@ucanto/interface' export type RateLimitID = string export interface RateLimit { - /** - * Identifier of rate limited subject - could be a DID, an email address, a URL or any other string. - */ - subject?: string /** * Identifier of this rate limit - can used to remove a limit. */ - id?: string + id: RateLimitID /** * Rate limit applied to the subject - intentionally unitless, should be interpreted by consumer. */ @@ -27,7 +23,10 @@ export interface RateLimitsStorage { * @param subject identifier for subject - could be a DID, a URI, or anything else * @param rate a limit to be interpreted by the consuming system - intentionally unitless */ - add: (subject: string, rate: number) => Promise> + add: ( + subject: string, + rate: number + ) => Promise> /** * Returns rate limits on subject. @@ -38,12 +37,14 @@ export interface RateLimitsStorage { list: (subject: string) => Promise> /** - * Remove a rate limit with a given ID. + * Remove a rate limit with given IDs. */ - remove: (id: RateLimitID) => Promise> + remove: (id: RateLimitID[]) => Promise> /** * Returns true if the given subject has a limit equal to 0. */ - areAnyBlocked: (subjects: string[]) => Promise> + areAnyBlocked: ( + subjects: string[] + ) => Promise> } diff --git a/packages/upload-api/test/rate-limit/add.js b/packages/upload-api/test/rate-limit/add.js new file mode 100644 index 000000000..90894c817 --- /dev/null +++ b/packages/upload-api/test/rate-limit/add.js @@ -0,0 +1,38 @@ +import * as API from '../types.js' +import { Absentee } from '@ucanto/principal' +import { alice, bob } from '../helpers/utils.js' +import { RateLimit } from '@web3-storage/capabilities' + +/** + * @type {API.Tests} + */ +export const test = { + 'rate-limit/add can be invoked': async (assert, context) => { + const { service, agent, space, connection } = await setup(context) + + const result = await RateLimit.add + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + rate: 0, + }, + }) + .execute(connection) + + assert.ok(result.out.ok) + }, +} + +/** + * @param {API.TestContext} context + */ +const setup = async (context) => { + const space = alice + const account = Absentee.from({ id: 'did:mailto:web.mail:alice' }) + const agent = bob + + return { space, account, agent, ...context } +} diff --git a/packages/upload-api/test/rate-limit/add.spec.js b/packages/upload-api/test/rate-limit/add.spec.js new file mode 100644 index 000000000..a24bcd557 --- /dev/null +++ b/packages/upload-api/test/rate-limit/add.spec.js @@ -0,0 +1,31 @@ +/* eslint-disable no-nested-ternary */ +/* eslint-disable no-only-tests/no-only-tests */ +import * as Suite from './add.js' +import * as assert from 'assert' +import { cleanupContext, createContext } from '../helpers/context.js' + +describe('rate-limit/add', () => { + for (const [name, test] of Object.entries(Suite.test)) { + const define = name.startsWith('only! ') + ? it.only + : name.startsWith('skip! ') + ? it.skip + : it + + define(name, async () => { + const context = await createContext() + try { + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + context + ) + } finally { + cleanupContext(context) + } + }) + } +}) diff --git a/packages/upload-api/test/rate-limits-storage.js b/packages/upload-api/test/rate-limits-storage.js index bba84afe4..36d019c62 100644 --- a/packages/upload-api/test/rate-limits-storage.js +++ b/packages/upload-api/test/rate-limits-storage.js @@ -6,17 +6,17 @@ import * as Types from '../src/types.js' export class RateLimitsStorage { constructor() { /** - * @type {Record}} + * @type {Record}} */ this.rateLimits = {} this.nextID = 0 } /** - * - * @param {string} subject + * + * @param {string} subject * @param {number} rate - * @returns + * @returns */ async add(subject, rate) { const id = this.nextID.toString() @@ -24,36 +24,45 @@ export class RateLimitsStorage { this.rateLimits[id] = { id, subject, - rate + rate, } - return { ok: {} } + return { ok: { id } } } /** - * - * @param {string} subject - * @returns + * + * @param {string} subject + * @returns */ async list(subject) { - return { ok: Object.values(this.rateLimits).filter((rl) => rl.subject === subject) || [] } + return { + ok: + Object.values(this.rateLimits).filter((rl) => rl.subject === subject) || + [], + } } /** - * - * @param {string} id + * + * @param {string[]} ids */ - async remove(id) { - delete this.rateLimits[id] + async remove(ids) { + for (const id of ids) { + delete this.rateLimits[id] + } return { ok: {} } } /** - * - * @param {string[]} subjects + * + * @param {string[]} subjects */ async areAnyBlocked(subjects) { - return { ok: Object.values(this.rateLimits).some(({subject, rate}) => (subject && subjects.includes(subject)) && (rate === 0))} + return { + ok: Object.values(this.rateLimits).some( + ({ subject, rate }) => + subject && subjects.includes(subject) && rate === 0 + ), + } } - - } From 58714b26f5ead9593df7c3b42a6078158d775ad2 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 26 Jul 2023 17:34:58 -0700 Subject: [PATCH 09/33] chore: pnpm format --- packages/aggregate-api/src/aggregate/offer.js | 16 +++++++---- packages/capabilities/src/types.ts | 1 - packages/upload-api/src/access/authorize.js | 10 +++++-- packages/upload-api/src/provider-add.js | 15 +++++----- packages/upload-api/src/space-allocate.js | 4 +-- packages/upload-api/src/types.ts | 28 +++++++++---------- packages/upload-api/src/utils/did-mailto.js | 4 +-- 7 files changed, 44 insertions(+), 34 deletions(-) diff --git a/packages/aggregate-api/src/aggregate/offer.js b/packages/aggregate-api/src/aggregate/offer.js index 340f7750c..7c1d57774 100644 --- a/packages/aggregate-api/src/aggregate/offer.js +++ b/packages/aggregate-api/src/aggregate/offer.js @@ -1,6 +1,10 @@ import * as Server from '@ucanto/server' import { CBOR } from '@ucanto/core' -import { Node, Piece, Aggregate as AggregateBuilder } from '@web3-storage/data-segment' +import { + Node, + Piece, + Aggregate as AggregateBuilder, +} from '@web3-storage/data-segment' import * as Aggregate from '@web3-storage/capabilities/aggregate' import * as Offer from '@web3-storage/capabilities/offer' import * as API from '../types.js' @@ -62,10 +66,12 @@ export const claim = async ( // Validate commP of commPs const aggregateBuild = AggregateBuilder.build({ size: aggregateSize, - pieces: offers.map(offer => Piece.fromJSON({ - height: offer.height, - link: { '/': offer.link.toString() } - })) + pieces: offers.map((offer) => + Piece.fromJSON({ + height: offer.height, + link: { '/': offer.link.toString() }, + }) + ), }) if (!aggregateBuild.link.equals(piece.link)) { return { diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index fbe148c69..002d95ae1 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -149,7 +149,6 @@ export interface RateLimitListSuccess { } export type RateLimitListFailure = Ucanto.Failure - // Space export type Space = InferInvokedCapability export type SpaceInfo = InferInvokedCapability diff --git a/packages/upload-api/src/access/authorize.js b/packages/upload-api/src/access/authorize.js index 2b59d28a7..4b22b763a 100644 --- a/packages/upload-api/src/access/authorize.js +++ b/packages/upload-api/src/access/authorize.js @@ -17,14 +17,18 @@ export const provide = (ctx) => */ export const authorize = async ({ capability }, ctx) => { const isBlocked = await ctx.rateLimitsStorage.areAnyBlocked( - emailAndDomainFromMailtoDid(/** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */(capability.nb.iss)) + emailAndDomainFromMailtoDid( + /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */ ( + capability.nb.iss + ) + ) ) if (isBlocked.error || isBlocked.ok) { return { error: { name: 'AccountBlocked', - message: `Account identified by {capability.nb.iss} is blocked` - } + message: `Account identified by {capability.nb.iss} is blocked`, + }, } } diff --git a/packages/upload-api/src/provider-add.js b/packages/upload-api/src/provider-add.js index 6cb8932ad..5ec2627f2 100644 --- a/packages/upload-api/src/provider-add.js +++ b/packages/upload-api/src/provider-add.js @@ -16,10 +16,7 @@ export const provide = (ctx) => */ export const add = async ( { capability, invocation }, - { - provisionsStorage: provisions, - rateLimitsStorage: rateLimits - } + { provisionsStorage: provisions, rateLimitsStorage: rateLimits } ) => { const { nb: { consumer, provider }, @@ -34,14 +31,18 @@ export const add = async ( } } const isBlocked = await rateLimits.areAnyBlocked( - emailAndDomainFromMailtoDid(/** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */(accountDID)) + emailAndDomainFromMailtoDid( + /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */ ( + accountDID + ) + ) ) if (isBlocked.error || isBlocked.ok) { return { error: { name: 'AccountBlocked', - message: `Account identified by {capability.nb.iss} is blocked` - } + message: `Account identified by {capability.nb.iss} is blocked`, + }, } } if (!provisions.services.includes(provider)) { diff --git a/packages/upload-api/src/space-allocate.js b/packages/upload-api/src/space-allocate.js index 3fc9f3acf..5a76c68f5 100644 --- a/packages/upload-api/src/space-allocate.js +++ b/packages/upload-api/src/space-allocate.js @@ -15,8 +15,8 @@ export const allocate = async ({ capability }, context) => { return { error: { name: 'InsufficientStorage', - message: `${space} is blocked` - } + message: `${space} is blocked`, + }, } } const { size } = nb diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index cccafff8e..46cfb6bab 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -32,7 +32,7 @@ export type SpaceDID = DIDKey export type ServiceDID = DID<'web'> export type ServiceSigner = Signer export interface SpaceProviderRegistry { - hasStorageProvider (space: SpaceDID): Promise> + hasStorageProvider(space: SpaceDID): Promise> } export interface InsufficientStorage extends Failure { @@ -238,15 +238,15 @@ export interface RateLimitServiceContext { export interface ServiceContext extends AccessServiceContext, - ConsoleServiceContext, - ConsumerServiceContext, - CustomerServiceContext, - ProviderServiceContext, - SpaceServiceContext, - StoreServiceContext, - SubscriptionServiceContext, - RateLimitServiceContext, - UploadServiceContext {} + ConsoleServiceContext, + ConsumerServiceContext, + CustomerServiceContext, + ProviderServiceContext, + SpaceServiceContext, + StoreServiceContext, + SubscriptionServiceContext, + RateLimitServiceContext, + UploadServiceContext {} export interface UcantoServerContext extends ServiceContext { id: Signer @@ -256,8 +256,8 @@ export interface UcantoServerContext extends ServiceContext { export interface UcantoServerTestContext extends UcantoServerContext, - StoreTestContext, - UploadTestContext { + StoreTestContext, + UploadTestContext { connection: ConnectionView mail: DebugEmail service: Signer @@ -299,7 +299,7 @@ export interface CarStoreBucketOptions { } export interface CarStoreBucketService { - use (options?: CarStoreBucketOptions): Promise + use(options?: CarStoreBucketOptions): Promise } export interface DudewhereBucket { @@ -317,7 +317,7 @@ export interface StoreTable { } export interface TestStoreTable { - get ( + get( space: DID, link: UnknownLink ): Promise<(StoreAddInput & StoreListItem) | undefined> diff --git a/packages/upload-api/src/utils/did-mailto.js b/packages/upload-api/src/utils/did-mailto.js index cdaa5a918..6a30e6412 100644 --- a/packages/upload-api/src/utils/did-mailto.js +++ b/packages/upload-api/src/utils/did-mailto.js @@ -3,8 +3,8 @@ import * as DidMailto from '@web3-storage/did-mailto' /** * @param {import("@web3-storage/did-mailto/dist/src/types").DidMailto} mailtoDid */ -export function emailAndDomainFromMailtoDid(mailtoDid){ +export function emailAndDomainFromMailtoDid(mailtoDid) { const accountEmail = DidMailto.email(DidMailto.fromString(mailtoDid)) const accountDomain = accountEmail.split('@')[1] return [accountEmail, accountDomain] -} \ No newline at end of file +} From ef9042c5995abd7a0092f988f41747d026f55336 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 26 Jul 2023 17:48:10 -0700 Subject: [PATCH 10/33] fix: more type updates it really wasn't happy with getCustomer potentially returning `null` and I think that should use a `Ucanto.Failure` anyway so I elected to make a backwards-incompatible change here since I think this work will merit a breaking change anyway --- packages/capabilities/src/types.ts | 2 +- packages/upload-api/src/customer/get.js | 4 ++-- packages/upload-api/src/types/provisions.ts | 2 +- packages/upload-api/test/provisions-storage.js | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 002d95ae1..7302e0df7 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -88,7 +88,7 @@ export interface CustomerGetSuccess { export interface CustomerNotFound extends Ucanto.Failure { name: 'CustomerNotFound' } -export type CustomerGetFailure = CustomerNotFound +export type CustomerGetFailure = CustomerNotFound | Ucanto.Failure // Consumer export type ConsumerHas = InferInvokedCapability diff --git a/packages/upload-api/src/customer/get.js b/packages/upload-api/src/customer/get.js index 9c930399b..ef7b421d0 100644 --- a/packages/upload-api/src/customer/get.js +++ b/packages/upload-api/src/customer/get.js @@ -24,11 +24,11 @@ const get = async ({ capability }, context) => { return { error: new UnknownProvider(capability.with) } } - const customer = await context.provisionsStorage.getCustomer( + const result = await context.provisionsStorage.getCustomer( capability.with, capability.nb.customer ) - return { ok: { customer: customer.ok ? { did: customer.ok.did } : null } } + return result.ok ? { ok: { did: result.ok.did } } : result } class UnknownProvider extends Provider.Failure { diff --git a/packages/upload-api/src/types/provisions.ts b/packages/upload-api/src/types/provisions.ts index a015e31e1..b5d1f99a6 100644 --- a/packages/upload-api/src/types/provisions.ts +++ b/packages/upload-api/src/types/provisions.ts @@ -72,7 +72,7 @@ export interface ProvisionsStorage> { getCustomer: ( provider: ProviderDID, customer: AccountDID - ) => Promise> + ) => Promise> /** * Returns information about a subscription to a provider. diff --git a/packages/upload-api/test/provisions-storage.js b/packages/upload-api/test/provisions-storage.js index 05b5daa93..e32e8514f 100644 --- a/packages/upload-api/test/provisions-storage.js +++ b/packages/upload-api/test/provisions-storage.js @@ -71,7 +71,7 @@ export class ProvisionsStorage { const exists = Object.values(this.provisions).find( (p) => p.provider === provider && p.customer === customer ) - return { ok: exists ? { did: customer } : null } + return exists ? { ok: { did: customer } } : { error: { name: 'CustomerNotFound', message: 'customer does not exist' } } } async count() { From 49793300f631fe8dbafae3883e9e2b1aad27ed93 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 27 Jul 2023 12:06:05 -0700 Subject: [PATCH 11/33] feat: implement new ProvisionsStorage methods --- .../test/provisions-storage-tests.js | 3 +- .../upload-api/test/provisions-storage.js | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packages/upload-api/test/provisions-storage-tests.js b/packages/upload-api/test/provisions-storage-tests.js index d826f1519..9a53e2136 100644 --- a/packages/upload-api/test/provisions-storage-tests.js +++ b/packages/upload-api/test/provisions-storage-tests.js @@ -83,7 +83,6 @@ export const test = { provider, 'did:mailto:example.com:travis' ) - assert.ok(!fakeCustomerResult.error, 'error getting fake customer record') - assert.equal(fakeCustomerResult.ok, null) + assert.equal(fakeCustomerResult.error?.name, 'CustomerNotFound') }, } diff --git a/packages/upload-api/test/provisions-storage.js b/packages/upload-api/test/provisions-storage.js index e32e8514f..d1f1363a8 100644 --- a/packages/upload-api/test/provisions-storage.js +++ b/packages/upload-api/test/provisions-storage.js @@ -74,6 +74,45 @@ export class ProvisionsStorage { return exists ? { ok: { did: customer } } : { error: { name: 'CustomerNotFound', message: 'customer does not exist' } } } + /** + * + * @param {Types.ProviderDID} provider + * @param {string} subscription + * @returns + */ + async getSubscription(provider, subscription) { + const provision = Object.values(this.provisions) + .find(p => ((p.customer === subscription) && (p.provider === provider))) + if (provision) { + return { ok: provision } + } else { + return { error: { name: 'SubscriptionNotFound', message: `could not find ${subscription}` } } + } + } + + /** + * + * @param {Types.ProviderDID} provider + * @param {*} consumer + * @returns + */ + async getConsumer(provider, consumer) { + const provision = Object.values(this.provisions) + .find(p => ((p.consumer === consumer) && (p.provider === provider))) + if (provision) { + return { + ok: { + did: provision.consumer, + allocated: 0, + total: 100, + subscription: provision.customer + } + } + } else { + return { error: { name: 'ConsumerNotFound', message: `could not find ${consumer}` } } + } + } + async count() { return BigInt(Object.values(this.provisions).length) } From fcee02406ae9a96736e9ec197e33dd968fb4e085 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 27 Jul 2023 12:14:00 -0700 Subject: [PATCH 12/33] feat: limit `rate-limit/remove` to a single ID this eliminates the specter of partial failure --- packages/capabilities/src/rate-limit.js | 4 ++-- .../capabilities/test/capabilities/rate-limit.test.js | 4 ++-- packages/upload-api/src/rate-limit/remove.js | 2 +- packages/upload-api/src/types/rate-limits.ts | 4 ++-- packages/upload-api/test/rate-limits-storage.js | 8 +++----- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/capabilities/src/rate-limit.js b/packages/capabilities/src/rate-limit.js index 7e36a1c8f..d07cbdecd 100644 --- a/packages/capabilities/src/rate-limit.js +++ b/packages/capabilities/src/rate-limit.js @@ -41,12 +41,12 @@ export const remove = capability({ can: 'rate-limit/remove', with: Provider, nb: struct({ - ids: Schema.string().array(), + id: Schema.string(), }), derives: (child, parent) => { return ( and(equalWith(child, parent)) || - and(equal(child.nb.ids, parent.nb.ids, 'ids')) || + and(equal(child.nb.id, parent.nb.id, 'id')) || ok({}) ) }, diff --git a/packages/capabilities/test/capabilities/rate-limit.test.js b/packages/capabilities/test/capabilities/rate-limit.test.js index 1e14434e0..fd3b04988 100644 --- a/packages/capabilities/test/capabilities/rate-limit.test.js +++ b/packages/capabilities/test/capabilities/rate-limit.test.js @@ -99,7 +99,7 @@ describe('rate-limit/remove', function () { audience: service, with: provider, nb: { - ids: ['123'], + id: '123', }, // TODO: check in with @gozala about whether passing provider as account makes sense proofs: await createAuthorization({ agent, service, account: provider }), @@ -127,7 +127,7 @@ describe('rate-limit/remove', function () { audience: service, with: provider, nb: { - ids: ['123'], + id: '123', }, }) diff --git a/packages/upload-api/src/rate-limit/remove.js b/packages/upload-api/src/rate-limit/remove.js index fd3fa6a0a..3fe4caead 100644 --- a/packages/upload-api/src/rate-limit/remove.js +++ b/packages/upload-api/src/rate-limit/remove.js @@ -13,5 +13,5 @@ export const provide = (ctx) => * @param {API.RateLimitServiceContext} ctx */ export const remove = async ({ capability }, ctx) => { - return ctx.rateLimitsStorage.remove(capability.nb.ids) + return ctx.rateLimitsStorage.remove(capability.nb.id) } diff --git a/packages/upload-api/src/types/rate-limits.ts b/packages/upload-api/src/types/rate-limits.ts index 099043c25..42f4a7b8f 100644 --- a/packages/upload-api/src/types/rate-limits.ts +++ b/packages/upload-api/src/types/rate-limits.ts @@ -37,9 +37,9 @@ export interface RateLimitsStorage { list: (subject: string) => Promise> /** - * Remove a rate limit with given IDs. + * Remove a rate limit with given ID. */ - remove: (id: RateLimitID[]) => Promise> + remove: (id: RateLimitID) => Promise> /** * Returns true if the given subject has a limit equal to 0. diff --git a/packages/upload-api/test/rate-limits-storage.js b/packages/upload-api/test/rate-limits-storage.js index 36d019c62..c5cfefa90 100644 --- a/packages/upload-api/test/rate-limits-storage.js +++ b/packages/upload-api/test/rate-limits-storage.js @@ -44,12 +44,10 @@ export class RateLimitsStorage { /** * - * @param {string[]} ids + * @param {string} id */ - async remove(ids) { - for (const id of ids) { - delete this.rateLimits[id] - } + async remove(id) { + delete this.rateLimits[id] return { ok: {} } } From 05ba4f743aadb23ffc968bf24e516edc73718082 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 27 Jul 2023 13:54:08 -0700 Subject: [PATCH 13/33] feat: move areAnyBlocked to a helper function Remove it from the interface - I thought that IN queries would be supported by Dynamo but they aren't. I was already on the fence about this interface method and this pushed me into the "not worth it" camp. --- packages/upload-api/src/access/authorize.js | 4 ++- packages/upload-api/src/provider-add.js | 4 ++- packages/upload-api/src/space-allocate.js | 3 +- packages/upload-api/src/types/rate-limits.ts | 7 ---- packages/upload-api/src/utils/rate-limits.js | 34 +++++++++++++++++++ .../upload-api/test/rate-limits-storage.js | 13 ------- 6 files changed, 42 insertions(+), 23 deletions(-) create mode 100644 packages/upload-api/src/utils/rate-limits.js diff --git a/packages/upload-api/src/access/authorize.js b/packages/upload-api/src/access/authorize.js index 4b22b763a..da4f52608 100644 --- a/packages/upload-api/src/access/authorize.js +++ b/packages/upload-api/src/access/authorize.js @@ -4,6 +4,7 @@ import * as Access from '@web3-storage/capabilities/access' import * as DidMailto from '@web3-storage/did-mailto' import { delegationToString } from '@web3-storage/access/encoding' import { emailAndDomainFromMailtoDid } from '../utils/did-mailto.js' +import { areAnyBlocked } from '../utils/rate-limits.js' /** * @param {API.AccessServiceContext} ctx @@ -16,7 +17,8 @@ export const provide = (ctx) => * @param {API.AccessServiceContext} ctx */ export const authorize = async ({ capability }, ctx) => { - const isBlocked = await ctx.rateLimitsStorage.areAnyBlocked( + const isBlocked = await areAnyBlocked( + ctx.rateLimitsStorage, emailAndDomainFromMailtoDid( /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */ ( capability.nb.iss diff --git a/packages/upload-api/src/provider-add.js b/packages/upload-api/src/provider-add.js index 5ec2627f2..cfad145f5 100644 --- a/packages/upload-api/src/provider-add.js +++ b/packages/upload-api/src/provider-add.js @@ -3,6 +3,7 @@ import * as Server from '@ucanto/server' import { Provider } from '@web3-storage/capabilities' import * as validator from '@ucanto/validator' import { emailAndDomainFromMailtoDid } from './utils/did-mailto.js' +import { areAnyBlocked } from './utils/rate-limits.js' /** * @param {API.ProviderServiceContext} ctx @@ -30,7 +31,8 @@ export const add = async ( }, } } - const isBlocked = await rateLimits.areAnyBlocked( + const isBlocked = await areAnyBlocked( + rateLimits, emailAndDomainFromMailtoDid( /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */ ( accountDID diff --git a/packages/upload-api/src/space-allocate.js b/packages/upload-api/src/space-allocate.js index 5a76c68f5..434b6ef4f 100644 --- a/packages/upload-api/src/space-allocate.js +++ b/packages/upload-api/src/space-allocate.js @@ -1,6 +1,7 @@ import * as API from './types.js' import * as Server from '@ucanto/server' import * as Space from '@web3-storage/capabilities/space' +import { areAnyBlocked } from './utils/rate-limits.js' /** * @@ -10,7 +11,7 @@ import * as Space from '@web3-storage/capabilities/space' */ export const allocate = async ({ capability }, context) => { const { with: space, nb } = capability - const isBlocked = await context.rateLimitsStorage.areAnyBlocked([space]) + const isBlocked = await areAnyBlocked(context.rateLimitsStorage, [space]) if (isBlocked.error || isBlocked.ok) { return { error: { diff --git a/packages/upload-api/src/types/rate-limits.ts b/packages/upload-api/src/types/rate-limits.ts index 42f4a7b8f..b1e8e94f7 100644 --- a/packages/upload-api/src/types/rate-limits.ts +++ b/packages/upload-api/src/types/rate-limits.ts @@ -40,11 +40,4 @@ export interface RateLimitsStorage { * Remove a rate limit with given ID. */ remove: (id: RateLimitID) => Promise> - - /** - * Returns true if the given subject has a limit equal to 0. - */ - areAnyBlocked: ( - subjects: string[] - ) => Promise> } diff --git a/packages/upload-api/src/utils/rate-limits.js b/packages/upload-api/src/utils/rate-limits.js new file mode 100644 index 000000000..05ef7e98f --- /dev/null +++ b/packages/upload-api/src/utils/rate-limits.js @@ -0,0 +1,34 @@ + + +/** + * + * @param {import("../types").RateLimit} rateLimit + */ +const isBlock = (rateLimit) => rateLimit.rate === 0 + +/** + * + * @param {import("../types").RateLimit[]} rateLimits + */ +const areAnyBlocks = (rateLimits) => rateLimits.some(isBlock) + +/** + * Query rate limits storage and find out if any of the given subjects + * have rate set to 0 + * + * @param {import("../types").RateLimitsStorage} storage + * @param {string[]} subjects + * @return {Promise>} + */ +export async function areAnyBlocked(storage, subjects) { + const results = await Promise.all(subjects.map(subject => storage.list(subject))) + let anyBlocks = false + for (const result of results) { + if (result.error) { + return result + } else { + anyBlocks = anyBlocks && areAnyBlocks(result.ok) + } + } + return { ok: anyBlocks } +} \ No newline at end of file diff --git a/packages/upload-api/test/rate-limits-storage.js b/packages/upload-api/test/rate-limits-storage.js index c5cfefa90..a7733adb3 100644 --- a/packages/upload-api/test/rate-limits-storage.js +++ b/packages/upload-api/test/rate-limits-storage.js @@ -50,17 +50,4 @@ export class RateLimitsStorage { delete this.rateLimits[id] return { ok: {} } } - - /** - * - * @param {string[]} subjects - */ - async areAnyBlocked(subjects) { - return { - ok: Object.values(this.rateLimits).some( - ({ subject, rate }) => - subject && subjects.includes(subject) && rate === 0 - ), - } - } } From a83335b1d934bafa63c406462a0c2ba6c4f80d79 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 27 Jul 2023 14:28:53 -0700 Subject: [PATCH 14/33] fix: fix bug caught in manual testing --- packages/upload-api/src/access/authorize.js | 2 +- packages/upload-api/src/utils/did-mailto.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/upload-api/src/access/authorize.js b/packages/upload-api/src/access/authorize.js index da4f52608..4e0d7082d 100644 --- a/packages/upload-api/src/access/authorize.js +++ b/packages/upload-api/src/access/authorize.js @@ -29,7 +29,7 @@ export const authorize = async ({ capability }, ctx) => { return { error: { name: 'AccountBlocked', - message: `Account identified by {capability.nb.iss} is blocked`, + message: `Account identified by ${capability.nb.iss} is blocked`, }, } } diff --git a/packages/upload-api/src/utils/did-mailto.js b/packages/upload-api/src/utils/did-mailto.js index 6a30e6412..c6a89af1b 100644 --- a/packages/upload-api/src/utils/did-mailto.js +++ b/packages/upload-api/src/utils/did-mailto.js @@ -4,7 +4,7 @@ import * as DidMailto from '@web3-storage/did-mailto' * @param {import("@web3-storage/did-mailto/dist/src/types").DidMailto} mailtoDid */ export function emailAndDomainFromMailtoDid(mailtoDid) { - const accountEmail = DidMailto.email(DidMailto.fromString(mailtoDid)) + const accountEmail = DidMailto.toEmail(DidMailto.fromString(mailtoDid)) const accountDomain = accountEmail.split('@')[1] return [accountEmail, accountDomain] } From 333ae9f7b98fef0105660357c6b3a6299975e5eb Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 27 Jul 2023 16:52:31 -0700 Subject: [PATCH 15/33] feat: invoke Space.allocate in Store.add I thought we were doing this, but the implementation of `allocateSpace` in the `AccessVerifier` never actually called into `Space.allocate`, it just used `Space.info` to make sure the space existed, which, until now, fair! I'm not sure this is the right way to call the `Space.allocate` provider - maybe it would be better to invoke and execute the actual capability rather than just calling the provider function? --- packages/upload-api/src/store/add.js | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/upload-api/src/store/add.js b/packages/upload-api/src/store/add.js index f53bda6f7..a6a6557da 100644 --- a/packages/upload-api/src/store/add.js +++ b/packages/upload-api/src/store/add.js @@ -1,23 +1,33 @@ import * as Server from '@ucanto/server' import * as Store from '@web3-storage/capabilities/store' import * as API from '../types.js' +import { allocate } from '../space-allocate.js' /** * @param {API.StoreServiceContext} context * @returns {API.ServiceMethod} */ -export function storeAddProvider({ - access, - storeTable, - carStoreBucket, - maxUploadSize, -}) { +export function storeAddProvider(context) { + const { + storeTable, + carStoreBucket, + maxUploadSize, + } = context return Server.provide(Store.add, async ({ capability, invocation }) => { const { link, origin, size } = capability.nb const space = Server.DID.parse(capability.with).did() const issuer = invocation.issuer.did() const [allocated, carIsLinkedToAccount, carExists] = await Promise.all([ - access.allocateSpace(invocation), + // TODO: ask @gozala if this is the right way to call this - maybe it should be an actual UCAN execution? + allocate({ + capability: { + // @ts-expect-error + with: space, + nb: { + size + } + } + }, context), storeTable.exists(space, link), carStoreBucket.has(link), ]) From 0cd93972adafb36ce643aeed5f36f8d7105e0149 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 27 Jul 2023 16:52:55 -0700 Subject: [PATCH 16/33] fix: fix logic bug in the new `areAnyBlocked` utility function --- packages/upload-api/src/utils/rate-limits.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-api/src/utils/rate-limits.js b/packages/upload-api/src/utils/rate-limits.js index 05ef7e98f..37309b829 100644 --- a/packages/upload-api/src/utils/rate-limits.js +++ b/packages/upload-api/src/utils/rate-limits.js @@ -27,7 +27,7 @@ export async function areAnyBlocked(storage, subjects) { if (result.error) { return result } else { - anyBlocks = anyBlocks && areAnyBlocks(result.ok) + anyBlocks = anyBlocks || areAnyBlocks(result.ok) } } return { ok: anyBlocks } From 44a28df25e833f44d6c2ab521ab6a0bd7d1101a4 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Tue, 1 Aug 2023 12:20:41 -0700 Subject: [PATCH 17/33] chore: add tests for new capabilities --- .../test/capabilities/consumer.test.js | 189 ++++++++++++++++++ .../test/capabilities/customer.test.js | 96 +++++++++ .../test/capabilities/rate-limit.test.js | 151 +++++++++++--- .../test/capabilities/subscription.test.js | 96 +++++++++ .../capabilities/test/helpers/fixtures.js | 12 +- 5 files changed, 515 insertions(+), 29 deletions(-) create mode 100644 packages/capabilities/test/capabilities/consumer.test.js create mode 100644 packages/capabilities/test/capabilities/customer.test.js create mode 100644 packages/capabilities/test/capabilities/subscription.test.js diff --git a/packages/capabilities/test/capabilities/consumer.test.js b/packages/capabilities/test/capabilities/consumer.test.js new file mode 100644 index 000000000..2b2b929d0 --- /dev/null +++ b/packages/capabilities/test/capabilities/consumer.test.js @@ -0,0 +1,189 @@ +import assert from 'assert' +import { access } from '@ucanto/validator' +import { delegate } from '@ucanto/core' +import { Verifier } from '@ucanto/principal/ed25519' +import * as Consumer from '../../src/consumer.js' +import { bob, bobAccount, service, alice } from '../helpers/fixtures.js' + +describe('consumer/get', function () { + const agent = alice + const space = bob + it('can be invoked by the service on the service', async function () { + const invocation = Consumer.get.invoke({ + issuer: service, + audience: service, + with: service.did(), + nb: { + consumer: space.did(), + } + }) + const result = await access(await invocation.delegate(), { + capability: Consumer.get, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail('error in self issue') + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'consumer/get') + assert.deepEqual(result.ok.capability.nb, { + consumer: space.did(), + }) + } + }) + + it('can be invoked by an agent delegated permissions by the service', async function () { + const auth = Consumer.get.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + consumer: space.did(), + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'consumer/get' }] + })], + }) + const result = await access(await auth.delegate(), { + capability: Consumer.get, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'consumer/get') + assert.deepEqual(result.ok.capability.nb, { + consumer: space.did(), + }) + } + }) + + it('fails without a delegation from the service delegation', async function () { + const agent = alice + const auth = Consumer.get.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + consumer: space.did(), + }, + }) + + const result = await access(await auth.delegate(), { + capability: Consumer.get, + principal: Verifier, + authority: service, + }) + + assert.equal(result.error?.message.includes('not authorized'), true) + }) + + it('requires nb.consumer', async function () { + assert.throws(() => { + Consumer.get.invoke({ + issuer: alice, + audience: service, + with: service.did(), + // @ts-ignore + nb: { + }, + }) + }, /Error: Invalid 'nb' - Object contains invalid field "consumer"/) + }) +}) + + +describe('consumer/has', function () { + const agent = alice + const space = bob + it('can be invoked by the service on the service', async function () { + const invocation = Consumer.has.invoke({ + issuer: service, + audience: service, + with: service.did(), + nb: { + consumer: space.did(), + } + }) + const result = await access(await invocation.delegate(), { + capability: Consumer.has, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail('error in self issue') + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'consumer/has') + assert.deepEqual(result.ok.capability.nb, { + consumer: space.did(), + }) + } + }) + + it('can be invoked by an agent delegated permissions by the service', async function () { + const auth = Consumer.has.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + consumer: space.did(), + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'consumer/has' }] + })], + }) + const result = await access(await auth.delegate(), { + capability: Consumer.has, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'consumer/has') + assert.deepEqual(result.ok.capability.nb, { + consumer: space.did(), + }) + } + }) + + it('fails without a delegation from the service delegation', async function () { + const agent = alice + const auth = Consumer.has.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + consumer: space.did(), + }, + }) + + const result = await access(await auth.delegate(), { + capability: Consumer.has, + principal: Verifier, + authority: service, + }) + + assert.equal(result.error?.message.includes('not authorized'), true) + }) + + it('requires nb.consumer', async function () { + assert.throws(() => { + Consumer.has.invoke({ + issuer: alice, + audience: service, + with: service.did(), + // @ts-ignore + nb: { + }, + }) + }, /Error: Invalid 'nb' - Object contains invalid field "consumer"/) + }) +}) \ No newline at end of file diff --git a/packages/capabilities/test/capabilities/customer.test.js b/packages/capabilities/test/capabilities/customer.test.js new file mode 100644 index 000000000..204148d6f --- /dev/null +++ b/packages/capabilities/test/capabilities/customer.test.js @@ -0,0 +1,96 @@ +import assert from 'assert' +import { access } from '@ucanto/validator' +import { delegate } from '@ucanto/core' +import { Verifier } from '@ucanto/principal/ed25519' +import * as Customer from '../../src/customer.js' +import { bobAccount, service, alice } from '../helpers/fixtures.js' + +describe('customer/get', function () { + const agent = alice + it('can be invoked by the service on the service', async function () { + const invocation = Customer.get.invoke({ + issuer: service, + audience: service, + with: service.did(), + nb: { + customer: bobAccount.did(), + } + }) + const result = await access(await invocation.delegate(), { + capability: Customer.get, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail('error in self issue') + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'customer/get') + assert.deepEqual(result.ok.capability.nb, { + customer: bobAccount.did(), + }) + } + }) + + it('can be invoked by an agent delegated permissions by the service', async function () { + const auth = Customer.get.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + customer: bobAccount.did(), + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'customer/get' }] + })], + }) + const result = await access(await auth.delegate(), { + capability: Customer.get, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'customer/get') + assert.deepEqual(result.ok.capability.nb, { + customer: bobAccount.did(), + }) + } + }) + + it('fails without a delegation from the service delegation', async function () { + const agent = alice + const auth = Customer.get.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + customer: bobAccount.did(), + }, + }) + + const result = await access(await auth.delegate(), { + capability: Customer.get, + principal: Verifier, + authority: service, + }) + + assert.equal(result.error?.message.includes('not authorized'), true) + }) + + it('requires nb.customer', async function () { + assert.throws(() => { + Customer.get.invoke({ + issuer: alice, + audience: service, + with: service.did(), + // @ts-ignore + nb: { + }, + }) + }, /Error: Invalid 'nb' - Object contains invalid field "customer"/) + }) +}) diff --git a/packages/capabilities/test/capabilities/rate-limit.test.js b/packages/capabilities/test/capabilities/rate-limit.test.js index fd3b04988..aedf8059f 100644 --- a/packages/capabilities/test/capabilities/rate-limit.test.js +++ b/packages/capabilities/test/capabilities/rate-limit.test.js @@ -1,26 +1,54 @@ import assert from 'assert' import { access } from '@ucanto/validator' +import { delegate } from '@ucanto/core' import { Verifier } from '@ucanto/principal/ed25519' import * as RateLimit from '../../src/rate-limit.js' import { bob, service, alice } from '../helpers/fixtures.js' -import { createAuthorization } from '../helpers/utils.js' - -const provider = 'did:web:test.web3.storage' describe('rate-limit/add', function () { - it('can by invoked as account', async function () { + const space = bob + it('can be invoked by the service on the service', async function () { + const invocation = RateLimit.add.invoke({ + issuer: service, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + rate: 0, + } + }) + const result = await access(await invocation.delegate(), { + capability: RateLimit.add, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail('error in self issue') + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'rate-limit/add') + assert.deepEqual(result.ok.capability.nb, { + subject: space.did(), + rate: 0, + }) + } + }) + + it('can be invoked by an agent delegated permissions by the service', async function () { const agent = alice - const space = bob + const auth = RateLimit.add.invoke({ issuer: agent, audience: service, - with: provider, + with: service.did(), nb: { subject: space.did(), rate: 0, }, - // TODO: check in with @gozala about whether passing provider as account makes sense - proofs: await createAuthorization({ agent, service, account: provider }), + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }] + })], }) const result = await access(await auth.delegate(), { capability: RateLimit.add, @@ -28,24 +56,24 @@ describe('rate-limit/add', function () { authority: service, }) if (result.error) { - assert.fail('error in self issue') + assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) } else { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'rate-limit/add') assert.deepEqual(result.ok.capability.nb, { - resource: space.did(), + subject: space.did(), rate: 0, }) } }) - it('fails without account delegation', async function () { + it('fails without a delegation from the service delegation', async function () { const agent = alice const space = bob const auth = RateLimit.add.invoke({ issuer: agent, audience: service, - with: provider, + with: service.did(), nb: { subject: space.did(), rate: 0, @@ -61,18 +89,18 @@ describe('rate-limit/add', function () { assert.equal(result.error?.message.includes('not authorized'), true) }) - it('requires nb.resource', async function () { + it('requires nb.subject', async function () { assert.throws(() => { RateLimit.add.invoke({ issuer: alice, audience: service, - with: provider, + with: service.did(), // @ts-ignore nb: { rate: 0, }, }) - }, /Error: Invalid 'nb' - Object contains invalid field "resource"/) + }, /Error: Invalid 'nb' - Object contains invalid field "subject"/) }) it('requires nb.rate', async function () { @@ -80,7 +108,7 @@ describe('rate-limit/add', function () { RateLimit.add.invoke({ issuer: alice, audience: service, - with: provider, + with: service.did(), // @ts-ignore nb: { subject: alice.did(), @@ -91,18 +119,20 @@ describe('rate-limit/add', function () { }) describe('rate-limit/remove', function () { + const rateLimitId = '123' it('can by invoked as account', async function () { const agent = alice - const space = bob const auth = RateLimit.remove.invoke({ issuer: agent, audience: service, - with: provider, + with: service.did(), nb: { - id: '123', + id: rateLimitId, }, - // TODO: check in with @gozala about whether passing provider as account makes sense - proofs: await createAuthorization({ agent, service, account: provider }), + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/remove' }] + })], }) const result = await access(await auth.delegate(), { capability: RateLimit.remove, @@ -115,19 +145,19 @@ describe('rate-limit/remove', function () { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'rate-limit/remove') assert.deepEqual(result.ok.capability.nb, { - resource: space.did(), + id: rateLimitId, }) } }) - it('fails without account delegation', async function () { + it('fails without a delegation from the service delegation', async function () { const agent = alice const auth = RateLimit.remove.invoke({ issuer: agent, audience: service, - with: provider, + with: service.did(), nb: { - id: '123', + id: rateLimitId, }, }) @@ -140,15 +170,80 @@ describe('rate-limit/remove', function () { assert.equal(result.error?.message.includes('not authorized'), true) }) - it('requires nb.resource', async function () { + it('requires nb.id', async function () { assert.throws(() => { RateLimit.remove.invoke({ issuer: alice, audience: service, - with: provider, + with: service.did(), // @ts-ignore nb: {}, }) - }, /Error: Invalid 'nb' - Object contains invalid field "resource"/) + }, /Error: Invalid 'nb' - Object contains invalid field "id"/) }) }) + +describe('rate-limit/list', function () { + const space = bob + it('can by invoked as account', async function () { + const agent = alice + const auth = RateLimit.list.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }] + })], + }) + const result = await access(await auth.delegate(), { + capability: RateLimit.list, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail('error in self issue') + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'rate-limit/list') + assert.deepEqual(result.ok.capability.nb, { + subject: space.did(), + }) + } + }) + + it('fails without a delegation from the service delegation', async function () { + const agent = alice + const auth = RateLimit.list.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + }, + }) + + const result = await access(await auth.delegate(), { + capability: RateLimit.list, + principal: Verifier, + authority: service, + }) + + assert.equal(result.error?.message.includes('not authorized'), true) + }) + + it('requires nb.subject', async function () { + assert.throws(() => { + RateLimit.list.invoke({ + issuer: alice, + audience: service, + with: service.did(), + // @ts-ignore + nb: {}, + }) + }, /Error: Invalid 'nb' - Object contains invalid field "subject"/) + }) +}) \ No newline at end of file diff --git a/packages/capabilities/test/capabilities/subscription.test.js b/packages/capabilities/test/capabilities/subscription.test.js new file mode 100644 index 000000000..9f4a47e2b --- /dev/null +++ b/packages/capabilities/test/capabilities/subscription.test.js @@ -0,0 +1,96 @@ +import assert from 'assert' +import { access } from '@ucanto/validator' +import { delegate } from '@ucanto/core' +import { Verifier } from '@ucanto/principal/ed25519' +import * as Subscription from '../../src/subscription.js' +import { bob, service, alice } from '../helpers/fixtures.js' + +describe('subscription/get', function () { + const agent = alice + it('can be invoked by the service on the service', async function () { + const invocation = Subscription.get.invoke({ + issuer: service, + audience: service, + with: service.did(), + nb: { + subscription: bob.did(), + } + }) + const result = await access(await invocation.delegate(), { + capability: Subscription.get, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail('error in self issue') + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'subscription/get') + assert.deepEqual(result.ok.capability.nb, { + subscription: bob.did(), + }) + } + }) + + it('can be invoked by an agent delegated permissions by the service', async function () { + const auth = Subscription.get.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subscription: bob.did(), + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'subscription/get' }] + })], + }) + const result = await access(await auth.delegate(), { + capability: Subscription.get, + principal: Verifier, + authority: service, + }) + if (result.error) { + assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) + } else { + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'subscription/get') + assert.deepEqual(result.ok.capability.nb, { + subscription: bob.did(), + }) + } + }) + + it('fails without a delegation from the service delegation', async function () { + const agent = alice + const auth = Subscription.get.invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subscription: bob.did(), + }, + }) + + const result = await access(await auth.delegate(), { + capability: Subscription.get, + principal: Verifier, + authority: service, + }) + + assert.equal(result.error?.message.includes('not authorized'), true) + }) + + it('requires nb.subscription', async function () { + assert.throws(() => { + Subscription.get.invoke({ + issuer: alice, + audience: service, + with: service.did(), + // @ts-ignore + nb: { + }, + }) + }, /Error: Invalid 'nb' - Object contains invalid field "subscription"/) + }) +}) diff --git a/packages/capabilities/test/helpers/fixtures.js b/packages/capabilities/test/helpers/fixtures.js index 5f03c4314..10784791b 100644 --- a/packages/capabilities/test/helpers/fixtures.js +++ b/packages/capabilities/test/helpers/fixtures.js @@ -1,18 +1,28 @@ +import { Absentee } from '@ucanto/principal' import { Signer } from '@ucanto/principal/ed25519' /** did:key:z6Mkqa4oY9Z5Pf5tUcjLHLUsDjKwMC95HGXdE1j22jkbhz6r */ export const alice = Signer.parse( 'MgCZT5vOnYZoVAeyjnzuJIVY9J4LNtJ+f8Js0cTPuKUpFne0BVEDJjEu6quFIU8yp91/TY/+MYK8GvlKoTDnqOCovCVM=' ) + +export const aliceAccount = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) + /** did:key:z6MkffDZCkCTWreg8868fG1FGFogcJj5X6PY93pPcWDn9bob */ export const bob = Signer.parse( 'MgCYbj5AJfVvdrjkjNCxB3iAUwx7RQHVQ7H1sKyHy46Iose0BEevXgL1V73PD9snOCIoONgb+yQ9sycYchQC8kygR4qY=' ) + +export const bobAccount = Absentee.from({ id: 'did:mailto:test.web3.storage:bob' }) + /** did:key:z6MktafZTREjJkvV5mfJxcLpNBoVPwDLhTuMg9ng7dY4zMAL */ export const mallory = Signer.parse( 'MgCYtH0AvYxiQwBG6+ZXcwlXywq9tI50G2mCAUJbwrrahkO0B0elFYkl3Ulf3Q3A/EvcVY0utb4etiSE8e6pi4H0FEmU=' ) +export const malloryAccount = Absentee.from({ id: 'did:mailto:test.web3.storage:mallory' }) + + export const service = Signer.parse( 'MgCYKXoHVy7Vk4/QjcEGi+MCqjntUiasxXJ8uJKY0qh11e+0Bs8WsdqGK7xothgrDzzWD0ME7ynPjz2okXDh8537lId8=' -) +).withDID('did:web:test.web3.storage') From fe1d5fc9d6045af14559d00407997316acbbbfc6 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Tue, 1 Aug 2023 14:02:07 -0700 Subject: [PATCH 18/33] chore: add tests for rate limit invocation handlers this gets basic testing in place for the new rate limit capability handlers the test implementations are a bit wordier and boilerplate-inclined than I'm strictly happy with, but I don't have any easy plans to clean them up - need to think and prototype a little --- packages/upload-api/test/rate-limit/add.js | 205 +++++++++++++++++- packages/upload-api/test/rate-limit/list.js | 63 ++++++ .../upload-api/test/rate-limit/list.spec.js | 31 +++ packages/upload-api/test/rate-limit/remove.js | 101 +++++++++ .../upload-api/test/rate-limit/remove.spec.js | 31 +++ 5 files changed, 429 insertions(+), 2 deletions(-) create mode 100644 packages/upload-api/test/rate-limit/list.js create mode 100644 packages/upload-api/test/rate-limit/list.spec.js create mode 100644 packages/upload-api/test/rate-limit/remove.js create mode 100644 packages/upload-api/test/rate-limit/remove.spec.js diff --git a/packages/upload-api/test/rate-limit/add.js b/packages/upload-api/test/rate-limit/add.js index 90894c817..38f3a5ebe 100644 --- a/packages/upload-api/test/rate-limit/add.js +++ b/packages/upload-api/test/rate-limit/add.js @@ -1,7 +1,12 @@ import * as API from '../types.js' -import { Absentee } from '@ucanto/principal' import { alice, bob } from '../helpers/utils.js' -import { RateLimit } from '@web3-storage/capabilities' +import { Absentee } from '@ucanto/principal' +import { delegate } from '@ucanto/core' +import { Access, RateLimit, Store } from '@web3-storage/capabilities' +import * as CAR from '@ucanto/transport/car' +import * as DidMailto from '@web3-storage/did-mailto' + + /** * @type {API.Tests} @@ -19,10 +24,206 @@ export const test = { subject: space.did(), rate: 0, }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }] + })] + }) + .execute(connection) + + console.log(result.out) + assert.ok(result.out.ok) + }, + + 'rate-limit/add creates a listable rate limit': async (assert, context) => { + const { service, agent, space, connection } = await setup(context) + + const result = await RateLimit.add + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + rate: 0, + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }] + })] + }) + .execute(connection) + + assert.ok(result.out.ok) + + const listResult = await RateLimit.list + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }] + })] + }) + .execute(connection) + + assert.ok(result.out.ok) + assert.equal(listResult.out.ok?.limits.length, 1) + assert.equal(listResult.out.ok?.limits[0].id, result.out.ok?.id) + }, + + 'rate-limit/add can be used to block space allocation': async (assert, context) => { + const { service, agent, space, connection } = await setup(context) + + const result = await RateLimit.add + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + rate: 0, + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }] + })] }) .execute(connection) assert.ok(result.out.ok) + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const link = await CAR.codec.link(data) + const size = data.byteLength + const storeResult = await Store.add + .invoke({ + issuer: agent, + audience: service, + with: space.did(), + nb: { link, size }, + proofs: [await delegate({ + issuer: space, audience: agent, + capabilities: [{ with: space.did(), can: 'store/add' }] + })] + }) + .execute(connection) + + assert.ok(storeResult.out.error) + assert.equal(storeResult.out.error?.name, 'InsufficientStorage') + assert.equal(storeResult.out.error?.message, `${space.did()} is blocked`) + }, + + 'rate-limit/add can be used to block authorization by email address': async (assert, context) => { + const { service, agent, account, connection } = await setup(context) + + // ensure the account can normally be authorized + const okAccessResult = await Access.authorize + .invoke({ + issuer: agent, + audience: service, + with: agent.did(), + nb: { + iss: account.did(), + att: [{ can: '*' }], + }, + }) + .execute(connection) + + assert.ok(okAccessResult.out.ok) + + // block the account's email address + const email = DidMailto.toEmail(DidMailto.fromString(account.did())) + const blockResult = await RateLimit.add + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: email, + rate: 0, + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }] + })] + }) + .execute(connection) + + assert.ok(blockResult.out.ok) + + // verify the account is blocked + const errorAccessResult = await Access.authorize + .invoke({ + issuer: agent, + audience: service, + with: agent.did(), + nb: { + iss: account.did(), + att: [{ can: '*' }], + }, + }) + .execute(connection) + + assert.equal(errorAccessResult.out.error?.name, 'AccountBlocked') + }, + + 'rate-limit/add can be used to block authorization by domain': async (assert, context) => { + const { service, agent, account, connection } = await setup(context) + + // ensure the account can normally be authorized + const okAccessResult = await Access.authorize + .invoke({ + issuer: agent, + audience: service, + with: agent.did(), + nb: { + iss: account.did(), + att: [{ can: '*' }], + }, + }) + .execute(connection) + + assert.ok(okAccessResult.out.ok) + + // block the account's domain + const domain = DidMailto.toEmail(DidMailto.fromString(account.did())).split('@')[1] + const blockResult = await RateLimit.add + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: domain, + rate: 0, + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }] + })] + }) + .execute(connection) + + assert.ok(blockResult.out.ok) + + // verify the account is blocked + const errorAccessResult = await Access.authorize + .invoke({ + issuer: agent, + audience: service, + with: agent.did(), + nb: { + iss: account.did(), + att: [{ can: '*' }], + }, + }) + .execute(connection) + + assert.equal(errorAccessResult.out.error?.name, 'AccountBlocked') }, } diff --git a/packages/upload-api/test/rate-limit/list.js b/packages/upload-api/test/rate-limit/list.js new file mode 100644 index 000000000..cdec138b7 --- /dev/null +++ b/packages/upload-api/test/rate-limit/list.js @@ -0,0 +1,63 @@ +import * as API from '../types.js' +import { alice, bob } from '../helpers/utils.js' +import { Absentee } from '@ucanto/principal' +import { delegate } from '@ucanto/core' +import { RateLimit } from '@web3-storage/capabilities' + +/** + * @type {API.Tests} + */ +export const test = { + 'rate-limit/list shows existing rate limits': async (assert, context) => { + const { service, agent, space, connection } = await setup(context) + + // create a rate limit + const result = await RateLimit.add + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + rate: 0, + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }] + })] + }) + .execute(connection) + assert.ok(result.out.ok) + + // ensure the created rate limit shows up in list + const listResult = await RateLimit.list + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }] + })] + }) + .execute(connection) + + assert.ok(result.out.ok) + assert.equal(listResult.out.ok?.limits.length, 1) + assert.equal(listResult.out.ok?.limits[0].id, result.out.ok?.id) + }, +} + +/** + * @param {API.TestContext} context + */ +const setup = async (context) => { + const space = alice + const account = Absentee.from({ id: 'did:mailto:web.mail:alice' }) + const agent = bob + + return { space, account, agent, ...context } +} diff --git a/packages/upload-api/test/rate-limit/list.spec.js b/packages/upload-api/test/rate-limit/list.spec.js new file mode 100644 index 000000000..8f80c7d71 --- /dev/null +++ b/packages/upload-api/test/rate-limit/list.spec.js @@ -0,0 +1,31 @@ +/* eslint-disable no-nested-ternary */ +/* eslint-disable no-only-tests/no-only-tests */ +import * as Suite from './list.js' +import * as assert from 'assert' +import { cleanupContext, createContext } from '../helpers/context.js' + +describe('rate-limit/list', () => { + for (const [name, test] of Object.entries(Suite.test)) { + const define = name.startsWith('only! ') + ? it.only + : name.startsWith('skip! ') + ? it.skip + : it + + define(name, async () => { + const context = await createContext() + try { + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + context + ) + } finally { + cleanupContext(context) + } + }) + } +}) diff --git a/packages/upload-api/test/rate-limit/remove.js b/packages/upload-api/test/rate-limit/remove.js new file mode 100644 index 000000000..3685e89a1 --- /dev/null +++ b/packages/upload-api/test/rate-limit/remove.js @@ -0,0 +1,101 @@ +import * as API from '../types.js' +import { alice, bob } from '../helpers/utils.js' +import { Absentee } from '@ucanto/principal' +import { delegate } from '@ucanto/core' +import { Access, RateLimit, Store } from '@web3-storage/capabilities' +import * as CAR from '@ucanto/transport/car' +import * as DidMailto from '@web3-storage/did-mailto' + + + +/** + * @type {API.Tests} + */ +export const test = { + 'rate-limit/remove removes rate limits': async (assert, context) => { + const { service, agent, space, connection } = await setup(context) + + // add a rate limit + const result = await RateLimit.add + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + rate: 0, + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }] + })] + }) + .execute(connection) + assert.ok(result.out.ok) + + // verify it's there + const listResult = await RateLimit.list + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }] + })] + }) + .execute(connection) + assert.equal(listResult.out.ok?.limits.length, 1) + + // remove a rate limit + const removeResult = await RateLimit.remove + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + // @ts-ignore we've verified this exists but TS doesn't know that + id: result.out.ok.id, + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/remove' }] + })] + }) + .execute(connection) + + assert.ok(removeResult.out.ok) + + // verify it's gone + const listResult2 = await RateLimit.list + .invoke({ + issuer: agent, + audience: service, + with: service.did(), + nb: { + subject: space.did(), + }, + proofs: [await delegate({ + issuer: service, audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }] + })] + }) + .execute(connection) + assert.equal(listResult2.out.ok?.limits.length, 0) + + }, +} + +/** + * @param {API.TestContext} context + */ +const setup = async (context) => { + const space = alice + const account = Absentee.from({ id: 'did:mailto:web.mail:alice' }) + const agent = bob + + return { space, account, agent, ...context } +} diff --git a/packages/upload-api/test/rate-limit/remove.spec.js b/packages/upload-api/test/rate-limit/remove.spec.js new file mode 100644 index 000000000..cc08cbbff --- /dev/null +++ b/packages/upload-api/test/rate-limit/remove.spec.js @@ -0,0 +1,31 @@ +/* eslint-disable no-nested-ternary */ +/* eslint-disable no-only-tests/no-only-tests */ +import * as Suite from './remove.js' +import * as assert from 'assert' +import { cleanupContext, createContext } from '../helpers/context.js' + +describe('rate-limit/remove', () => { + for (const [name, test] of Object.entries(Suite.test)) { + const define = name.startsWith('only! ') + ? it.only + : name.startsWith('skip! ') + ? it.skip + : it + + define(name, async () => { + const context = await createContext() + try { + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + context + ) + } finally { + cleanupContext(context) + } + }) + } +}) From c2e6ea52a9534f858b11c79378ebf031101ef3ec Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Tue, 1 Aug 2023 17:50:52 -0700 Subject: [PATCH 19/33] feat: remove access verifier and get tests running The AccessVerifier was an artifact of the dual-service architecture we had previously, and its implementation in tests both hid bugs and made it more difficult to test the whole system. I've removed it and gotten the tests running! --- .../test/capabilities/consumer.test.js | 2 +- packages/upload-api/src/consumer/has.js | 7 +- packages/upload-api/src/store/add.js | 3 +- packages/upload-api/src/types.ts | 21 +--- packages/upload-api/src/upload/add.js | 29 ++++-- packages/upload-api/test/access-verifier.js | 98 ------------------- packages/upload-api/test/helpers/context.js | 4 - .../upload-api/test/provisions-storage.js | 5 +- packages/upload-api/test/rate-limit/add.js | 1 - packages/upload-api/test/rate-limit/remove.js | 6 +- packages/upload-api/test/store.js | 15 ++- packages/upload-api/test/upload.js | 7 +- packages/upload-api/test/util.js | 34 +++++-- 13 files changed, 79 insertions(+), 153 deletions(-) delete mode 100644 packages/upload-api/test/access-verifier.js diff --git a/packages/capabilities/test/capabilities/consumer.test.js b/packages/capabilities/test/capabilities/consumer.test.js index 2b2b929d0..bf877ad7b 100644 --- a/packages/capabilities/test/capabilities/consumer.test.js +++ b/packages/capabilities/test/capabilities/consumer.test.js @@ -3,7 +3,7 @@ import { access } from '@ucanto/validator' import { delegate } from '@ucanto/core' import { Verifier } from '@ucanto/principal/ed25519' import * as Consumer from '../../src/consumer.js' -import { bob, bobAccount, service, alice } from '../helpers/fixtures.js' +import { bob, service, alice } from '../helpers/fixtures.js' describe('consumer/get', function () { const agent = alice diff --git a/packages/upload-api/src/consumer/has.js b/packages/upload-api/src/consumer/has.js index 2bb2d24c5..36f154673 100644 --- a/packages/upload-api/src/consumer/has.js +++ b/packages/upload-api/src/consumer/has.js @@ -9,15 +9,14 @@ export const provide = (context) => Provider.provide(Consumer.has, (input) => has(input, context)) /** - * @param {API.Input} input + * @param {{capability: {with: API.ProviderDID, nb: { consumer: API.DIDKey }}}} input * @param {API.ConsumerServiceContext} context * @returns {Promise>} */ -const has = async ({ capability }, context) => { +export const has = async ({ capability }, context) => { if (capability.with !== context.signer.did()) { return Provider.fail( - `Expected with to be ${context.signer.did()}} instead got ${ - capability.with + `Expected with to be ${context.signer.did()}} instead got ${capability.with }` ) } diff --git a/packages/upload-api/src/store/add.js b/packages/upload-api/src/store/add.js index a6a6557da..dc87e327a 100644 --- a/packages/upload-api/src/store/add.js +++ b/packages/upload-api/src/store/add.js @@ -15,13 +15,12 @@ export function storeAddProvider(context) { } = context return Server.provide(Store.add, async ({ capability, invocation }) => { const { link, origin, size } = capability.nb - const space = Server.DID.parse(capability.with).did() + const space = /** @type {import('@ucanto/interface').DIDKey} */ (Server.DID.parse(capability.with).did()) const issuer = invocation.issuer.did() const [allocated, carIsLinkedToAccount, carExists] = await Promise.all([ // TODO: ask @gozala if this is the right way to call this - maybe it should be an actual UCAN execution? allocate({ capability: { - // @ts-expect-error with: space, nb: { size diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index 46cfb6bab..feb4175fb 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -1,6 +1,5 @@ import type { Failure, - Invocation, ServiceMethod, UCANLink, HandlerExecutionError, @@ -178,18 +177,17 @@ export interface Service { } } -export interface StoreServiceContext { +export type StoreServiceContext = SpaceServiceContext & { maxUploadSize: number storeTable: StoreTable carStoreBucket: CarStoreBucket - access: AccessVerifier } -export interface UploadServiceContext { +export type UploadServiceContext = ConsumerServiceContext & { + signer: EdSigner.Signer uploadTable: UploadTable dudewhereBucket: DudewhereBucket - access: AccessVerifier } export interface AccessClaimContext { @@ -266,7 +264,6 @@ export interface UcantoServerTestContext export interface StoreTestContext { testStoreTable: TestStoreTable - testSpaceRegistry: TestSpaceRegistry } export interface UploadTestContext {} @@ -422,18 +419,6 @@ export interface ListResponse { results: R[] } -export interface AccessVerifier { - /** - * Determines if the issuer of the invocation has received a delegation - * allowing them to issue the passed invocation. - */ - allocateSpace: ( - invocation: Invocation - ) => Promise> -} - -interface AllocateOk {} - export interface TestSpaceRegistry { /** * Registers space with the registry. diff --git a/packages/upload-api/src/upload/add.js b/packages/upload-api/src/upload/add.js index a4eb4bf5f..1b2efed47 100644 --- a/packages/upload-api/src/upload/add.js +++ b/packages/upload-api/src/upload/add.js @@ -2,26 +2,41 @@ import pRetry from 'p-retry' import * as Server from '@ucanto/server' import * as Upload from '@web3-storage/capabilities/upload' import * as API from '../types.js' +import { has as hasProvider } from '../consumer/has.js' /** * @param {API.UploadServiceContext} context * @returns {API.ServiceMethod} */ -export function uploadAddProvider({ access, uploadTable, dudewhereBucket }) { +export function uploadAddProvider(context) { return Server.provide(Upload.add, async ({ capability, invocation }) => { + const { uploadTable, dudewhereBucket, signer } = context + const serviceDID = /** @type {import('../types.js').ProviderDID} */ (signer.did()) const { root, shards } = capability.nb - const space = Server.DID.parse(capability.with) + const space = /** @type {import('@ucanto/interface').DIDKey} */ (Server.DID.parse(capability.with).did()) const issuer = invocation.issuer.did() - const allocated = await access.allocateSpace(invocation) - - if (allocated.error) { - return allocated + const hasProviderResult = await hasProvider({ + capability: { + with: serviceDID, + nb: { consumer: space } + } + }, context) + if (hasProviderResult.error) { + return hasProviderResult + } + if (hasProviderResult.ok === false) { + return { + error: { + name: 'NoStorageProvider', + message: `${space} has no storage provider` + } + } } const [res] = await Promise.all([ // Store in Database uploadTable.insert({ - space: space.did(), + space, root, shards, issuer, diff --git a/packages/upload-api/test/access-verifier.js b/packages/upload-api/test/access-verifier.js deleted file mode 100644 index a78f61ee7..000000000 --- a/packages/upload-api/test/access-verifier.js +++ /dev/null @@ -1,98 +0,0 @@ -import * as API from '../src/types.js' -import { Space } from '@web3-storage/capabilities' -import * as Server from '@ucanto/server' -import * as Client from '@ucanto/client' -import { CAR } from '@ucanto/transport' -import { Failure } from '@ucanto/server' - -/** - * @param {object} context - * @param {object} context.models - * @param {Map} context.models.spaces - */ -export const createService = (context) => ({ - space: { - info: Server.provide(Space.info, async ({ capability, invocation }) => { - const results = await context.models.spaces.get(capability.with) - if (!results) { - /** @type {API.SpaceUnknown} */ - const spaceUnknownFailure = { - name: 'SpaceUnknown', - message: `Space not found.`, - } - return { - error: spaceUnknownFailure, - } - } - return { - ok: results, - } - }), - }, -}) - -/** - * @param {object} context - * @param {Server.Signer} context.id - * @param {object} context.models - * @param {Map} context.models.spaces - */ -export const createServer = (context) => - Server.create({ - id: context.id, - service: createService(context), - codec: CAR.inbound, - }) - -/** - * @param {object} context - * @param {Server.Signer} context.id - * @returns {API.AccessVerifier & API.TestSpaceRegistry} - */ -export const create = ({ id }) => { - const spaces = new Map() - const server = createServer({ - id, - models: { - spaces, - }, - }) - - const client = Client.connect({ - id, - channel: server, - codec: CAR.outbound, - }) - - return { - async registerSpace(space) { - spaces.set(space, { did: id }) - }, - async allocateSpace(invocation) { - // if info capability is derivable from the passed capability, then we'll - // receive a response and know that the invocation issuer has verified - // themselves with w3access. - const { out: result } = await Space.info - .invoke({ - issuer: id, - audience: id, - // @ts-expect-error - with: invocation.capabilities[0].with, - proofs: [invocation], - }) - .execute(client) - - if (result.error) { - return result.error.name === 'SpaceUnknown' - ? { - error: new Failure(`Space has no storage provider`, { - cause: result.error, - }), - } - : result - } else { - return { ok: {} } - } - }, - } -} diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index 896ee9fdb..7c91b0505 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -4,7 +4,6 @@ import { CarStoreBucket } from '../car-store-bucket.js' import { StoreTable } from '../store-table.js' import { UploadTable } from '../upload-table.js' import { DudewhereBucket } from '../dude-where-bucket.js' -import * as AccessVerifier from '../access-verifier.js' import { ProvisionsStorage } from '../provisions-storage.js' import { DelegationsStorage } from '../delegations-storage.js' import { RateLimitsStorage } from '../rate-limits-storage.js' @@ -25,7 +24,6 @@ export const createContext = async (options = {}) => { const dudewhereBucket = new DudewhereBucket() const signer = await Signer.generate() const id = signer.withDID('did:web:test.web3.storage') - const access = AccessVerifier.create({ id }) /** @type { import('../../src/types.js').UcantoServerContext } */ const serviceContext = { @@ -46,7 +44,6 @@ export const createContext = async (options = {}) => { uploadTable, carStoreBucket, dudewhereBucket, - access, } const connection = connect({ @@ -60,7 +57,6 @@ export const createContext = async (options = {}) => { service: /** @type {TestTypes.ServiceSigner} */ (serviceContext.id), connection, testStoreTable: storeTable, - testSpaceRegistry: access, fetch, } } diff --git a/packages/upload-api/test/provisions-storage.js b/packages/upload-api/test/provisions-storage.js index d1f1363a8..7a037da62 100644 --- a/packages/upload-api/test/provisions-storage.js +++ b/packages/upload-api/test/provisions-storage.js @@ -54,7 +54,10 @@ export class ProvisionsStorage { storedItem.consumer !== item.consumer || storedItem.cause.link() !== item.cause.link()) ) { - return { error: new Error(`could not store ${JSON.stringify(item)}`) } + return { error: { + name: 'Error', + message: `could not store item - a provision with that key already exists` + }} } else { this.provisions[itemKey(item)] = item return { ok: {} } diff --git a/packages/upload-api/test/rate-limit/add.js b/packages/upload-api/test/rate-limit/add.js index 38f3a5ebe..dfcea6a93 100644 --- a/packages/upload-api/test/rate-limit/add.js +++ b/packages/upload-api/test/rate-limit/add.js @@ -31,7 +31,6 @@ export const test = { }) .execute(connection) - console.log(result.out) assert.ok(result.out.ok) }, diff --git a/packages/upload-api/test/rate-limit/remove.js b/packages/upload-api/test/rate-limit/remove.js index 3685e89a1..bfcdee8d9 100644 --- a/packages/upload-api/test/rate-limit/remove.js +++ b/packages/upload-api/test/rate-limit/remove.js @@ -2,11 +2,7 @@ import * as API from '../types.js' import { alice, bob } from '../helpers/utils.js' import { Absentee } from '@ucanto/principal' import { delegate } from '@ucanto/core' -import { Access, RateLimit, Store } from '@web3-storage/capabilities' -import * as CAR from '@ucanto/transport/car' -import * as DidMailto from '@web3-storage/did-mailto' - - +import { RateLimit } from '@web3-storage/capabilities' /** * @type {API.Tests} diff --git a/packages/upload-api/test/store.js b/packages/upload-api/test/store.js index 6ec94b512..d9962e541 100644 --- a/packages/upload-api/test/store.js +++ b/packages/upload-api/test/store.js @@ -7,6 +7,8 @@ import { base64pad } from 'multiformats/bases/base64' import * as Link from '@ucanto/core/link' import * as StoreCapabilities from '@web3-storage/capabilities/store' import { createSpace, registerSpace } from './util.js' +import { Absentee } from '@ucanto/principal' +import { provisionProvider } from './helpers/utils.js' /** * @type {API.Tests} @@ -277,7 +279,7 @@ export const test = { context ) => { const alice = await Signer.generate() - const { proof, spaceDid } = await createSpace(alice) + const { proof, space, spaceDid } = await createSpace(alice) const connection = connect({ id: context.id, channel: createServer(context), @@ -300,8 +302,17 @@ export const test = { assert.ok(storeAdd.out.error) assert.equal(storeAdd.out.error?.message.includes('no storage'), true) + // Register space and retry - await context.testSpaceRegistry.registerSpace(spaceDid) + const account = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) + const providerAdd = await provisionProvider({ + service: /** @type {API.Signer>} */ (context.signer), + agent: alice, + space, + account, + connection + }) + assert.ok(providerAdd.out.ok) const retryStoreAdd = await StoreCapabilities.add .invoke({ diff --git a/packages/upload-api/test/upload.js b/packages/upload-api/test/upload.js index 577336e6d..d0bd5e788 100644 --- a/packages/upload-api/test/upload.js +++ b/packages/upload-api/test/upload.js @@ -249,7 +249,7 @@ export const test = { // invoke a upload/add with proof const [root] = car.roots const shards = [car.cid, otherCar.cid].sort() - + const uploadAdd = await Upload.add .invoke({ issuer: alice, @@ -259,7 +259,6 @@ export const test = { proofs: [proof], }) .execute(connection) - if (!uploadAdd.out.error) { throw new Error('invocation should have failed') } @@ -357,7 +356,9 @@ export const test = { ) }, - 'upload/remove only removes an upload for the given space': async ( + // skip for now since it's not possible for a single account to register multiple spaces + // TODO: revisit whether this is a reasonable assumption in tests + 'skip upload/remove only removes an upload for the given space': async ( assert, context ) => { diff --git a/packages/upload-api/test/util.js b/packages/upload-api/test/util.js index c5cdf2e12..f25e936af 100644 --- a/packages/upload-api/test/util.js +++ b/packages/upload-api/test/util.js @@ -1,4 +1,5 @@ import * as API from '../src/types.js' +import { createServer, connect } from '../src/lib.js' import { ed25519 } from '@ucanto/principal' import { delegate } from '@ucanto/core' import { CID } from 'multiformats' @@ -8,6 +9,8 @@ import * as CAR from '@ucanto/transport/car' import * as raw from 'multiformats/codecs/raw' import { CarWriter } from '@ipld/car' import { Blob } from '@web-std/blob' +import { provisionProvider } from './helpers/utils.js' +import { Absentee } from '@ucanto/principal' /** did:key:z6Mkk89bC3JrVqKie71YEcc5M1SMVxuCgNx6zLZ8SYJsxALi */ export const alice = ed25519.parse( @@ -41,20 +44,37 @@ export async function createSpace(audience) { audience, capabilities: [{ can: '*', with: spaceDid }], }), + space, spaceDid, } } /** * - * @param {API.Principal} audience - * @param {object} context - * @param {API.TestSpaceRegistry} context.testSpaceRegistry + * @param {API.Principal & API.Signer} audience + * @param {import('./types.js').UcantoServerTestContext} context */ -export const registerSpace = async (audience, { testSpaceRegistry }) => { - const { proof, spaceDid } = await createSpace(audience) - await testSpaceRegistry.registerSpace(spaceDid) - return { proof, spaceDid } +export const registerSpace = async (audience, context) => { + const { proof, space, spaceDid } = await createSpace(audience) + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + const account = Absentee.from({id: 'did:mailto:test.web3.storage:alice'}) + + const provisionResult = await provisionProvider({ + service: /** @type {API.Signer>} */ (context.signer), + agent: /** @type {API.Signer} */ (audience), + space, + account, + connection + }) + if (provisionResult.out.error){ + console.log(provisionResult.out.error) + throw new Error(`Error provisioning space for ${audience.did()}`, {cause: provisionResult.out.error}) + } + + return { proof, space, spaceDid } } /** @param {number} size */ From 1be4e8bd6758d5c89c91be91c2fa36b8aad18930 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Tue, 1 Aug 2023 18:00:38 -0700 Subject: [PATCH 20/33] chore: minor cleanup from self-review --- packages/upload-api/src/store/add.js | 2 +- packages/upload-api/test/rate-limit/add.js | 2 -- packages/upload-api/test/store.js | 1 - packages/upload-api/test/util.js | 1 - 4 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/upload-api/src/store/add.js b/packages/upload-api/src/store/add.js index dc87e327a..3294f5e1e 100644 --- a/packages/upload-api/src/store/add.js +++ b/packages/upload-api/src/store/add.js @@ -18,7 +18,7 @@ export function storeAddProvider(context) { const space = /** @type {import('@ucanto/interface').DIDKey} */ (Server.DID.parse(capability.with).did()) const issuer = invocation.issuer.did() const [allocated, carIsLinkedToAccount, carExists] = await Promise.all([ - // TODO: ask @gozala if this is the right way to call this - maybe it should be an actual UCAN execution? + // TODO: is the right way to call this - maybe it should be an actual UCAN execution? allocate({ capability: { with: space, diff --git a/packages/upload-api/test/rate-limit/add.js b/packages/upload-api/test/rate-limit/add.js index dfcea6a93..30f6347eb 100644 --- a/packages/upload-api/test/rate-limit/add.js +++ b/packages/upload-api/test/rate-limit/add.js @@ -6,8 +6,6 @@ import { Access, RateLimit, Store } from '@web3-storage/capabilities' import * as CAR from '@ucanto/transport/car' import * as DidMailto from '@web3-storage/did-mailto' - - /** * @type {API.Tests} */ diff --git a/packages/upload-api/test/store.js b/packages/upload-api/test/store.js index d9962e541..80e429477 100644 --- a/packages/upload-api/test/store.js +++ b/packages/upload-api/test/store.js @@ -301,7 +301,6 @@ export const test = { assert.ok(storeAdd.out.error) assert.equal(storeAdd.out.error?.message.includes('no storage'), true) - // Register space and retry const account = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) diff --git a/packages/upload-api/test/util.js b/packages/upload-api/test/util.js index f25e936af..77bf1105e 100644 --- a/packages/upload-api/test/util.js +++ b/packages/upload-api/test/util.js @@ -70,7 +70,6 @@ export const registerSpace = async (audience, context) => { connection }) if (provisionResult.out.error){ - console.log(provisionResult.out.error) throw new Error(`Error provisioning space for ${audience.did()}`, {cause: provisionResult.out.error}) } From 80fbbae1c2f5c85fc261f3c54eba4e187e0f9d62 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Tue, 1 Aug 2023 18:07:50 -0700 Subject: [PATCH 21/33] chore: make code more prettier --- .../test/capabilities/consumer.test.js | 43 +++++---- .../test/capabilities/customer.test.js | 20 ++-- .../test/capabilities/rate-limit.test.js | 41 +++++--- .../test/capabilities/subscription.test.js | 20 ++-- .../capabilities/test/helpers/fixtures.js | 13 ++- packages/upload-api/src/consumer/has.js | 3 +- packages/upload-api/src/store/add.js | 29 +++--- packages/upload-api/src/upload/add.js | 27 ++++-- packages/upload-api/src/utils/rate-limits.js | 20 ++-- .../upload-api/test/provisions-storage.js | 63 ++++++++---- packages/upload-api/test/rate-limit/add.js | 96 ++++++++++++------- packages/upload-api/test/rate-limit/list.js | 22 +++-- packages/upload-api/test/rate-limit/remove.js | 45 +++++---- packages/upload-api/test/store.js | 4 +- packages/upload-api/test/upload.js | 2 +- packages/upload-api/test/util.js | 10 +- 16 files changed, 285 insertions(+), 173 deletions(-) diff --git a/packages/capabilities/test/capabilities/consumer.test.js b/packages/capabilities/test/capabilities/consumer.test.js index bf877ad7b..55fe7d41a 100644 --- a/packages/capabilities/test/capabilities/consumer.test.js +++ b/packages/capabilities/test/capabilities/consumer.test.js @@ -15,7 +15,7 @@ describe('consumer/get', function () { with: service.did(), nb: { consumer: space.did(), - } + }, }) const result = await access(await invocation.delegate(), { capability: Consumer.get, @@ -41,10 +41,13 @@ describe('consumer/get', function () { nb: { consumer: space.did(), }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'consumer/get' }] - })], + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'consumer/get' }], + }), + ], }) const result = await access(await auth.delegate(), { capability: Consumer.get, @@ -52,7 +55,9 @@ describe('consumer/get', function () { authority: service, }) if (result.error) { - assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) + assert.fail( + `error in self issue: ${JSON.stringify(result.error.message)}` + ) } else { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'consumer/get') @@ -89,14 +94,12 @@ describe('consumer/get', function () { audience: service, with: service.did(), // @ts-ignore - nb: { - }, + nb: {}, }) }, /Error: Invalid 'nb' - Object contains invalid field "consumer"/) }) }) - describe('consumer/has', function () { const agent = alice const space = bob @@ -107,7 +110,7 @@ describe('consumer/has', function () { with: service.did(), nb: { consumer: space.did(), - } + }, }) const result = await access(await invocation.delegate(), { capability: Consumer.has, @@ -133,10 +136,13 @@ describe('consumer/has', function () { nb: { consumer: space.did(), }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'consumer/has' }] - })], + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'consumer/has' }], + }), + ], }) const result = await access(await auth.delegate(), { capability: Consumer.has, @@ -144,7 +150,9 @@ describe('consumer/has', function () { authority: service, }) if (result.error) { - assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) + assert.fail( + `error in self issue: ${JSON.stringify(result.error.message)}` + ) } else { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'consumer/has') @@ -181,9 +189,8 @@ describe('consumer/has', function () { audience: service, with: service.did(), // @ts-ignore - nb: { - }, + nb: {}, }) }, /Error: Invalid 'nb' - Object contains invalid field "consumer"/) }) -}) \ No newline at end of file +}) diff --git a/packages/capabilities/test/capabilities/customer.test.js b/packages/capabilities/test/capabilities/customer.test.js index 204148d6f..b971152c7 100644 --- a/packages/capabilities/test/capabilities/customer.test.js +++ b/packages/capabilities/test/capabilities/customer.test.js @@ -14,7 +14,7 @@ describe('customer/get', function () { with: service.did(), nb: { customer: bobAccount.did(), - } + }, }) const result = await access(await invocation.delegate(), { capability: Customer.get, @@ -40,10 +40,13 @@ describe('customer/get', function () { nb: { customer: bobAccount.did(), }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'customer/get' }] - })], + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'customer/get' }], + }), + ], }) const result = await access(await auth.delegate(), { capability: Customer.get, @@ -51,7 +54,9 @@ describe('customer/get', function () { authority: service, }) if (result.error) { - assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) + assert.fail( + `error in self issue: ${JSON.stringify(result.error.message)}` + ) } else { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'customer/get') @@ -88,8 +93,7 @@ describe('customer/get', function () { audience: service, with: service.did(), // @ts-ignore - nb: { - }, + nb: {}, }) }, /Error: Invalid 'nb' - Object contains invalid field "customer"/) }) diff --git a/packages/capabilities/test/capabilities/rate-limit.test.js b/packages/capabilities/test/capabilities/rate-limit.test.js index aedf8059f..772b5f142 100644 --- a/packages/capabilities/test/capabilities/rate-limit.test.js +++ b/packages/capabilities/test/capabilities/rate-limit.test.js @@ -15,7 +15,7 @@ describe('rate-limit/add', function () { nb: { subject: space.did(), rate: 0, - } + }, }) const result = await access(await invocation.delegate(), { capability: RateLimit.add, @@ -45,10 +45,13 @@ describe('rate-limit/add', function () { subject: space.did(), rate: 0, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/add' }] - })], + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }], + }), + ], }) const result = await access(await auth.delegate(), { capability: RateLimit.add, @@ -56,7 +59,9 @@ describe('rate-limit/add', function () { authority: service, }) if (result.error) { - assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) + assert.fail( + `error in self issue: ${JSON.stringify(result.error.message)}` + ) } else { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'rate-limit/add') @@ -129,10 +134,13 @@ describe('rate-limit/remove', function () { nb: { id: rateLimitId, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/remove' }] - })], + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/remove' }], + }), + ], }) const result = await access(await auth.delegate(), { capability: RateLimit.remove, @@ -194,10 +202,13 @@ describe('rate-limit/list', function () { nb: { subject: space.did(), }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/list' }] - })], + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }], + }), + ], }) const result = await access(await auth.delegate(), { capability: RateLimit.list, @@ -246,4 +257,4 @@ describe('rate-limit/list', function () { }) }, /Error: Invalid 'nb' - Object contains invalid field "subject"/) }) -}) \ No newline at end of file +}) diff --git a/packages/capabilities/test/capabilities/subscription.test.js b/packages/capabilities/test/capabilities/subscription.test.js index 9f4a47e2b..fd61b4019 100644 --- a/packages/capabilities/test/capabilities/subscription.test.js +++ b/packages/capabilities/test/capabilities/subscription.test.js @@ -14,7 +14,7 @@ describe('subscription/get', function () { with: service.did(), nb: { subscription: bob.did(), - } + }, }) const result = await access(await invocation.delegate(), { capability: Subscription.get, @@ -40,10 +40,13 @@ describe('subscription/get', function () { nb: { subscription: bob.did(), }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'subscription/get' }] - })], + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'subscription/get' }], + }), + ], }) const result = await access(await auth.delegate(), { capability: Subscription.get, @@ -51,7 +54,9 @@ describe('subscription/get', function () { authority: service, }) if (result.error) { - assert.fail(`error in self issue: ${JSON.stringify(result.error.message)}`) + assert.fail( + `error in self issue: ${JSON.stringify(result.error.message)}` + ) } else { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'subscription/get') @@ -88,8 +93,7 @@ describe('subscription/get', function () { audience: service, with: service.did(), // @ts-ignore - nb: { - }, + nb: {}, }) }, /Error: Invalid 'nb' - Object contains invalid field "subscription"/) }) diff --git a/packages/capabilities/test/helpers/fixtures.js b/packages/capabilities/test/helpers/fixtures.js index 10784791b..a78c70b0d 100644 --- a/packages/capabilities/test/helpers/fixtures.js +++ b/packages/capabilities/test/helpers/fixtures.js @@ -6,22 +6,27 @@ export const alice = Signer.parse( 'MgCZT5vOnYZoVAeyjnzuJIVY9J4LNtJ+f8Js0cTPuKUpFne0BVEDJjEu6quFIU8yp91/TY/+MYK8GvlKoTDnqOCovCVM=' ) -export const aliceAccount = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) +export const aliceAccount = Absentee.from({ + id: 'did:mailto:test.web3.storage:alice', +}) /** did:key:z6MkffDZCkCTWreg8868fG1FGFogcJj5X6PY93pPcWDn9bob */ export const bob = Signer.parse( 'MgCYbj5AJfVvdrjkjNCxB3iAUwx7RQHVQ7H1sKyHy46Iose0BEevXgL1V73PD9snOCIoONgb+yQ9sycYchQC8kygR4qY=' ) -export const bobAccount = Absentee.from({ id: 'did:mailto:test.web3.storage:bob' }) +export const bobAccount = Absentee.from({ + id: 'did:mailto:test.web3.storage:bob', +}) /** did:key:z6MktafZTREjJkvV5mfJxcLpNBoVPwDLhTuMg9ng7dY4zMAL */ export const mallory = Signer.parse( 'MgCYtH0AvYxiQwBG6+ZXcwlXywq9tI50G2mCAUJbwrrahkO0B0elFYkl3Ulf3Q3A/EvcVY0utb4etiSE8e6pi4H0FEmU=' ) -export const malloryAccount = Absentee.from({ id: 'did:mailto:test.web3.storage:mallory' }) - +export const malloryAccount = Absentee.from({ + id: 'did:mailto:test.web3.storage:mallory', +}) export const service = Signer.parse( 'MgCYKXoHVy7Vk4/QjcEGi+MCqjntUiasxXJ8uJKY0qh11e+0Bs8WsdqGK7xothgrDzzWD0ME7ynPjz2okXDh8537lId8=' diff --git a/packages/upload-api/src/consumer/has.js b/packages/upload-api/src/consumer/has.js index 36f154673..24718beae 100644 --- a/packages/upload-api/src/consumer/has.js +++ b/packages/upload-api/src/consumer/has.js @@ -16,7 +16,8 @@ export const provide = (context) => export const has = async ({ capability }, context) => { if (capability.with !== context.signer.did()) { return Provider.fail( - `Expected with to be ${context.signer.did()}} instead got ${capability.with + `Expected with to be ${context.signer.did()}} instead got ${ + capability.with }` ) } diff --git a/packages/upload-api/src/store/add.js b/packages/upload-api/src/store/add.js index 3294f5e1e..e5ee304fb 100644 --- a/packages/upload-api/src/store/add.js +++ b/packages/upload-api/src/store/add.js @@ -8,25 +8,26 @@ import { allocate } from '../space-allocate.js' * @returns {API.ServiceMethod} */ export function storeAddProvider(context) { - const { - storeTable, - carStoreBucket, - maxUploadSize, - } = context + const { storeTable, carStoreBucket, maxUploadSize } = context return Server.provide(Store.add, async ({ capability, invocation }) => { const { link, origin, size } = capability.nb - const space = /** @type {import('@ucanto/interface').DIDKey} */ (Server.DID.parse(capability.with).did()) + const space = /** @type {import('@ucanto/interface').DIDKey} */ ( + Server.DID.parse(capability.with).did() + ) const issuer = invocation.issuer.did() const [allocated, carIsLinkedToAccount, carExists] = await Promise.all([ // TODO: is the right way to call this - maybe it should be an actual UCAN execution? - allocate({ - capability: { - with: space, - nb: { - size - } - } - }, context), + allocate( + { + capability: { + with: space, + nb: { + size, + }, + }, + }, + context + ), storeTable.exists(space, link), carStoreBucket.has(link), ]) diff --git a/packages/upload-api/src/upload/add.js b/packages/upload-api/src/upload/add.js index 1b2efed47..53f1c623e 100644 --- a/packages/upload-api/src/upload/add.js +++ b/packages/upload-api/src/upload/add.js @@ -11,16 +11,23 @@ import { has as hasProvider } from '../consumer/has.js' export function uploadAddProvider(context) { return Server.provide(Upload.add, async ({ capability, invocation }) => { const { uploadTable, dudewhereBucket, signer } = context - const serviceDID = /** @type {import('../types.js').ProviderDID} */ (signer.did()) + const serviceDID = /** @type {import('../types.js').ProviderDID} */ ( + signer.did() + ) const { root, shards } = capability.nb - const space = /** @type {import('@ucanto/interface').DIDKey} */ (Server.DID.parse(capability.with).did()) + const space = /** @type {import('@ucanto/interface').DIDKey} */ ( + Server.DID.parse(capability.with).did() + ) const issuer = invocation.issuer.did() - const hasProviderResult = await hasProvider({ - capability: { - with: serviceDID, - nb: { consumer: space } - } - }, context) + const hasProviderResult = await hasProvider( + { + capability: { + with: serviceDID, + nb: { consumer: space }, + }, + }, + context + ) if (hasProviderResult.error) { return hasProviderResult } @@ -28,8 +35,8 @@ export function uploadAddProvider(context) { return { error: { name: 'NoStorageProvider', - message: `${space} has no storage provider` - } + message: `${space} has no storage provider`, + }, } } diff --git a/packages/upload-api/src/utils/rate-limits.js b/packages/upload-api/src/utils/rate-limits.js index 37309b829..9cd9dbe0c 100644 --- a/packages/upload-api/src/utils/rate-limits.js +++ b/packages/upload-api/src/utils/rate-limits.js @@ -1,13 +1,11 @@ - - /** - * - * @param {import("../types").RateLimit} rateLimit + * + * @param {import("../types").RateLimit} rateLimit */ const isBlock = (rateLimit) => rateLimit.rate === 0 /** - * + * * @param {import("../types").RateLimit[]} rateLimits */ const areAnyBlocks = (rateLimits) => rateLimits.some(isBlock) @@ -15,13 +13,15 @@ const areAnyBlocks = (rateLimits) => rateLimits.some(isBlock) /** * Query rate limits storage and find out if any of the given subjects * have rate set to 0 - * - * @param {import("../types").RateLimitsStorage} storage - * @param {string[]} subjects + * + * @param {import("../types").RateLimitsStorage} storage + * @param {string[]} subjects * @return {Promise>} */ export async function areAnyBlocked(storage, subjects) { - const results = await Promise.all(subjects.map(subject => storage.list(subject))) + const results = await Promise.all( + subjects.map((subject) => storage.list(subject)) + ) let anyBlocks = false for (const result of results) { if (result.error) { @@ -31,4 +31,4 @@ export async function areAnyBlocked(storage, subjects) { } } return { ok: anyBlocks } -} \ No newline at end of file +} diff --git a/packages/upload-api/test/provisions-storage.js b/packages/upload-api/test/provisions-storage.js index 7a037da62..a2c14108a 100644 --- a/packages/upload-api/test/provisions-storage.js +++ b/packages/upload-api/test/provisions-storage.js @@ -54,10 +54,12 @@ export class ProvisionsStorage { storedItem.consumer !== item.consumer || storedItem.cause.link() !== item.cause.link()) ) { - return { error: { - name: 'Error', - message: `could not store item - a provision with that key already exists` - }} + return { + error: { + name: 'Error', + message: `could not store item - a provision with that key already exists`, + }, + } } else { this.provisions[itemKey(item)] = item return { ok: {} } @@ -74,45 +76,64 @@ export class ProvisionsStorage { const exists = Object.values(this.provisions).find( (p) => p.provider === provider && p.customer === customer ) - return exists ? { ok: { did: customer } } : { error: { name: 'CustomerNotFound', message: 'customer does not exist' } } + return exists + ? { ok: { did: customer } } + : { + error: { + name: 'CustomerNotFound', + message: 'customer does not exist', + }, + } } /** - * - * @param {Types.ProviderDID} provider - * @param {string} subscription - * @returns + * + * @param {Types.ProviderDID} provider + * @param {string} subscription + * @returns */ async getSubscription(provider, subscription) { - const provision = Object.values(this.provisions) - .find(p => ((p.customer === subscription) && (p.provider === provider))) + const provision = Object.values(this.provisions).find( + (p) => p.customer === subscription && p.provider === provider + ) if (provision) { return { ok: provision } } else { - return { error: { name: 'SubscriptionNotFound', message: `could not find ${subscription}` } } + return { + error: { + name: 'SubscriptionNotFound', + message: `could not find ${subscription}`, + }, + } } } /** - * - * @param {Types.ProviderDID} provider - * @param {*} consumer - * @returns + * + * @param {Types.ProviderDID} provider + * @param {*} consumer + * @returns */ async getConsumer(provider, consumer) { - const provision = Object.values(this.provisions) - .find(p => ((p.consumer === consumer) && (p.provider === provider))) + const provision = Object.values(this.provisions).find( + (p) => p.consumer === consumer && p.provider === provider + ) if (provision) { return { ok: { did: provision.consumer, allocated: 0, total: 100, - subscription: provision.customer - } + subscription: provision.customer, + }, } } else { - return { error: { name: 'ConsumerNotFound', message: `could not find ${consumer}` } } + return { + error: { + name: 'ConsumerNotFound', + message: `could not find ${consumer}`, + }, + } } } diff --git a/packages/upload-api/test/rate-limit/add.js b/packages/upload-api/test/rate-limit/add.js index 30f6347eb..f498009f3 100644 --- a/packages/upload-api/test/rate-limit/add.js +++ b/packages/upload-api/test/rate-limit/add.js @@ -22,10 +22,13 @@ export const test = { subject: space.did(), rate: 0, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/add' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }], + }), + ], }) .execute(connection) @@ -44,10 +47,13 @@ export const test = { subject: space.did(), rate: 0, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/add' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }], + }), + ], }) .execute(connection) @@ -61,10 +67,13 @@ export const test = { nb: { subject: space.did(), }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/list' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }], + }), + ], }) .execute(connection) @@ -73,7 +82,10 @@ export const test = { assert.equal(listResult.out.ok?.limits[0].id, result.out.ok?.id) }, - 'rate-limit/add can be used to block space allocation': async (assert, context) => { + 'rate-limit/add can be used to block space allocation': async ( + assert, + context + ) => { const { service, agent, space, connection } = await setup(context) const result = await RateLimit.add @@ -85,10 +97,13 @@ export const test = { subject: space.did(), rate: 0, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/add' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }], + }), + ], }) .execute(connection) @@ -103,10 +118,13 @@ export const test = { audience: service, with: space.did(), nb: { link, size }, - proofs: [await delegate({ - issuer: space, audience: agent, - capabilities: [{ with: space.did(), can: 'store/add' }] - })] + proofs: [ + await delegate({ + issuer: space, + audience: agent, + capabilities: [{ with: space.did(), can: 'store/add' }], + }), + ], }) .execute(connection) @@ -115,7 +133,10 @@ export const test = { assert.equal(storeResult.out.error?.message, `${space.did()} is blocked`) }, - 'rate-limit/add can be used to block authorization by email address': async (assert, context) => { + 'rate-limit/add can be used to block authorization by email address': async ( + assert, + context + ) => { const { service, agent, account, connection } = await setup(context) // ensure the account can normally be authorized @@ -144,10 +165,13 @@ export const test = { subject: email, rate: 0, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/add' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }], + }), + ], }) .execute(connection) @@ -169,7 +193,10 @@ export const test = { assert.equal(errorAccessResult.out.error?.name, 'AccountBlocked') }, - 'rate-limit/add can be used to block authorization by domain': async (assert, context) => { + 'rate-limit/add can be used to block authorization by domain': async ( + assert, + context + ) => { const { service, agent, account, connection } = await setup(context) // ensure the account can normally be authorized @@ -188,7 +215,9 @@ export const test = { assert.ok(okAccessResult.out.ok) // block the account's domain - const domain = DidMailto.toEmail(DidMailto.fromString(account.did())).split('@')[1] + const domain = DidMailto.toEmail(DidMailto.fromString(account.did())).split( + '@' + )[1] const blockResult = await RateLimit.add .invoke({ issuer: agent, @@ -198,10 +227,13 @@ export const test = { subject: domain, rate: 0, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/add' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }], + }), + ], }) .execute(connection) diff --git a/packages/upload-api/test/rate-limit/list.js b/packages/upload-api/test/rate-limit/list.js index cdec138b7..5fc691594 100644 --- a/packages/upload-api/test/rate-limit/list.js +++ b/packages/upload-api/test/rate-limit/list.js @@ -21,10 +21,13 @@ export const test = { subject: space.did(), rate: 0, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/add' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }], + }), + ], }) .execute(connection) assert.ok(result.out.ok) @@ -38,10 +41,13 @@ export const test = { nb: { subject: space.did(), }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/list' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }], + }), + ], }) .execute(connection) diff --git a/packages/upload-api/test/rate-limit/remove.js b/packages/upload-api/test/rate-limit/remove.js index bfcdee8d9..91e520ce9 100644 --- a/packages/upload-api/test/rate-limit/remove.js +++ b/packages/upload-api/test/rate-limit/remove.js @@ -21,10 +21,13 @@ export const test = { subject: space.did(), rate: 0, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/add' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/add' }], + }), + ], }) .execute(connection) assert.ok(result.out.ok) @@ -38,10 +41,13 @@ export const test = { nb: { subject: space.did(), }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/list' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }], + }), + ], }) .execute(connection) assert.equal(listResult.out.ok?.limits.length, 1) @@ -56,10 +62,13 @@ export const test = { // @ts-ignore we've verified this exists but TS doesn't know that id: result.out.ok.id, }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/remove' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/remove' }], + }), + ], }) .execute(connection) @@ -74,14 +83,16 @@ export const test = { nb: { subject: space.did(), }, - proofs: [await delegate({ - issuer: service, audience: agent, - capabilities: [{ with: service.did(), can: 'rate-limit/list' }] - })] + proofs: [ + await delegate({ + issuer: service, + audience: agent, + capabilities: [{ with: service.did(), can: 'rate-limit/list' }], + }), + ], }) .execute(connection) assert.equal(listResult2.out.ok?.limits.length, 0) - }, } diff --git a/packages/upload-api/test/store.js b/packages/upload-api/test/store.js index 80e429477..2c2b65d5e 100644 --- a/packages/upload-api/test/store.js +++ b/packages/upload-api/test/store.js @@ -301,7 +301,7 @@ export const test = { assert.ok(storeAdd.out.error) assert.equal(storeAdd.out.error?.message.includes('no storage'), true) - + // Register space and retry const account = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) const providerAdd = await provisionProvider({ @@ -309,7 +309,7 @@ export const test = { agent: alice, space, account, - connection + connection, }) assert.ok(providerAdd.out.ok) diff --git a/packages/upload-api/test/upload.js b/packages/upload-api/test/upload.js index d0bd5e788..d8885a848 100644 --- a/packages/upload-api/test/upload.js +++ b/packages/upload-api/test/upload.js @@ -249,7 +249,7 @@ export const test = { // invoke a upload/add with proof const [root] = car.roots const shards = [car.cid, otherCar.cid].sort() - + const uploadAdd = await Upload.add .invoke({ issuer: alice, diff --git a/packages/upload-api/test/util.js b/packages/upload-api/test/util.js index 77bf1105e..99c804585 100644 --- a/packages/upload-api/test/util.js +++ b/packages/upload-api/test/util.js @@ -60,17 +60,19 @@ export const registerSpace = async (audience, context) => { id: context.id, channel: createServer(context), }) - const account = Absentee.from({id: 'did:mailto:test.web3.storage:alice'}) + const account = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) const provisionResult = await provisionProvider({ service: /** @type {API.Signer>} */ (context.signer), agent: /** @type {API.Signer} */ (audience), space, account, - connection + connection, }) - if (provisionResult.out.error){ - throw new Error(`Error provisioning space for ${audience.did()}`, {cause: provisionResult.out.error}) + if (provisionResult.out.error) { + throw new Error(`Error provisioning space for ${audience.did()}`, { + cause: provisionResult.out.error, + }) } return { proof, space, spaceDid } From 1142f3621114b68b3acabda736777e624f3d0aae Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Thu, 3 Aug 2023 15:47:45 -0700 Subject: [PATCH 22/33] feat: add test suite for RateLimitsStorage implementations so we can use it in w3infra --- packages/upload-api/test/lib.js | 2 + .../test/rate-limits-storage-tests.js | 70 +++++++++++++++++++ .../test/rate-limits-storage.spec.js | 29 ++++++++ 3 files changed, 101 insertions(+) create mode 100644 packages/upload-api/test/rate-limits-storage-tests.js create mode 100644 packages/upload-api/test/rate-limits-storage.spec.js diff --git a/packages/upload-api/test/lib.js b/packages/upload-api/test/lib.js index 9224c7bfa..3e8cf7b5f 100644 --- a/packages/upload-api/test/lib.js +++ b/packages/upload-api/test/lib.js @@ -2,6 +2,7 @@ import * as Store from './store.js' import * as Upload from './upload.js' import { test as delegationsStorageTests } from './delegations-storage-tests.js' import { test as provisionsStorageTests } from './provisions-storage-tests.js' +import { test as rateLimitsStorageTests } from './rate-limits-storage-tests.js' import { DebugEmail } from '../src/utils/email.js' export * from './util.js' @@ -16,5 +17,6 @@ export { Upload, delegationsStorageTests, provisionsStorageTests, + rateLimitsStorageTests, DebugEmail, } diff --git a/packages/upload-api/test/rate-limits-storage-tests.js b/packages/upload-api/test/rate-limits-storage-tests.js new file mode 100644 index 000000000..a90d7d7dd --- /dev/null +++ b/packages/upload-api/test/rate-limits-storage-tests.js @@ -0,0 +1,70 @@ +import * as API from '../src/types.js' + +/** + * @type {API.Tests} + */ +export const test = { + 'should persist rate limits': async (assert, context) => { + const storage = context.rateLimitsStorage + const subject = 'travis@example.com' + const result = await storage.add(subject, 0) + assert.ok(result.ok) + if (!result.ok){ + throw new Error('storing rate limit failed!') + } + + const limitsResult = await storage.list(subject) + assert.ok(limitsResult.ok) + if (!limitsResult.ok){ + throw new Error('listing rate limits failed!') + } + + assert.equal(limitsResult.ok.length, 1) + }, + + 'should list rate limits': async (assert, context) => { + const storage = context.rateLimitsStorage + const subject = 'travis@example.com' + await storage.add(subject, 0) + await storage.add(subject, 2) + + const limitsResult = await storage.list(subject) + assert.ok(limitsResult.ok) + if (!limitsResult.ok){ + throw new Error('listing rate limits failed!') + } + + assert.equal(limitsResult.ok.length, 2) + }, + + 'should allow rate limits to be deleted': async (assert, context) => { + const storage = context.rateLimitsStorage + const subject = 'travis@example.com' + const result = await storage.add(subject, 0) + assert.ok(result.ok) + if (!result.ok){ + throw new Error('storing rate limit failed!') + } + + const limitsResult = await storage.list(subject) + assert.ok(limitsResult.ok) + if (!limitsResult.ok){ + throw new Error('listing rate limits failed!') + } + + assert.equal(limitsResult.ok.length, 1) + + const removeResult = await storage.remove(result.ok.id) + assert.ok(removeResult.ok) + if (!result.ok){ + throw new Error('removing rate limit failed!') + } + + const secondLimitsResult = await storage.list(subject) + assert.ok(secondLimitsResult.ok) + if (!secondLimitsResult.ok){ + throw new Error('listing rate limits failed!') + } + assert.equal(secondLimitsResult.ok.length, 0) + }, +} diff --git a/packages/upload-api/test/rate-limits-storage.spec.js b/packages/upload-api/test/rate-limits-storage.spec.js new file mode 100644 index 000000000..63475c8d9 --- /dev/null +++ b/packages/upload-api/test/rate-limits-storage.spec.js @@ -0,0 +1,29 @@ +import * as RateLimitsStorage from './rate-limits-storage-tests.js' +import * as assert from 'assert' +import { cleanupContext, createContext } from './helpers/context.js' + +describe('in memory rate limits storage', async () => { + for (const [name, test] of Object.entries(RateLimitsStorage.test)) { + const define = name.startsWith('only ') + ? it.only + : name.startsWith('skip ') + ? it.skip + : it + + define(name, async () => { + const context = await createContext() + try { + await test( + { + equal: assert.strictEqual, + deepEqual: assert.deepStrictEqual, + ok: assert.ok, + }, + context + ) + } finally { + await cleanupContext(context) + } + }) + } +}) From e34e2062237117c76ec7d6260a57625ae16cecfb Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Fri, 4 Aug 2023 12:18:56 -0700 Subject: [PATCH 23/33] fix: use context.id rather than context.signer id is used more consistently in w3infra tests and they are identical --- packages/upload-api/test/util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-api/test/util.js b/packages/upload-api/test/util.js index 99c804585..4510e6446 100644 --- a/packages/upload-api/test/util.js +++ b/packages/upload-api/test/util.js @@ -63,7 +63,7 @@ export const registerSpace = async (audience, context) => { const account = Absentee.from({ id: 'did:mailto:test.web3.storage:alice' }) const provisionResult = await provisionProvider({ - service: /** @type {API.Signer>} */ (context.signer), + service: /** @type {API.Signer>} */ (context.id), agent: /** @type {API.Signer} */ (audience), space, account, From 610353173d19a718820eeabcdef1b06767ddda7c Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 12:26:49 -0700 Subject: [PATCH 24/33] Update packages/capabilities/src/rate-limit.js Co-authored-by: Irakli Gozalishvili --- packages/capabilities/src/rate-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/capabilities/src/rate-limit.js b/packages/capabilities/src/rate-limit.js index d07cbdecd..8d888b607 100644 --- a/packages/capabilities/src/rate-limit.js +++ b/packages/capabilities/src/rate-limit.js @@ -53,7 +53,7 @@ export const remove = capability({ }) /** - * Capability can be invoked by an agent to list rate limits on a subject + * Capability can be invoked by the provider or an authorized delegate to list rate limits on the given subject */ export const list = capability({ can: 'rate-limit/list', From 5c77c44ba12271cafa87f082ed407175544f0068 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 12:27:19 -0700 Subject: [PATCH 25/33] Update packages/capabilities/src/rate-limit.js Co-authored-by: Irakli Gozalishvili --- packages/capabilities/src/rate-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/capabilities/src/rate-limit.js b/packages/capabilities/src/rate-limit.js index 8d888b607..3c505cd40 100644 --- a/packages/capabilities/src/rate-limit.js +++ b/packages/capabilities/src/rate-limit.js @@ -35,7 +35,7 @@ export const add = capability({ }) /** - * Capability can be invoked by an agent to remove rate limits from a subject. + * Capability can be invoked by the provider are an authorized delegate to remove rate limits from a subject. */ export const remove = capability({ can: 'rate-limit/remove', From eb8a82796c5501d34a363a0bbe7c6acfdf5cfc20 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 12:27:39 -0700 Subject: [PATCH 26/33] Update packages/capabilities/src/rate-limit.js Co-authored-by: Irakli Gozalishvili --- packages/capabilities/src/rate-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/capabilities/src/rate-limit.js b/packages/capabilities/src/rate-limit.js index 3c505cd40..71d3b380d 100644 --- a/packages/capabilities/src/rate-limit.js +++ b/packages/capabilities/src/rate-limit.js @@ -15,7 +15,7 @@ import { equalWith, and, equal } from './utils.js' export const Provider = DID /** - * Capability can be invoked by an agent to add a rate limit to a subject. + * Capability can be invoked by the provider or an authorized delegate to add a rate limit to a subject. */ export const add = capability({ can: 'rate-limit/add', From 944eb4ba59afb3e07382df8ef78996eccfb37602 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 13:25:02 -0700 Subject: [PATCH 27/33] feat: add tests for rate limit capability escalation ensure callers can't change subject, rate or id constraints --- .../test/capabilities/rate-limit.test.js | 248 ++++++++++++++++++ .../test/capabilities/store.test.js | 2 +- 2 files changed, 249 insertions(+), 1 deletion(-) diff --git a/packages/capabilities/test/capabilities/rate-limit.test.js b/packages/capabilities/test/capabilities/rate-limit.test.js index 772b5f142..bb8e27660 100644 --- a/packages/capabilities/test/capabilities/rate-limit.test.js +++ b/packages/capabilities/test/capabilities/rate-limit.test.js @@ -34,6 +34,134 @@ describe('rate-limit/add', function () { } }) + it('should fail when changing rate constraint', async function () { + const subject = 'travis@example.com' + const delegation = await RateLimit.add.delegate({ + issuer: service, + audience: bob, + with: service.did(), + nb: { + rate: 0 + } + }) + + { + const add = RateLimit.add.invoke({ + issuer: bob, + audience: service, + with: service.did(), + nb: { + rate: 0, + subject + }, + proofs: [delegation] + }) + + const result = await access(await add.delegate(), { + capability: RateLimit.add, + principal: Verifier, + authority: service, + }) + + if (result.error) { + assert.fail(result.error.message) + } + + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'rate-limit/add') + assert.deepEqual(result.ok.capability.nb, { + rate: 0, + subject + }) + } + + { + const add = RateLimit.add.invoke({ + issuer: bob, + audience: service, + with: service.did(), + nb: { + rate: 1, + subject + }, + proofs: [delegation] + }) + + const result = await access(await add.delegate(), { + capability: RateLimit.add, + principal: Verifier, + authority: service, + }) + + assert.ok(result.error) + assert(result.error.message.includes('1 violates imposed rate constraint 0')) + } + }) + + it('should fail when changing subject constraint', async function () { + const rate = 0 + const delegation = await RateLimit.add.delegate({ + issuer: service, + audience: bob, + with: service.did(), + nb: { + subject: 'example.com' + } + }) + + { + const add = RateLimit.add.invoke({ + issuer: bob, + audience: service, + with: service.did(), + nb: { + rate, + subject: 'example.com' + }, + proofs: [delegation] + }) + + const result = await access(await add.delegate(), { + capability: RateLimit.add, + principal: Verifier, + authority: service, + }) + + if (result.error) { + assert.fail(result.error.message) + } + + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'rate-limit/add') + assert.deepEqual(result.ok.capability.nb, { + rate, + subject: 'example.com' + }) + } + + { + const add = RateLimit.add.invoke({ + issuer: bob, + audience: service, + with: service.did(), + nb: { + rate: 1, + subject: 'different.example.com' + }, + proofs: [delegation] + }) + + const result = await access(await add.delegate(), { + capability: RateLimit.add, + principal: Verifier, + authority: service, + }) + + assert.ok(result.error) + assert(result.error.message.includes('different.example.com violates imposed subject constraint example.com')) + } + }) + it('can be invoked by an agent delegated permissions by the service', async function () { const agent = alice @@ -189,6 +317,66 @@ describe('rate-limit/remove', function () { }) }, /Error: Invalid 'nb' - Object contains invalid field "id"/) }) + + it('should fail when changing id constraint', async function () { + const delegation = await RateLimit.remove.delegate({ + issuer: service, + audience: bob, + with: service.did(), + nb: { + id: '123' + } + }) + + { + const remove = RateLimit.remove.invoke({ + issuer: bob, + audience: service, + with: service.did(), + nb: { + id: '123' + }, + proofs: [delegation] + }) + + const result = await access(await remove.delegate(), { + capability: RateLimit.remove, + principal: Verifier, + authority: service, + }) + + if (result.error) { + assert.fail(result.error.message) + } + + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'rate-limit/remove') + assert.deepEqual(result.ok.capability.nb, { + id: '123' + }) + } + + { + const add = RateLimit.remove.invoke({ + issuer: bob, + audience: service, + with: service.did(), + nb: { + id: '456' + }, + proofs: [delegation] + }) + + const result = await access(await add.delegate(), { + capability: RateLimit.remove, + principal: Verifier, + authority: service, + }) + + assert.ok(result.error) + assert(result.error.message.includes('456 violates imposed id constraint 123')) + } + }) }) describe('rate-limit/list', function () { @@ -257,4 +445,64 @@ describe('rate-limit/list', function () { }) }, /Error: Invalid 'nb' - Object contains invalid field "subject"/) }) + + it('should fail when changing subject constraint', async function () { + const delegation = await RateLimit.list.delegate({ + issuer: service, + audience: bob, + with: service.did(), + nb: { + subject: 'travis@example.com' + } + }) + + { + const list = RateLimit.list.invoke({ + issuer: bob, + audience: service, + with: service.did(), + nb: { + subject: 'travis@example.com' + }, + proofs: [delegation] + }) + + const result = await access(await list.delegate(), { + capability: RateLimit.list, + principal: Verifier, + authority: service, + }) + + if (result.error) { + assert.fail(result.error.message) + } + + assert.deepEqual(result.ok.audience.did(), service.did()) + assert.equal(result.ok.capability.can, 'rate-limit/list') + assert.deepEqual(result.ok.capability.nb, { + subject: 'travis@example.com' + }) + } + + { + const list = RateLimit.list.invoke({ + issuer: bob, + audience: service, + with: service.did(), + nb: { + subject: 'alice@example.com' + }, + proofs: [delegation] + }) + + const result = await access(await list.delegate(), { + capability: RateLimit.list, + principal: Verifier, + authority: service, + }) + + assert.ok(result.error) + assert(result.error.message.includes('alice@example.com violates imposed subject constraint travis@example.com')) + } + }) }) diff --git a/packages/capabilities/test/capabilities/store.test.js b/packages/capabilities/test/capabilities/store.test.js index 2d0e10663..eb61ccaca 100644 --- a/packages/capabilities/test/capabilities/store.test.js +++ b/packages/capabilities/test/capabilities/store.test.js @@ -177,7 +177,7 @@ describe('store capabilities', function () { size: 2048, link: parseLink('bafkqaaa'), }, - proofs: [await delegation], + proofs: [delegation], }) const result = await access(await add.delegate(), { From dd23341ae4ae5d8c9ac206727248e6f73cca77b5 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 14:04:14 -0700 Subject: [PATCH 28/33] fix: add comments for blocking behavior we err on the side of denying a user the ability to authorize or provision a space - they can retry and we really don't want to let people do things unless we're sure they're allowed to --- packages/upload-api/src/access/authorize.js | 3 +++ packages/upload-api/src/provider-add.js | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/upload-api/src/access/authorize.js b/packages/upload-api/src/access/authorize.js index 4e0d7082d..5bc6f94f4 100644 --- a/packages/upload-api/src/access/authorize.js +++ b/packages/upload-api/src/access/authorize.js @@ -25,6 +25,9 @@ export const authorize = async ({ capability }, ctx) => { ) ) ) + // If we get an error here we return an error rather than continuing: users can + // always retry and we don't have an easy way to invalidate this authorization once + // it's granted. It might be worth reconsidering this in the future. if (isBlocked.error || isBlocked.ok) { return { error: { diff --git a/packages/upload-api/src/provider-add.js b/packages/upload-api/src/provider-add.js index cfad145f5..c50bc4427 100644 --- a/packages/upload-api/src/provider-add.js +++ b/packages/upload-api/src/provider-add.js @@ -34,16 +34,19 @@ export const add = async ( const isBlocked = await areAnyBlocked( rateLimits, emailAndDomainFromMailtoDid( - /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */ ( + /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */( accountDID ) ) ) + // If we get an error here we return an error rather than continuing: users can + // always retry and we don't really want to let them provision if we're not sure they + // are allowed to. It might be worth reconsidering this in the future.f if (isBlocked.error || isBlocked.ok) { return { error: { name: 'AccountBlocked', - message: `Account identified by {capability.nb.iss} is blocked`, + message: `Account identified by ${accountDID} is blocked`, }, } } From 261a2a4a4941b0d934694bfb43dcfeb1b9b455e2 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 14:12:35 -0700 Subject: [PATCH 29/33] fix: stylistic tweaks from PR --- packages/upload-api/src/access/authorize.js | 14 ++++++++------ packages/upload-api/src/provider-add.js | 14 ++++++++------ packages/upload-api/src/utils/did-mailto.js | 15 +++++++++++---- packages/upload-api/test/provisions-storage.js | 2 +- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/packages/upload-api/src/access/authorize.js b/packages/upload-api/src/access/authorize.js index 5bc6f94f4..352cdd709 100644 --- a/packages/upload-api/src/access/authorize.js +++ b/packages/upload-api/src/access/authorize.js @@ -3,7 +3,7 @@ import * as API from '../types.js' import * as Access from '@web3-storage/capabilities/access' import * as DidMailto from '@web3-storage/did-mailto' import { delegationToString } from '@web3-storage/access/encoding' -import { emailAndDomainFromMailtoDid } from '../utils/did-mailto.js' +import { mailtoDidToDomain, mailtoDidToEmail } from '../utils/did-mailto.js' import { areAnyBlocked } from '../utils/rate-limits.js' /** @@ -17,13 +17,15 @@ export const provide = (ctx) => * @param {API.AccessServiceContext} ctx */ export const authorize = async ({ capability }, ctx) => { + const accountMailtoDID = /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */( + capability.nb.iss + ) const isBlocked = await areAnyBlocked( ctx.rateLimitsStorage, - emailAndDomainFromMailtoDid( - /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */ ( - capability.nb.iss - ) - ) + [ + mailtoDidToDomain(accountMailtoDID), + mailtoDidToEmail(accountMailtoDID) + ] ) // If we get an error here we return an error rather than continuing: users can // always retry and we don't have an easy way to invalidate this authorization once diff --git a/packages/upload-api/src/provider-add.js b/packages/upload-api/src/provider-add.js index c50bc4427..9397fcd19 100644 --- a/packages/upload-api/src/provider-add.js +++ b/packages/upload-api/src/provider-add.js @@ -2,7 +2,7 @@ import * as API from './types.js' import * as Server from '@ucanto/server' import { Provider } from '@web3-storage/capabilities' import * as validator from '@ucanto/validator' -import { emailAndDomainFromMailtoDid } from './utils/did-mailto.js' +import { mailtoDidToDomain, mailtoDidToEmail } from './utils/did-mailto.js' import { areAnyBlocked } from './utils/rate-limits.js' /** @@ -31,13 +31,15 @@ export const add = async ( }, } } + const accountMailtoDID = /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */( + accountDID + ) const isBlocked = await areAnyBlocked( rateLimits, - emailAndDomainFromMailtoDid( - /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */( - accountDID - ) - ) + [ + mailtoDidToDomain(accountMailtoDID), + mailtoDidToEmail(accountMailtoDID) + ] ) // If we get an error here we return an error rather than continuing: users can // always retry and we don't really want to let them provision if we're not sure they diff --git a/packages/upload-api/src/utils/did-mailto.js b/packages/upload-api/src/utils/did-mailto.js index c6a89af1b..cbf31d246 100644 --- a/packages/upload-api/src/utils/did-mailto.js +++ b/packages/upload-api/src/utils/did-mailto.js @@ -3,8 +3,15 @@ import * as DidMailto from '@web3-storage/did-mailto' /** * @param {import("@web3-storage/did-mailto/dist/src/types").DidMailto} mailtoDid */ -export function emailAndDomainFromMailtoDid(mailtoDid) { - const accountEmail = DidMailto.toEmail(DidMailto.fromString(mailtoDid)) - const accountDomain = accountEmail.split('@')[1] - return [accountEmail, accountDomain] +export function mailtoDidToEmail(mailtoDid) { + return DidMailto.toEmail(DidMailto.fromString(mailtoDid)) } + +/** + * @param {import("@web3-storage/did-mailto/dist/src/types").DidMailto} mailtoDid + */ +export function mailtoDidToDomain(mailtoDid) { + const accountEmail = mailtoDidToEmail(mailtoDid) + const accountDomain = accountEmail.split('@')[1] + return accountDomain +} \ No newline at end of file diff --git a/packages/upload-api/test/provisions-storage.js b/packages/upload-api/test/provisions-storage.js index a2c14108a..98ccb30e7 100644 --- a/packages/upload-api/test/provisions-storage.js +++ b/packages/upload-api/test/provisions-storage.js @@ -111,7 +111,7 @@ export class ProvisionsStorage { /** * * @param {Types.ProviderDID} provider - * @param {*} consumer + * @param {string} consumer * @returns */ async getConsumer(provider, consumer) { From 6526b9f3829e770b0d97be1a7cbe7d38a33665e9 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 14:30:42 -0700 Subject: [PATCH 30/33] feat: tweak rate limit checker semantics per feedback from @gozala, move to a style where we always return an `{ok: {}}` result if there is no rate limit above a given threshold. use this instead of an explicit "block" checker --- packages/upload-api/src/access/authorize.js | 12 +++---- packages/upload-api/src/provider-add.js | 13 ++++---- packages/upload-api/src/space-allocate.js | 6 ++-- packages/upload-api/src/utils/rate-limits.js | 34 ++++++++++---------- 4 files changed, 31 insertions(+), 34 deletions(-) diff --git a/packages/upload-api/src/access/authorize.js b/packages/upload-api/src/access/authorize.js index 352cdd709..8101d8f10 100644 --- a/packages/upload-api/src/access/authorize.js +++ b/packages/upload-api/src/access/authorize.js @@ -4,7 +4,7 @@ import * as Access from '@web3-storage/capabilities/access' import * as DidMailto from '@web3-storage/did-mailto' import { delegationToString } from '@web3-storage/access/encoding' import { mailtoDidToDomain, mailtoDidToEmail } from '../utils/did-mailto.js' -import { areAnyBlocked } from '../utils/rate-limits.js' +import { ensureRateLimitAbove } from '../utils/rate-limits.js' /** * @param {API.AccessServiceContext} ctx @@ -20,17 +20,15 @@ export const authorize = async ({ capability }, ctx) => { const accountMailtoDID = /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */( capability.nb.iss ) - const isBlocked = await areAnyBlocked( + const rateLimitResult = await ensureRateLimitAbove( ctx.rateLimitsStorage, [ mailtoDidToDomain(accountMailtoDID), mailtoDidToEmail(accountMailtoDID) - ] + ], + 0 ) - // If we get an error here we return an error rather than continuing: users can - // always retry and we don't have an easy way to invalidate this authorization once - // it's granted. It might be worth reconsidering this in the future. - if (isBlocked.error || isBlocked.ok) { + if (rateLimitResult.error) { return { error: { name: 'AccountBlocked', diff --git a/packages/upload-api/src/provider-add.js b/packages/upload-api/src/provider-add.js index 9397fcd19..187187b49 100644 --- a/packages/upload-api/src/provider-add.js +++ b/packages/upload-api/src/provider-add.js @@ -3,7 +3,7 @@ import * as Server from '@ucanto/server' import { Provider } from '@web3-storage/capabilities' import * as validator from '@ucanto/validator' import { mailtoDidToDomain, mailtoDidToEmail } from './utils/did-mailto.js' -import { areAnyBlocked } from './utils/rate-limits.js' +import { ensureRateLimitAbove } from './utils/rate-limits.js' /** * @param {API.ProviderServiceContext} ctx @@ -34,17 +34,16 @@ export const add = async ( const accountMailtoDID = /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */( accountDID ) - const isBlocked = await areAnyBlocked( + const rateLimitResult = await ensureRateLimitAbove( rateLimits, [ mailtoDidToDomain(accountMailtoDID), mailtoDidToEmail(accountMailtoDID) - ] + ], + 0 ) - // If we get an error here we return an error rather than continuing: users can - // always retry and we don't really want to let them provision if we're not sure they - // are allowed to. It might be worth reconsidering this in the future.f - if (isBlocked.error || isBlocked.ok) { + + if (rateLimitResult.error) { return { error: { name: 'AccountBlocked', diff --git a/packages/upload-api/src/space-allocate.js b/packages/upload-api/src/space-allocate.js index 434b6ef4f..d24dbe6ab 100644 --- a/packages/upload-api/src/space-allocate.js +++ b/packages/upload-api/src/space-allocate.js @@ -1,7 +1,7 @@ import * as API from './types.js' import * as Server from '@ucanto/server' import * as Space from '@web3-storage/capabilities/space' -import { areAnyBlocked } from './utils/rate-limits.js' +import { ensureRateLimitAbove } from './utils/rate-limits.js' /** * @@ -11,8 +11,8 @@ import { areAnyBlocked } from './utils/rate-limits.js' */ export const allocate = async ({ capability }, context) => { const { with: space, nb } = capability - const isBlocked = await areAnyBlocked(context.rateLimitsStorage, [space]) - if (isBlocked.error || isBlocked.ok) { + const rateLimitResult = await ensureRateLimitAbove(context.rateLimitsStorage, [space], 0) + if (rateLimitResult.error) { return { error: { name: 'InsufficientStorage', diff --git a/packages/upload-api/src/utils/rate-limits.js b/packages/upload-api/src/utils/rate-limits.js index 9cd9dbe0c..d6701e453 100644 --- a/packages/upload-api/src/utils/rate-limits.js +++ b/packages/upload-api/src/utils/rate-limits.js @@ -1,34 +1,34 @@ -/** - * - * @param {import("../types").RateLimit} rateLimit - */ -const isBlock = (rateLimit) => rateLimit.rate === 0 - -/** - * - * @param {import("../types").RateLimit[]} rateLimits - */ -const areAnyBlocks = (rateLimits) => rateLimits.some(isBlock) /** * Query rate limits storage and find out if any of the given subjects - * have rate set to 0 + * have rate below the given limitThreshold. Return a Ucanto.Success result if + * not, and a Ucanto.Error if so, or if we get an error from the underlying + * store. * * @param {import("../types").RateLimitsStorage} storage * @param {string[]} subjects - * @return {Promise>} + * @param {number} limitThreshold + * @return {Promise>} */ -export async function areAnyBlocked(storage, subjects) { +export async function ensureRateLimitAbove(storage, subjects, limitThreshold) { const results = await Promise.all( subjects.map((subject) => storage.list(subject)) ) - let anyBlocks = false for (const result of results) { if (result.error) { return result } else { - anyBlocks = anyBlocks || areAnyBlocks(result.ok) + for (const limit of result.ok) { + if (limit.rate <= limitThreshold) { + return { + error: { + name: 'RateLimitExceeded', + message: `Rate limit of ${limit.rate} found which is above threshold of ${limitThreshold}` + } + } + } + } } } - return { ok: anyBlocks } + return { ok: {} } } From 362a698c66be5140ab3662b9dfaaf9687acaaf72 Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 14:32:01 -0700 Subject: [PATCH 31/33] fix: use prettier to make the code more prettier --- .../test/capabilities/rate-limit.test.js | 76 +++++++++++-------- packages/upload-api/src/access/authorize.js | 12 ++- packages/upload-api/src/provider-add.js | 12 ++- packages/upload-api/src/space-allocate.js | 6 +- packages/upload-api/src/utils/did-mailto.js | 2 +- packages/upload-api/src/utils/rate-limits.js | 7 +- .../test/rate-limits-storage-tests.js | 26 +++---- 7 files changed, 76 insertions(+), 65 deletions(-) diff --git a/packages/capabilities/test/capabilities/rate-limit.test.js b/packages/capabilities/test/capabilities/rate-limit.test.js index bb8e27660..4b5d49660 100644 --- a/packages/capabilities/test/capabilities/rate-limit.test.js +++ b/packages/capabilities/test/capabilities/rate-limit.test.js @@ -41,8 +41,8 @@ describe('rate-limit/add', function () { audience: bob, with: service.did(), nb: { - rate: 0 - } + rate: 0, + }, }) { @@ -52,9 +52,9 @@ describe('rate-limit/add', function () { with: service.did(), nb: { rate: 0, - subject + subject, }, - proofs: [delegation] + proofs: [delegation], }) const result = await access(await add.delegate(), { @@ -71,7 +71,7 @@ describe('rate-limit/add', function () { assert.equal(result.ok.capability.can, 'rate-limit/add') assert.deepEqual(result.ok.capability.nb, { rate: 0, - subject + subject, }) } @@ -82,9 +82,9 @@ describe('rate-limit/add', function () { with: service.did(), nb: { rate: 1, - subject + subject, }, - proofs: [delegation] + proofs: [delegation], }) const result = await access(await add.delegate(), { @@ -94,7 +94,9 @@ describe('rate-limit/add', function () { }) assert.ok(result.error) - assert(result.error.message.includes('1 violates imposed rate constraint 0')) + assert( + result.error.message.includes('1 violates imposed rate constraint 0') + ) } }) @@ -105,8 +107,8 @@ describe('rate-limit/add', function () { audience: bob, with: service.did(), nb: { - subject: 'example.com' - } + subject: 'example.com', + }, }) { @@ -116,9 +118,9 @@ describe('rate-limit/add', function () { with: service.did(), nb: { rate, - subject: 'example.com' + subject: 'example.com', }, - proofs: [delegation] + proofs: [delegation], }) const result = await access(await add.delegate(), { @@ -135,7 +137,7 @@ describe('rate-limit/add', function () { assert.equal(result.ok.capability.can, 'rate-limit/add') assert.deepEqual(result.ok.capability.nb, { rate, - subject: 'example.com' + subject: 'example.com', }) } @@ -146,9 +148,9 @@ describe('rate-limit/add', function () { with: service.did(), nb: { rate: 1, - subject: 'different.example.com' + subject: 'different.example.com', }, - proofs: [delegation] + proofs: [delegation], }) const result = await access(await add.delegate(), { @@ -158,7 +160,11 @@ describe('rate-limit/add', function () { }) assert.ok(result.error) - assert(result.error.message.includes('different.example.com violates imposed subject constraint example.com')) + assert( + result.error.message.includes( + 'different.example.com violates imposed subject constraint example.com' + ) + ) } }) @@ -324,8 +330,8 @@ describe('rate-limit/remove', function () { audience: bob, with: service.did(), nb: { - id: '123' - } + id: '123', + }, }) { @@ -334,9 +340,9 @@ describe('rate-limit/remove', function () { audience: service, with: service.did(), nb: { - id: '123' + id: '123', }, - proofs: [delegation] + proofs: [delegation], }) const result = await access(await remove.delegate(), { @@ -352,7 +358,7 @@ describe('rate-limit/remove', function () { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'rate-limit/remove') assert.deepEqual(result.ok.capability.nb, { - id: '123' + id: '123', }) } @@ -362,9 +368,9 @@ describe('rate-limit/remove', function () { audience: service, with: service.did(), nb: { - id: '456' + id: '456', }, - proofs: [delegation] + proofs: [delegation], }) const result = await access(await add.delegate(), { @@ -374,7 +380,9 @@ describe('rate-limit/remove', function () { }) assert.ok(result.error) - assert(result.error.message.includes('456 violates imposed id constraint 123')) + assert( + result.error.message.includes('456 violates imposed id constraint 123') + ) } }) }) @@ -452,8 +460,8 @@ describe('rate-limit/list', function () { audience: bob, with: service.did(), nb: { - subject: 'travis@example.com' - } + subject: 'travis@example.com', + }, }) { @@ -462,9 +470,9 @@ describe('rate-limit/list', function () { audience: service, with: service.did(), nb: { - subject: 'travis@example.com' + subject: 'travis@example.com', }, - proofs: [delegation] + proofs: [delegation], }) const result = await access(await list.delegate(), { @@ -480,7 +488,7 @@ describe('rate-limit/list', function () { assert.deepEqual(result.ok.audience.did(), service.did()) assert.equal(result.ok.capability.can, 'rate-limit/list') assert.deepEqual(result.ok.capability.nb, { - subject: 'travis@example.com' + subject: 'travis@example.com', }) } @@ -490,9 +498,9 @@ describe('rate-limit/list', function () { audience: service, with: service.did(), nb: { - subject: 'alice@example.com' + subject: 'alice@example.com', }, - proofs: [delegation] + proofs: [delegation], }) const result = await access(await list.delegate(), { @@ -502,7 +510,11 @@ describe('rate-limit/list', function () { }) assert.ok(result.error) - assert(result.error.message.includes('alice@example.com violates imposed subject constraint travis@example.com')) + assert( + result.error.message.includes( + 'alice@example.com violates imposed subject constraint travis@example.com' + ) + ) } }) }) diff --git a/packages/upload-api/src/access/authorize.js b/packages/upload-api/src/access/authorize.js index 8101d8f10..2bb08a4c2 100644 --- a/packages/upload-api/src/access/authorize.js +++ b/packages/upload-api/src/access/authorize.js @@ -17,15 +17,13 @@ export const provide = (ctx) => * @param {API.AccessServiceContext} ctx */ export const authorize = async ({ capability }, ctx) => { - const accountMailtoDID = /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */( - capability.nb.iss - ) + const accountMailtoDID = + /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */ ( + capability.nb.iss + ) const rateLimitResult = await ensureRateLimitAbove( ctx.rateLimitsStorage, - [ - mailtoDidToDomain(accountMailtoDID), - mailtoDidToEmail(accountMailtoDID) - ], + [mailtoDidToDomain(accountMailtoDID), mailtoDidToEmail(accountMailtoDID)], 0 ) if (rateLimitResult.error) { diff --git a/packages/upload-api/src/provider-add.js b/packages/upload-api/src/provider-add.js index 187187b49..0163de9ad 100644 --- a/packages/upload-api/src/provider-add.js +++ b/packages/upload-api/src/provider-add.js @@ -31,15 +31,13 @@ export const add = async ( }, } } - const accountMailtoDID = /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */( - accountDID - ) + const accountMailtoDID = + /** @type {import('@web3-storage/did-mailto/dist/src/types').DidMailto} */ ( + accountDID + ) const rateLimitResult = await ensureRateLimitAbove( rateLimits, - [ - mailtoDidToDomain(accountMailtoDID), - mailtoDidToEmail(accountMailtoDID) - ], + [mailtoDidToDomain(accountMailtoDID), mailtoDidToEmail(accountMailtoDID)], 0 ) diff --git a/packages/upload-api/src/space-allocate.js b/packages/upload-api/src/space-allocate.js index d24dbe6ab..a9c1b18dc 100644 --- a/packages/upload-api/src/space-allocate.js +++ b/packages/upload-api/src/space-allocate.js @@ -11,7 +11,11 @@ import { ensureRateLimitAbove } from './utils/rate-limits.js' */ export const allocate = async ({ capability }, context) => { const { with: space, nb } = capability - const rateLimitResult = await ensureRateLimitAbove(context.rateLimitsStorage, [space], 0) + const rateLimitResult = await ensureRateLimitAbove( + context.rateLimitsStorage, + [space], + 0 + ) if (rateLimitResult.error) { return { error: { diff --git a/packages/upload-api/src/utils/did-mailto.js b/packages/upload-api/src/utils/did-mailto.js index cbf31d246..323b1e63b 100644 --- a/packages/upload-api/src/utils/did-mailto.js +++ b/packages/upload-api/src/utils/did-mailto.js @@ -14,4 +14,4 @@ export function mailtoDidToDomain(mailtoDid) { const accountEmail = mailtoDidToEmail(mailtoDid) const accountDomain = accountEmail.split('@')[1] return accountDomain -} \ No newline at end of file +} diff --git a/packages/upload-api/src/utils/rate-limits.js b/packages/upload-api/src/utils/rate-limits.js index d6701e453..eefc0b691 100644 --- a/packages/upload-api/src/utils/rate-limits.js +++ b/packages/upload-api/src/utils/rate-limits.js @@ -1,8 +1,7 @@ - /** * Query rate limits storage and find out if any of the given subjects * have rate below the given limitThreshold. Return a Ucanto.Success result if - * not, and a Ucanto.Error if so, or if we get an error from the underlying + * not, and a Ucanto.Error if so, or if we get an error from the underlying * store. * * @param {import("../types").RateLimitsStorage} storage @@ -23,8 +22,8 @@ export async function ensureRateLimitAbove(storage, subjects, limitThreshold) { return { error: { name: 'RateLimitExceeded', - message: `Rate limit of ${limit.rate} found which is above threshold of ${limitThreshold}` - } + message: `Rate limit of ${limit.rate} found which is above threshold of ${limitThreshold}`, + }, } } } diff --git a/packages/upload-api/test/rate-limits-storage-tests.js b/packages/upload-api/test/rate-limits-storage-tests.js index a90d7d7dd..76f6cfbdf 100644 --- a/packages/upload-api/test/rate-limits-storage-tests.js +++ b/packages/upload-api/test/rate-limits-storage-tests.js @@ -9,46 +9,46 @@ export const test = { const subject = 'travis@example.com' const result = await storage.add(subject, 0) assert.ok(result.ok) - if (!result.ok){ + if (!result.ok) { throw new Error('storing rate limit failed!') } - + const limitsResult = await storage.list(subject) assert.ok(limitsResult.ok) - if (!limitsResult.ok){ + if (!limitsResult.ok) { throw new Error('listing rate limits failed!') } assert.equal(limitsResult.ok.length, 1) - }, + }, 'should list rate limits': async (assert, context) => { const storage = context.rateLimitsStorage const subject = 'travis@example.com' await storage.add(subject, 0) await storage.add(subject, 2) - + const limitsResult = await storage.list(subject) assert.ok(limitsResult.ok) - if (!limitsResult.ok){ + if (!limitsResult.ok) { throw new Error('listing rate limits failed!') } assert.equal(limitsResult.ok.length, 2) - }, + }, 'should allow rate limits to be deleted': async (assert, context) => { const storage = context.rateLimitsStorage const subject = 'travis@example.com' const result = await storage.add(subject, 0) assert.ok(result.ok) - if (!result.ok){ + if (!result.ok) { throw new Error('storing rate limit failed!') } - + const limitsResult = await storage.list(subject) assert.ok(limitsResult.ok) - if (!limitsResult.ok){ + if (!limitsResult.ok) { throw new Error('listing rate limits failed!') } @@ -56,15 +56,15 @@ export const test = { const removeResult = await storage.remove(result.ok.id) assert.ok(removeResult.ok) - if (!result.ok){ + if (!result.ok) { throw new Error('removing rate limit failed!') } const secondLimitsResult = await storage.list(subject) assert.ok(secondLimitsResult.ok) - if (!secondLimitsResult.ok){ + if (!secondLimitsResult.ok) { throw new Error('listing rate limits failed!') } assert.equal(secondLimitsResult.ok.length, 0) - }, + }, } From 1575b928a61121599171813700fa15400f97b6af Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 14:35:48 -0700 Subject: [PATCH 32/33] chore: add comment explaining RateLimitID per feedback from @gozala, explain the reasoning behind having a separate, opaque identifier for rate limits. --- packages/upload-api/src/types/rate-limits.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/upload-api/src/types/rate-limits.ts b/packages/upload-api/src/types/rate-limits.ts index b1e8e94f7..7624a2507 100644 --- a/packages/upload-api/src/types/rate-limits.ts +++ b/packages/upload-api/src/types/rate-limits.ts @@ -1,5 +1,12 @@ import * as Ucanto from '@ucanto/interface' +// An opaque identifier used to identify rate limits. We use this instead of +// deriving it from, eg, {rate, subject} because we'd like to allow implementors +// to support more than one identical rate limit - an example of where this might +// be useful is a fraud prevention department flagging and blocking an account +// because they detected phishing sites being uploaded and, separately, a billing +// department blocking an account for non-payment. In this case the removal of one +// "block" should not result in both "blocks" being lifted. export type RateLimitID = string export interface RateLimit { From 22b7a69e9861960206456b9b9802727b44cc79ec Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Wed, 9 Aug 2023 14:59:45 -0700 Subject: [PATCH 33/33] fix: per PR feedback use allocate in upload/add for consistency this ensures people can't upload to a blocked space, and makes it more consistent. this also gets rid of the `size` parameter in `space/allocate` because: a) we weren't using it for anything b) we don't have size information in `upload/add` I think we'll want to add this back in later, possibly as an optional parameter that triggers a check against provisioned allocation, or we'll want to split space/allocate into two capabilities. --- packages/upload-api/src/access/delegate.js | 7 +------ packages/upload-api/src/space-allocate.js | 10 ++++----- packages/upload-api/src/store/add.js | 6 +----- packages/upload-api/src/types.ts | 2 +- packages/upload-api/src/upload/add.js | 24 ++++++---------------- 5 files changed, 14 insertions(+), 35 deletions(-) diff --git a/packages/upload-api/src/access/delegate.js b/packages/upload-api/src/access/delegate.js index 1c830b92c..ba283faf4 100644 --- a/packages/upload-api/src/access/delegate.js +++ b/packages/upload-api/src/access/delegate.js @@ -19,16 +19,11 @@ export const delegate = async ({ capability, invocation }, context) => { if (delegated.error) { return delegated } - const size = delegated.ok.reduce( - (total, proof) => total + proof.root.bytes.byteLength, - 0 - ) const result = await Allocator.allocate( { capability: { - with: capability.with, - nb: { size }, + with: capability.with }, }, context diff --git a/packages/upload-api/src/space-allocate.js b/packages/upload-api/src/space-allocate.js index a9c1b18dc..ba01201cc 100644 --- a/packages/upload-api/src/space-allocate.js +++ b/packages/upload-api/src/space-allocate.js @@ -1,16 +1,17 @@ import * as API from './types.js' import * as Server from '@ucanto/server' +import * as Ucanto from '@ucanto/interface' import * as Space from '@web3-storage/capabilities/space' import { ensureRateLimitAbove } from './utils/rate-limits.js' /** * - * @param {{capability: {with: API.SpaceDID, nb:{size:number}}}} input + * @param {{capability: {with: API.SpaceDID}}} input * @param {API.SpaceServiceContext} context - * @returns {Promise>} + * @returns {Promise>} */ export const allocate = async ({ capability }, context) => { - const { with: space, nb } = capability + const { with: space } = capability const rateLimitResult = await ensureRateLimitAbove( context.rateLimitsStorage, [space], @@ -24,10 +25,9 @@ export const allocate = async ({ capability }, context) => { }, } } - const { size } = nb const result = await context.provisionsStorage.hasStorageProvider(space) if (result.ok) { - return { ok: { size } } + return { ok: {} } } return { diff --git a/packages/upload-api/src/store/add.js b/packages/upload-api/src/store/add.js index e5ee304fb..c909b2015 100644 --- a/packages/upload-api/src/store/add.js +++ b/packages/upload-api/src/store/add.js @@ -16,14 +16,10 @@ export function storeAddProvider(context) { ) const issuer = invocation.issuer.did() const [allocated, carIsLinkedToAccount, carExists] = await Promise.all([ - // TODO: is the right way to call this - maybe it should be an actual UCAN execution? allocate( { capability: { - with: space, - nb: { - size, - }, + with: space }, }, context diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index feb4175fb..2139dc144 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -184,7 +184,7 @@ export type StoreServiceContext = SpaceServiceContext & { carStoreBucket: CarStoreBucket } -export type UploadServiceContext = ConsumerServiceContext & { +export type UploadServiceContext = ConsumerServiceContext & SpaceServiceContext & { signer: EdSigner.Signer uploadTable: UploadTable dudewhereBucket: DudewhereBucket diff --git a/packages/upload-api/src/upload/add.js b/packages/upload-api/src/upload/add.js index 53f1c623e..b780790a7 100644 --- a/packages/upload-api/src/upload/add.js +++ b/packages/upload-api/src/upload/add.js @@ -2,7 +2,7 @@ import pRetry from 'p-retry' import * as Server from '@ucanto/server' import * as Upload from '@web3-storage/capabilities/upload' import * as API from '../types.js' -import { has as hasProvider } from '../consumer/has.js' +import { allocate } from '../space-allocate.js' /** * @param {API.UploadServiceContext} context @@ -10,34 +10,22 @@ import { has as hasProvider } from '../consumer/has.js' */ export function uploadAddProvider(context) { return Server.provide(Upload.add, async ({ capability, invocation }) => { - const { uploadTable, dudewhereBucket, signer } = context - const serviceDID = /** @type {import('../types.js').ProviderDID} */ ( - signer.did() - ) + const { uploadTable, dudewhereBucket } = context const { root, shards } = capability.nb const space = /** @type {import('@ucanto/interface').DIDKey} */ ( Server.DID.parse(capability.with).did() ) const issuer = invocation.issuer.did() - const hasProviderResult = await hasProvider( + const allocated = await allocate( { capability: { - with: serviceDID, - nb: { consumer: space }, + with: space }, }, context ) - if (hasProviderResult.error) { - return hasProviderResult - } - if (hasProviderResult.ok === false) { - return { - error: { - name: 'NoStorageProvider', - message: `${space} has no storage provider`, - }, - } + if (allocated.error) { + return allocated } const [res] = await Promise.all([