From 3dbe8c8f48baa27de45f601b03eb8f4d59fa07af Mon Sep 17 00:00:00 2001 From: Kyle Espinola Date: Thu, 5 Oct 2023 14:21:11 +0200 Subject: [PATCH] feat: /uploads allow users to upload a single file and returns its uri and cid --- .gitignore | 1 + README.md | 33 +++++++--- package-lock.json | 58 ++++++++++++++--- package.json | 6 +- server.js | 154 ++++++++++++++++++++++++++++++++++++++++++---- web3.js | 68 ++++++++++++++++++++ 6 files changed, 292 insertions(+), 28 deletions(-) create mode 100644 web3.js diff --git a/.gitignore b/.gitignore index ce7abf3..4f09c70 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ yarn-error.log* .pnpm-debug.log* # local env files +.env .env*.local # vercel diff --git a/README.md b/README.md index 1049088..50e923f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,29 @@ -# HUB Uploads +# HUB Upload API -A simple REST wrapper for web3up used as a shared uploads service within HUB. +This is the Upload API for HUB. It allows users to upload files to the server. -## Getting started +## Setup -``` -npm i -npm run start -``` \ No newline at end of file +To setup the server, follow these steps: + +1. Install the dependencies by running `npm install` +2. Set the environment variables `WEB3_UP_KEY`, `WEB3_UP_PROOF`, and `WEB3_UP_GATEWAY`. Note that `WEB3_UP_GATEWAY` should be an IPFS compatible gateway for retrieving assets over HTTP. +3. Start the server by running `npm start` + +## Routes + +The available routes are: + +- POST /uploads: Upload a file. This route consumes multipart/form-data and returns the URI and CID of the uploaded file. + +## Swagger Documentation + +The Swagger documentation for the API is available at /documentation. + +## Error Handling + +If there is a validation error in the request, the server will respond with a 400 status code and the validation error. + +## File Upload + +The server uses the `fastify-multipart` plugin to handle file uploads. The server is configured to accept a maximum of 1 file field and 0 non-file fields. diff --git a/package-lock.json b/package-lock.json index eecbee5..b5cf6b7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,19 @@ { "name": "hub-uploads", - "version": "1.0.0", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "hub-uploads", - "version": "1.0.0", + "version": "0.1.0", "license": "ISC", "dependencies": { + "@fastify/env": "^4.2.0", + "@fastify/error": "^3.4.0", "@fastify/multipart": "^8.0.0", + "@fastify/swagger": "^8.11.0", + "@fastify/swagger-ui": "^1.10.0", "@ipld/car": "^5.2.4", "@ucanto/principal": "^8.1.0", "@web3-storage/w3up-client": "^8.0.3", @@ -50,6 +54,15 @@ "resolved": "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-1.3.0.tgz", "integrity": "sha512-J8TOSBq3SoZbDhM9+R/u77hP93gz/rajSA+K2kGyijPpORPWUXHUpTaleoj+92As0S9uPRP7Oi8IqMf0u+ro6A==" }, + "node_modules/@fastify/env": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@fastify/env/-/env-4.2.0.tgz", + "integrity": "sha512-sj1ehQZPD6tty+6bhzZw1uS2K2s6eOB46maJ2fE+AuTYxdHKVVW/AHXqCYGu3nH9kgzdXsheu3/148ZoBeoQOw==", + "dependencies": { + "env-schema": "^5.0.0", + "fastify-plugin": "^4.0.0" + } + }, "node_modules/@fastify/error": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/@fastify/error/-/error-3.4.0.tgz", @@ -118,9 +131,9 @@ } }, "node_modules/@fastify/swagger": { - "version": "8.10.1", - "resolved": "https://registry.npmjs.org/@fastify/swagger/-/swagger-8.10.1.tgz", - "integrity": "sha512-NZ4PyppZWEd4j8qPt4AKGhuMm7dALe2IntmI2NrdlnPno+rFRyQJHw3XHdziN7yirYGhCGM+vByItWEnPHLu4w==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/@fastify/swagger/-/swagger-8.11.0.tgz", + "integrity": "sha512-K8hAWr3wTBUojjJfNUy0TAl766ga2O/L5mWVbBZ6MImOOB3GYUgqNXC4MKrOejt8Y4Ps4cdKjkgzWLcUtg8SFg==", "dependencies": { "fastify-plugin": "^4.0.0", "json-schema-resolver": "^2.0.0", @@ -130,9 +143,9 @@ } }, "node_modules/@fastify/swagger-ui": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/@fastify/swagger-ui/-/swagger-ui-1.9.3.tgz", - "integrity": "sha512-YYqce4CydjDIEry6Zo4JLjVPe5rjS8iGnk3fHiIQnth9sFSLeyG0U1DCH+IyYmLddNDg1uWJOuErlVqnu/jI3w==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@fastify/swagger-ui/-/swagger-ui-1.10.0.tgz", + "integrity": "sha512-vps8KRDQ8JWQGG9Dh6fCOzAc81Y5a6RnCU/Sumyw3n7+HXJSYExnLLgv4aS9eRFSODtQ8kzZJ//IxAbf0nZ+sQ==", "dependencies": { "@fastify/static": "^6.0.0", "fastify-plugin": "^4.0.0", @@ -1030,6 +1043,25 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, + "node_modules/dotenv-expand": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-9.0.0.tgz", + "integrity": "sha512-uW8Hrhp5ammm9x7kBLR6jDfujgaDarNA02tprvZdyrJ7MpdzD1KyrIHG4l+YoC2fJ2UcdFdNWNWIjt+sexBHJw==", + "engines": { + "node": ">=12" + } + }, "node_modules/electron-fetch": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/electron-fetch/-/electron-fetch-1.9.1.tgz", @@ -1073,6 +1105,16 @@ "node": ">=6" } }, + "node_modules/env-schema": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/env-schema/-/env-schema-5.2.0.tgz", + "integrity": "sha512-36/6cZ+zIbcPA2ANrzp7vTz2bS8/zdZaq2RPFqJVtCGJ4P55EakgJ1BeKP8RMvEmM7ndrnHdJXzL3J1dHrEm1w==", + "dependencies": { + "ajv": "^8.0.0", + "dotenv": "^16.0.0", + "dotenv-expand": "^9.0.0" + } + }, "node_modules/err-code": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz", diff --git a/package.json b/package.json index 7af2f74..23125bf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hub-uploads", - "version": "1.0.0", + "version": "0.1.0", "description": "HUB file uploads services wrapper for web3 up.", "main": "server.js", "scripts": { @@ -11,7 +11,11 @@ "author": "", "license": "ISC", "dependencies": { + "@fastify/env": "^4.2.0", + "@fastify/error": "^3.4.0", "@fastify/multipart": "^8.0.0", + "@fastify/swagger": "^8.11.0", + "@fastify/swagger-ui": "^1.10.0", "@ipld/car": "^5.2.4", "@ucanto/principal": "^8.1.0", "@web3-storage/w3up-client": "^8.0.3", diff --git a/server.js b/server.js index 64cc56c..bbc0122 100644 --- a/server.js +++ b/server.js @@ -1,31 +1,161 @@ import Fastify from "fastify"; -import MultiPart from "@fastify/multipart"; -import { Signer } from "@ucanto/principal/ed25519"; +import fastifyMultiPart from "@fastify/multipart"; +import fastifySwagger from "@fastify/swagger"; +import fastifyEnv from "@fastify/env"; +import fastifySwaggerUi from "@fastify/swagger-ui"; +import { w3 } from "./web3.js"; +import createError from "@fastify/error"; + +const UploadError = createError( + "UPLOAD_ERROR", + "The upload was not successful" +); + +// Initialize Fastify with logging enabled const fastify = Fastify({ logger: true, }); -fastify.register(MultiPart); +// Register environment variables +await fastify.register(fastifyEnv, { + dotenv: true, + schema: { + type: "object", + required: ["WEB3_UP_PROOF", "WEB3_UP_KEY", "WEB3_UP_GATEWAY"], + properties: { + WEB3_UP_PROOF: { + type: "string", + }, + WEB3_UP_KEY: { + type: "string", + }, + WEB3_UP_GATEWAY: { + type: "string", + }, + PORT: { + type: "number", + default: 3000, + }, + }, + }, +}); + +// Register multipart plugin with limits +await fastify.register(fastifyMultiPart, { + limits: { + fields: 0, // Max number of non-file fields + files: 1, // Max number of file fields + }, +}); + +await fastify.register(fastifySwagger, { + exposeRoute: true, + swagger: { + info: { + title: "HUB Upload API", + description: "Upload API for HUB", + version: "0.1.0", + }, + externalDocs: { + url: "https://docs.holaplex.com", + description: "HUB documentation", + }, + host: "localhost:3000", + schemes: ["http"], + consumes: ["application/json"], + produces: ["application/json"], + tags: [], + securityDefinitions: { + apiKey: { + type: "apiKey", + name: "Authorization", + in: "header", + }, + }, + }, +}); +await fastify.register(fastifySwaggerUi, { + routePrefix: "/documentation", + initOAuth: {}, + uiConfig: { + docExpansion: "full", + deepLinking: false, + }, + uiHooks: { + onRequest: function (request, reply, next) { + next(); + }, + preHandler: function (request, reply, next) { + next(); + }, + }, + staticCSP: true, + transformStaticCSP: (header) => header, +}); + +// Initialize uploader with environment variables +const uploader = await w3( + fastify.config.WEB3_UP_KEY, + fastify.config.WEB3_UP_PROOF, + fastify.config.WEB3_UP_GATEWAY +); + +// Health check endpoint fastify.get("/health", async function handler(request, reply) { reply.send({ status: "ok" }); }); -// Declare a route -fastify.post("/uploads", async function handler(request, reply) { - const parts = req.files(); +// Upload endpoint +fastify.post( + "/uploads", + { + schema: { + description: "Upload a file", + tags: ["file"], + consumes: ["multipart/form-data"], + response: { + 200: { + description: "File uploaded successfully", + type: "object", + properties: { + uri: { type: "string" }, + cid: { type: "string" }, + }, + }, + }, + }, + }, + async function handler(request, reply) { + if (request.validationError) { + reply.status(400).send(request.validationError); + return; + } - for await (const part of parts) { - } + try { + // Get file from request and convert to buffer + const data = await request.file(); + const buffer = await data.toBuffer(); + // Upload file and get results + const results = await uploader.uploadFile(buffer); - reply.send(); -}); + // Send results as response + reply.send(results); + } catch { + // Send upload error as response if any error occurs + reply.send(new UploadError()); + } + } +); -// Run the server! +// Start the server try { - await fastify.listen({ port: 3000, host: "0.0.0.0" }); + await fastify.ready(); + fastify.swagger(); + await fastify.listen({ port: fastify.config.PORT, bind: "0.0.0.0" }); } catch (err) { + // Log error and exit process if server fails to start fastify.log.error(err); process.exit(1); } diff --git a/web3.js b/web3.js new file mode 100644 index 0000000..d4eb91a --- /dev/null +++ b/web3.js @@ -0,0 +1,68 @@ +"use strict"; + +import { Signer } from "@ucanto/principal/ed25519"; +import { CarReader } from "@ipld/car"; +import { importDAG } from "@ucanto/core/delegation"; +import * as Client from "@web3-storage/w3up-client"; +import { StoreMemory } from "@web3-storage/access/stores/store-memory"; +import { Blob } from "node:buffer"; + +/** + * Class representing an Uploader. + */ +class Uploader { + /** + * Create an uploader. + * @param {Object} client - The client object. + * @param {string} gateway - The gateway URL. + */ + constructor(client, gateway) { + this.client = client; + this.gateway = gateway; + } + + /** + * Upload a file. + * @param {Buffer} buffer - The file data as a buffer. + * @return {Object} The upload response containing the URI and CID. + * @throws {Error} If an error occurs during upload. + */ + async uploadFile(buffer) { + const blob = new Blob([buffer]); + + const response = await this.client.uploadFile(blob); + const cid = response.toString(); + const uri = `${this.gateway}/${cid}`; + + return { uri, cid }; + } +} + +/** + * Create a new uploader with a given key, proof, and gateway. + * @param {string} key - The key for the uploader. + * @param {string} proof - The proof for the uploader. + * @param {string} gateway - The gateway for the uploader. + * @return {Uploader} A new Uploader instance. + */ +export async function w3(key, proof, gateway) { + const reader = await CarReader.fromBytes(Buffer.from(proof, "base64")); + const principal = Signer.parse(key); + + const store = new StoreMemory(); + const client = await Client.create({ principal, store }); + + const blocks = []; + + for await (const block of reader.blocks()) { + blocks.push(block); + } + + const dag = importDAG(blocks); + + const space = await client.addSpace(dag); + + await client.setCurrentSpace(space.did()); + + return new Uploader(client, gateway); +}