diff --git a/deno.json b/deno.json new file mode 100644 index 00000000..2cd68577 --- /dev/null +++ b/deno.json @@ -0,0 +1,13 @@ +{ + "tasks": { + "dev": "deno run --watch main.ts" + }, + "imports": { + "@std/assert": "jsr:@std/assert@1", + "@internal/": "./src/internal/", + "@storage/": "./src/storage/", + "postgres-migrations/": "npm:postgres-migrations/", + "postgres-migrations/dist/": "npm:postgres-migrations/dist/", + "ajv": "npm:ajv" + } +} \ No newline at end of file diff --git a/src/http/error-handler.ts b/src/http/error-handler.ts index 344390eb..03d7e25a 100644 --- a/src/http/error-handler.ts +++ b/src/http/error-handler.ts @@ -1,6 +1,6 @@ import { FastifyInstance } from 'fastify' import { FastifyError } from '@fastify/error' -import { DatabaseError } from 'pg' +import pg from 'pg' import { ErrorCode, isRenderableError } from '@internal/errors' /** @@ -17,7 +17,7 @@ export const setErrorHandler = (app: FastifyInstance) => { // database error if ( - error instanceof DatabaseError && + error instanceof pg.DatabaseError && [ 'Authentication error', // supavisor specific 'Max client connections reached', @@ -25,7 +25,7 @@ export const setErrorHandler = (app: FastifyInstance) => { 'no more connections allowed', 'sorry, too many clients already', 'server login has been failing, try again later', - ].some((msg) => (error as DatabaseError).message.includes(msg)) + ].some((msg) => (error as pg.DatabaseError).message.includes(msg)) ) { return reply.status(429).send({ statusCode: `429`, diff --git a/src/http/plugins/log-request.ts b/src/http/plugins/log-request.ts index b5693ce9..63512841 100644 --- a/src/http/plugins/log-request.ts +++ b/src/http/plugins/log-request.ts @@ -1,8 +1,7 @@ import fastifyPlugin from 'fastify-plugin' import { logSchema, redactQueryParamFromRequest } from '@internal/monitoring' import { trace } from '@opentelemetry/api' -import { FastifyRequest } from 'fastify/types/request' -import { FastifyReply } from 'fastify/types/reply' +import { FastifyRequest, FastifyReply } from 'fastify' interface RequestLoggerOptions { excludeUrls?: string[] diff --git a/src/http/plugins/tracing.ts b/src/http/plugins/tracing.ts index a3859c73..c74ff765 100644 --- a/src/http/plugins/tracing.ts +++ b/src/http/plugins/tracing.ts @@ -1,5 +1,5 @@ import fastifyPlugin from 'fastify-plugin' -import { isIP } from 'net' +import { isIP } from 'node:net' import { getTenantConfig } from '@internal/database' import { getConfig } from '../../config' diff --git a/src/http/routes/object/getObject.ts b/src/http/routes/object/getObject.ts index 74fc8ba2..8611426c 100644 --- a/src/http/routes/object/getObject.ts +++ b/src/http/routes/object/getObject.ts @@ -1,6 +1,6 @@ import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify' import { FromSchema } from 'json-schema-to-ts' -import { IncomingMessage, Server, ServerResponse } from 'http' +import { IncomingMessage, Server, ServerResponse } from 'node:http' import { getConfig } from '../../../config' import { AuthenticatedRangeRequest } from '../../types' import { ROUTE_OPERATIONS } from '../operations' diff --git a/src/http/routes/object/getObjectInfo.ts b/src/http/routes/object/getObjectInfo.ts index 8ac6e41f..e48ad99e 100644 --- a/src/http/routes/object/getObjectInfo.ts +++ b/src/http/routes/object/getObjectInfo.ts @@ -1,6 +1,6 @@ import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify' import { FromSchema } from 'json-schema-to-ts' -import { IncomingMessage, Server, ServerResponse } from 'http' +import { IncomingMessage, Server, ServerResponse } from 'node:http' import { getConfig } from '../../../config' import { AuthenticatedRangeRequest } from '../../types' import { Obj } from '@storage/schemas' diff --git a/src/http/routes/s3/error-handler.ts b/src/http/routes/s3/error-handler.ts index a6e7fe1f..8f125550 100644 --- a/src/http/routes/s3/error-handler.ts +++ b/src/http/routes/s3/error-handler.ts @@ -1,8 +1,7 @@ import { FastifyError } from '@fastify/error' -import { FastifyRequest } from 'fastify/types/request' -import { FastifyReply } from 'fastify/types/reply' +import { FastifyReply, FastifyRequest } from 'fastify' import { S3ServiceException } from '@aws-sdk/client-s3' -import { DatabaseError } from 'pg' +import pg from 'pg' import { ErrorCode, StorageBackendError } from '@internal/errors' export const s3ErrorHandler = ( @@ -41,14 +40,14 @@ export const s3ErrorHandler = ( // database error if ( - error instanceof DatabaseError && + error instanceof pg.DatabaseError && [ 'Max client connections reached', 'remaining connection slots are reserved for non-replication superuser connections', 'no more connections allowed', 'sorry, too many clients already', 'server login has been failing, try again later', - ].some((msg) => (error as DatabaseError).message.includes(msg)) + ].some((msg) => (error as pg.DatabaseError).message.includes(msg)) ) { return reply.status(429).send({ Error: { diff --git a/src/http/routes/tus/index.ts b/src/http/routes/tus/index.ts index 4e91f009..c79c2b0b 100644 --- a/src/http/routes/tus/index.ts +++ b/src/http/routes/tus/index.ts @@ -1,6 +1,6 @@ import { FastifyBaseLogger, FastifyInstance } from 'fastify' import fastifyPlugin from 'fastify-plugin' -import * as http from 'http' +import * as http from 'node:http' import { ServerOptions, DataStore } from '@tus/server' import { getFileSizeLimit } from '@storage/limits' import { Storage } from '@storage/storage' diff --git a/src/http/routes/tus/lifecycle.ts b/src/http/routes/tus/lifecycle.ts index 5769d995..ac7f0c2c 100644 --- a/src/http/routes/tus/lifecycle.ts +++ b/src/http/routes/tus/lifecycle.ts @@ -1,7 +1,7 @@ -import http from 'http' +import http from 'node:http' import { BaseLogger } from 'pino' import { Upload } from '@tus/server' -import { randomUUID } from 'crypto' +import { randomUUID } from 'node:crypto' import { TenantConnection } from '@internal/database' import { ERRORS, isRenderableError } from '@internal/errors' import { Storage } from '@storage/storage' diff --git a/src/index.ts b/src/index.ts index 0eaee6e8..4729d72e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1 +1 @@ -import './start/server' +import './start/server.ts' diff --git a/src/internal/auth/crypto.ts b/src/internal/auth/crypto.ts index ca826b65..06b18a3a 100644 --- a/src/internal/auth/crypto.ts +++ b/src/internal/auth/crypto.ts @@ -1,21 +1,53 @@ -import AES from 'crypto-js/aes' -import Utf8 from 'crypto-js/enc-utf8' +import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto' import { getConfig } from '../../config' - const { encryptionKey } = getConfig() /** - * Decrypts a text with the configured encryption key via ENCRYPTION_KEY env - * @param ciphertext - */ + * Generate CryptoJs.AES key from passphrase + * https://github.com/brix/crypto-js/issues/468 + * */ +function convertPassphraseToAesKeyBuffer(key: string, salt: Buffer): Buffer { + const password = Buffer.concat([Buffer.from(key, 'binary'), salt]) + const hash: Buffer[] = [] + let digest = password + for (let i = 0; i < 3; i++) { + hash[i] = createHash('md5').update(digest).digest() + digest = Buffer.concat([hash[i]!, password]) + } + return Buffer.concat(hash) +} + +/** + * Replicate CryptoJs.AES.decrypt method + * */ export function decrypt(ciphertext: string): string { - return AES.decrypt(ciphertext, encryptionKey).toString(Utf8) + try { + const cipherBuffer = Buffer.from(ciphertext, 'base64') + const salt = cipherBuffer.subarray(8, 16) + const keyDerivation = convertPassphraseToAesKeyBuffer(encryptionKey, salt) + const [key, iv] = [keyDerivation.subarray(0, 32), keyDerivation.subarray(32)] + const contents = cipherBuffer.subarray(16) + const decipher = createDecipheriv('aes-256-cbc', key, iv) + const decrypted = Buffer.concat([decipher.update(contents), decipher.final()]) + return decrypted.toString('utf8') + } catch (e) { + throw e + } } /** - * Encrypts a text with the configured encryption key via ENCRYPTION_KEY env - * @param plaintext - */ + * Replicate CryptoJs.AES.encrypt method + * */ export function encrypt(plaintext: string): string { - return AES.encrypt(plaintext, encryptionKey).toString() + try { + const salt = randomBytes(8) + const keyDerivation = convertPassphraseToAesKeyBuffer(encryptionKey, salt) + const [key, iv] = [keyDerivation.subarray(0, 32), keyDerivation.subarray(32)] + const cipher = createCipheriv('aes-256-cbc', key, iv) + const contents = Buffer.concat([cipher.update(plaintext), cipher.final()]) + const encrypted = Buffer.concat([Buffer.from('Salted__', 'utf8'), salt, contents]) + return encrypted.toString('base64') + } catch (e) { + throw e + } } diff --git a/src/internal/concurrency/stream.ts b/src/internal/concurrency/stream.ts index d3551736..0200c364 100644 --- a/src/internal/concurrency/stream.ts +++ b/src/internal/concurrency/stream.ts @@ -1,4 +1,4 @@ -import { Transform, TransformCallback } from 'stream' +import { Transform, TransformCallback } from 'node:stream' export const createByteCounterStream = () => { let bytes = 0 diff --git a/src/internal/database/connection.ts b/src/internal/database/connection.ts index ceb2d20e..6b33b3fe 100644 --- a/src/internal/database/connection.ts +++ b/src/internal/database/connection.ts @@ -1,5 +1,6 @@ -import pg, { DatabaseError } from 'pg' -import { Knex, knex } from 'knex' +import pg from 'pg' +import { Knex } from 'knex' +import knex from 'knex' import { JwtPayload } from 'jsonwebtoken' import retry from 'async-retry' import TTLCache from '@isaacs/ttlcache' @@ -163,7 +164,7 @@ export class TenantConnection { return await pool.transaction() } catch (e) { if ( - e instanceof DatabaseError && + e instanceof pg.DatabaseError && e.code === '08P01' && e.message.includes('no more connections allowed') ) { diff --git a/src/internal/database/migrations/migrate.ts b/src/internal/database/migrations/migrate.ts index d5abcf74..7fb2a381 100644 --- a/src/internal/database/migrations/migrate.ts +++ b/src/internal/database/migrations/migrate.ts @@ -1,11 +1,11 @@ -import { Client, ClientConfig } from 'pg' +import pg from 'pg' import SQL from 'sql-template-strings' import { loadMigrationFiles, MigrationError } from 'postgres-migrations' import { getConfig, MultitenantMigrationStrategy } from '../../../config' import { logger, logSchema } from '../../monitoring' -import { BasicPgClient, Migration } from 'postgres-migrations/dist/types' -import { validateMigrationHashes } from 'postgres-migrations/dist/validation' -import { runMigration } from 'postgres-migrations/dist/run-migration' +import type { BasicPgClient, Migration } from 'postgres-migrations/dist/types' +import { validateMigrationHashes } from 'postgres-migrations/dist/validation.js' +import { runMigration } from 'postgres-migrations/dist/run-migration.js' import { searchPath } from '../connection' import { getTenantConfig, listTenantsToMigrate } from '../tenant' import { multitenantKnex } from '../multitenant-db' @@ -179,7 +179,7 @@ export async function runMigrationsOnTenant( tenantId?: string, waitForLock = true ): Promise { - let ssl: ClientConfig['ssl'] | undefined = undefined + let ssl: pg.ClientConfig['ssl'] | undefined = undefined if (databaseSSLRootCert) { ssl = { ca: databaseSSLRootCert } @@ -202,7 +202,7 @@ export async function runMigrationsOnTenant( async function connectAndMigrate(options: { databaseUrl: string | undefined migrationsDirectory: string - ssl?: ClientConfig['ssl'] + ssl?: pg.ClientConfig['ssl'] shouldCreateStorageSchema?: boolean tenantId?: string waitForLock?: boolean @@ -216,14 +216,14 @@ async function connectAndMigrate(options: { waitForLock, } = options - const dbConfig: ClientConfig = { + const dbConfig: pg.ClientConfig = { connectionString: databaseUrl, connectionTimeoutMillis: 60_000, options: `-c search_path=${searchPath}`, ssl, } - const client = new Client(dbConfig) + const client = new pg.Client(dbConfig) client.on('error', (err) => { logSchema.error(logger, 'Error on database connection', { type: 'error', diff --git a/src/internal/queue/database.ts b/src/internal/queue/database.ts index e1172556..7503b5ea 100644 --- a/src/internal/queue/database.ts +++ b/src/internal/queue/database.ts @@ -1,5 +1,5 @@ import { Db } from 'pg-boss' -import EventEmitter from 'events' +import EventEmitter from 'node:events' import pg from 'pg' import { ERRORS } from '@internal/errors' diff --git a/src/start/shutdown.ts b/src/start/shutdown.ts index cd571dfa..0345772d 100644 --- a/src/start/shutdown.ts +++ b/src/start/shutdown.ts @@ -1,7 +1,7 @@ import { logger, logSchema } from '@internal/monitoring' import { AsyncAbortController } from '@internal/concurrency' import { multitenantKnex, TenantConnection } from '@internal/database' -import http from 'http' +import http from 'node:http' /** * Binds shutdown handlers to the process diff --git a/src/storage/backend/adapter.ts b/src/storage/backend/adapter.ts index c29490e6..763f3449 100644 --- a/src/storage/backend/adapter.ts +++ b/src/storage/backend/adapter.ts @@ -1,4 +1,4 @@ -import { Readable } from 'stream' +import { Readable } from 'node:stream' import { getConfig } from '../../config' /** diff --git a/src/storage/backend/file.ts b/src/storage/backend/file.ts index 5e37044b..7364a8d1 100644 --- a/src/storage/backend/file.ts +++ b/src/storage/backend/file.ts @@ -1,9 +1,9 @@ import * as xattr from 'fs-xattr' import fs from 'fs-extra' -import path from 'path' +import path from 'node:path' import fileChecksum from 'md5-file' -import { promisify } from 'util' -import stream from 'stream' +import { promisify } from 'node:util' +import stream from 'node:stream' import MultiStream from 'multistream' import { getConfig } from '../../config' import { @@ -15,7 +15,7 @@ import { UploadPart, } from './adapter' import { ERRORS, StorageBackendError } from '@internal/errors' -import { randomUUID } from 'crypto' +import { randomUUID } from 'node:crypto' import fsExtra from 'fs-extra' const pipeline = promisify(stream.pipeline) diff --git a/src/storage/backend/s3.ts b/src/storage/backend/s3.ts index 681048f7..c5a8c362 100644 --- a/src/storage/backend/s3.ts +++ b/src/storage/backend/s3.ts @@ -583,6 +583,7 @@ export class S3Backend implements StorageBackendAdapter { const params: S3ClientConfig = { region: options.region, runtime: 'node', + cacheMiddleware: true, requestHandler: new NodeHttpHandler({ httpAgent: options.httpAgent?.httpAgent, httpsAgent: options.httpAgent?.httpsAgent, diff --git a/src/storage/protocols/s3/byte-limit-stream.ts b/src/storage/protocols/s3/byte-limit-stream.ts index fce7c98a..912dde53 100644 --- a/src/storage/protocols/s3/byte-limit-stream.ts +++ b/src/storage/protocols/s3/byte-limit-stream.ts @@ -1,4 +1,4 @@ -import { Transform, TransformCallback } from 'stream' +import { Transform, TransformCallback } from 'node:stream' import { ERRORS } from '@internal/errors' export class ByteLimitTransformStream extends Transform { diff --git a/src/storage/protocols/s3/s3-handler.ts b/src/storage/protocols/s3/s3-handler.ts index e3b05e80..6968d33a 100644 --- a/src/storage/protocols/s3/s3-handler.ts +++ b/src/storage/protocols/s3/s3-handler.ts @@ -20,8 +20,8 @@ import { UploadPartCommandInput, UploadPartCopyCommandInput, } from '@aws-sdk/client-s3' -import { PassThrough, Readable } from 'stream' -import stream from 'stream/promises' +import { PassThrough, Readable } from 'node:stream' +import stream from 'node:stream/promises' import { getFileSizeLimit, mustBeValidBucketName, mustBeValidKey } from '../../limits' import { ERRORS } from '@internal/errors' import { S3MultipartUpload, Obj } from '../../schemas' diff --git a/src/storage/protocols/s3/signature-v4.ts b/src/storage/protocols/s3/signature-v4.ts index 944d2817..3c5e90b9 100644 --- a/src/storage/protocols/s3/signature-v4.ts +++ b/src/storage/protocols/s3/signature-v4.ts @@ -1,4 +1,4 @@ -import crypto from 'crypto' +import crypto from 'node:crypto' import { ERRORS } from '@internal/errors' interface SignatureV4Options { diff --git a/src/storage/protocols/tus/als-memory-kv.ts b/src/storage/protocols/tus/als-memory-kv.ts index 1a6b8bfa..7f40459e 100644 --- a/src/storage/protocols/tus/als-memory-kv.ts +++ b/src/storage/protocols/tus/als-memory-kv.ts @@ -1,4 +1,4 @@ -import { AsyncLocalStorage } from 'async_hooks' +import { AsyncLocalStorage } from 'node:async_hooks' import { KvStore } from '@tus/server' import { MetadataValue } from '@tus/s3-store' diff --git a/src/storage/protocols/tus/file-store.ts b/src/storage/protocols/tus/file-store.ts index bd84fb1f..05133329 100644 --- a/src/storage/protocols/tus/file-store.ts +++ b/src/storage/protocols/tus/file-store.ts @@ -1,7 +1,7 @@ import { FileStore as TusFileStore } from '@tus/file-store' import { Upload } from '@tus/server' import fsExtra from 'fs-extra' -import path from 'path' +import path from 'node:path' import { Configstore } from '@tus/file-store' import { FileBackend } from '../../backend' diff --git a/src/storage/protocols/tus/postgres-locker.ts b/src/storage/protocols/tus/postgres-locker.ts index cf4fc025..36f67b37 100644 --- a/src/storage/protocols/tus/postgres-locker.ts +++ b/src/storage/protocols/tus/postgres-locker.ts @@ -1,10 +1,10 @@ import { Lock, Locker, RequestRelease } from '@tus/server' -import { clearTimeout } from 'timers' -import EventEmitter from 'events' -import { Database, DBError } from '../../database' +import { clearTimeout } from 'node:timers' +import EventEmitter from 'node:events' +import { Database } from '../../database' import { PubSubAdapter } from '@internal/pubsub' import { UploadId } from './upload-id' -import { ERRORS } from '@internal/errors' +import { ErrorCode, ERRORS, StorageBackendError } from '@internal/errors' const REQUEST_LOCK_RELEASE_MESSAGE = 'REQUEST_LOCK_RELEASE' @@ -94,7 +94,7 @@ export class PgLock implements Lock { await db.mustLockObject(uploadId.bucket, uploadId.objectName, uploadId.version) return true } catch (e) { - if (e instanceof DBError && e.message === 'resource_locked') { + if (e instanceof StorageBackendError && e.code === ErrorCode.ResourceLocked) { await this.notifier.release(id) await new Promise((resolve) => { setTimeout(resolve, 500) diff --git a/src/storage/renderer/image.ts b/src/storage/renderer/image.ts index 40de8f28..70b6eb98 100644 --- a/src/storage/renderer/image.ts +++ b/src/storage/renderer/image.ts @@ -5,7 +5,7 @@ import { FastifyRequest } from 'fastify' import { Renderer, RenderOptions } from './renderer' import axiosRetry from 'axios-retry' import { ERRORS } from '@internal/errors' -import { Stream } from 'stream' +import { Stream } from 'node:stream' import Agent from 'agentkeepalive' /** diff --git a/src/storage/renderer/renderer.ts b/src/storage/renderer/renderer.ts index 2e43d44f..812b59bc 100644 --- a/src/storage/renderer/renderer.ts +++ b/src/storage/renderer/renderer.ts @@ -1,6 +1,6 @@ import { FastifyReply, FastifyRequest } from 'fastify' import { ObjectMetadata } from '../backend' -import { Readable } from 'stream' +import { Readable } from 'node:stream' import { getConfig } from '../../config' import { Obj } from '../schemas' diff --git a/src/storage/uploader.ts b/src/storage/uploader.ts index aff4ea4e..66d72008 100644 --- a/src/storage/uploader.ts +++ b/src/storage/uploader.ts @@ -1,4 +1,4 @@ -import { randomUUID } from 'crypto' +import { randomUUID } from 'node:crypto' import { FastifyRequest } from 'fastify' import { ERRORS } from '@internal/errors'