diff --git a/Documentation/docs/plugins.md b/Documentation/docs/plugins.md index ba83a027b..78754b12a 100644 --- a/Documentation/docs/plugins.md +++ b/Documentation/docs/plugins.md @@ -43,7 +43,7 @@ Links in top right drop down application grid can be configured in the `applicat Plugins are defined as entries under the `applications.plugins` array, with at least the following properties: -- `id` is mandatory and must be unique to each plugin +- `id` is mandatory and must be unique to each plugin. It must be rather short, and alphanumeric with `_`s only. - `internal_domain` is the internal domain of the plugin's server, it must be the same as the one in the docker-compose.yml file for ex. or resolvable and accessible to the Twake Drive backend server. - `external_prefix` is the external URL prefix of the plugin exposed by the backend and proxied to the `internal_domain`. - `api.private_key` is the shared secret used by the plugin's server to authentify to the backend @@ -133,6 +133,119 @@ When a user requests a preview and then possibly to edit the file, an IFrame is In the case of the OnlyOffice application, these URLs are pointing to the connector plugin, which then proxies back and forth with the OnlyOffice document server. +#### Editing session key + +When editing a file, at first, a new session key is created on the file by the +backend and atomically swapped. + +This session key uniquely identifies the user, company, and plugin and it is not encrypted. + +An editing session key prevents others from starting, can be updated with new versions, and +ended with and without a new version. + +See operations `beginEditing` and `updateEditing` in `tdrive/backend/node/src/services/documents/services/index.ts` for operations available on that key. + +Basic flow is: + - Call `beginEditing` + - If a key previously was set for that file, uses the `/check` endpoint of the plugin to update the key status first + - Twake Drive generates an editing session key and returns it (or returns the existing one). + - Optionally call `updateEditing?keepEditing=true` with a file stream to generate intermediary versions + - End the session by calling `updateEditing` (without `keepEditing=true`). If a file is provided, a new (and final) FileVersion + is created with that content. Otherwise, the editing session is cleared without creating a new version. + + +#### API to expose by the application + +Authentication from the Twake Drive backend is a JWT with the property `type` being +`"tdriveToApplication"` and signed with the shared secret. The application must accept +multiple parallel requests for the same key gracefully. + +- `POST /tdriveApi/1/session/${editing_session_key}/check` + + Sent by Twake Drive backend when beginning a new editing session or investigating + stored keys. The application is expected to process the key if possible before responding, + and provide a response in JSON. The `error` key of that body should be truthy if the response + is not known. Otherwise it should respond: + + - `{ status: 'unknown' }`: the key isn't known and maybe used for a new session + - `{ status: 'updated' }`: the key needed updating but is now invalid + - `{ status: 'expired' }`: the key was already used in a finished session and can't be used again + - `{ status: 'live' }`: the key is valid and current and should be used again for the same file (if multiple writers are allowed) + +#### Example flow of editing session + +```mermaid +sequenceDiagram + autonumber + actor User as User/Browser + participant Front as Frontend + box Server side + participant Back as Backend + participant Conn as onlyoffice
connector + participant OO as Only Office
Edition Server + end + + User->>Front: Open in
editor + Front->>Back: Get plugin config + Front->>User: Open plugin config
editor.edition_url + + User->>Conn: Open editor_url
(proxied by backend) + Conn->>Back: beginEditing + alt existing key + Back->>Conn: checkSessionStatus + Conn->>OO: getForgotten + note over Conn, OO: recover forgotten
process, and new key + Conn->>OO: info command + note right of Conn: decide status of key
live or stale + note over Conn, OO: detect ended but
not changed keys + note over Conn, OO: normal callback processing with
update if required + OO->>Conn: callback with key status + Conn->>Back: key status
(live/expired/updated/etc) + end + activate Back + Back->>Conn: editing_session_key + Conn->>User: HTML host for Editor with
special callback URL + User->>OO: Load JS Editor directly from OO server + activate User + loop User editing + User->>User: Furious Document
Editing + User-->>OO: Periodic saving + end + deactivate User + note left of User: Closes editor + note over User,OO: 10 seconds after last user closes their editor + OO->>Conn: callback to save the new version
or close without changes + Conn->>Back: updateEditing?keepEditing=false
with URL to new version from OO + deactivate Back +``` + +#### Batch processing of unknown keys + +Periodically, the plugin, and twake drive, should run batch cleanup operations on editing session keys +to ensure they are live, or removed, as they may block other operations until then. + +Here is an example initiated by the plugin: + +```mermaid +sequenceDiagram + autonumber + actor User as User/Browser + participant Front as Frontend + box Server side + participant Back as Backend + participant Conn as onlyoffice
connector + participant OO as Only Office
Edition Server + end + + alt Periodic scheduled task + Conn->>OO: getForgottenList + loop Each forgotten file + Conn->>Back: Save new version
end editing session + Conn->>OO: Delete forgotten file + end + end +``` + ### Example: OnlyOffice plugin The [OnlyOffice connector plugin](https://github.com/linagora/twake-drive/tree/main/tdrive/connectors/onlyoffice-connector) is an example of plugin. It's readme includes an example configuration for the backend. diff --git a/tdrive/backend/node/src/cli/cmds/editing_session.ts b/tdrive/backend/node/src/cli/cmds/editing_session.ts new file mode 100644 index 000000000..7e79fd4f8 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/editing_session.ts @@ -0,0 +1,16 @@ +import { CommandModule } from "yargs"; + +const command: CommandModule = { + describe: "Editing sessions tools", + command: "editing_session", + builder: yargs => + yargs.commandDir("editing_session_cmds", { + visit: commandModule => commandModule.default, + }), + // eslint-disable-next-line @typescript-eslint/no-empty-function + handler: () => { + throw new Error("Missing sub-command"); + }, +}; + +export default command; diff --git a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts new file mode 100644 index 000000000..97e2d68ed --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts @@ -0,0 +1,166 @@ +import yargs from "yargs"; + +import runWithPlatform from "../../lib/run-with-platform"; +import type { TdrivePlatform } from "../../../core/platform/platform"; +import type { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; +import { + DriveFile, + EditingSessionKeyFormat, + TYPE as DriveFile_TYPE, +} from "../../../services/documents/entities/drive-file"; +import { + FileVersion, + TYPE as FileVersion_TYPE, +} from "../../../services/documents/entities/file-version"; +import User, { TYPE as User_TYPE } from "../../../services/user/entities/user"; +import { FindOptions } from "../../../core/platform/services/database/services/orm/repository/repository"; + +async function makeUserCache(platform: TdrivePlatform) { + const usersRepo = await platform + .getProvider("database") + .getRepository(User_TYPE, User); + const cache: { [id: string]: User } = {}; + return async (id): Promise<[boolean, User]> => { + if (id in cache) return [true, cache[id]]; + const user = await usersRepo.findOne({ id }); + if (user) cache[id] = user; + return [false, user]; + }; +} + +interface ListArguments { + all: boolean; + name: string; +} + +async function report(platform: TdrivePlatform, args: ListArguments) { + const users = await makeUserCache(platform); + async function formatUser(id) { + const [wasKnown, user] = await users(id); + return user?.email_canonical + ? user.email_canonical + (wasKnown ? "" : ` (${id})`) + : `${JSON.stringify(id)} (user id not found)`; + } + const drivesRepo = await platform + .getProvider("database") + .getRepository(DriveFile_TYPE, DriveFile); + const versionsRepo = await platform + .getProvider("database") + .getRepository(FileVersion_TYPE, FileVersion); + const filter = { + is_in_trash: false, + }; + if (!args.all) filter["editing_session_key"] = { $ne: null }; + const opts: FindOptions = { sort: { name: "asc" } }; + if (args.name) filter["name"] = args.name; + const editedFiles = (await drivesRepo.find(filter, opts)).getEntities(); + const formatDate = (date: Date) => date.toISOString(); + const formatTS = (ts: number) => formatDate(new Date(ts)); + console.error(`DriveFiles${args.all ? "" : " with non-null editing_session_key"}:`); + console.error(""); + for (const dfile of editedFiles) { + console.error(`- ${dfile.name} (${dfile.id}) of ${await formatUser(dfile.creator)}`); + if (dfile.scope !== "personal") console.error(` - scope: ${dfile.scope}`); + if (dfile.is_directory) console.error(" - directory !"); + if (dfile.is_in_trash) console.error(" - in trash !"); + console.error(` - modified: ${formatTS(dfile.last_modified)}`); + if (dfile.editing_session_key) { + const parsed = EditingSessionKeyFormat.parse(dfile.editing_session_key); + console.error(" - editing_session_key:"); + console.error(` - URL encoded: ${encodeURIComponent(dfile.editing_session_key)}`); + console.error(` - applicationId: ${parsed.applicationId}`); + console.error(` - companyId: ${parsed.companyId}`); + console.error(` - instanceId: ${JSON.stringify(parsed.instanceId)}`); + console.error( + ` - userId: ${await formatUser(parsed.userId)} (${ + parsed.userId === dfile.creator ? "same as creator ID" : "not the creator" + })`, + ); + console.error( + ` - timestamp: ${formatDate(parsed.timestamp)} (${Math.floor( + (new Date().getTime() - parsed.timestamp.getTime()) / 1000, + )}s ago)`, + ); + } + + const versions = ( + await versionsRepo.find({ drive_item_id: dfile.id }, { sort: { date_added: "asc" } }) + ).getEntities(); + let previousSize = 0; + let lastVersion: FileVersion; + console.error(" - Versions:"); + for (const version of versions) { + console.error( + ` - ${formatTS(version.date_added)} by ${await formatUser(version.creator_id)}`, + ); + console.error(` - id: ${version.id}`); + console.error( + ` - size: ${version.file_metadata.size} (${ + version.file_metadata.size > previousSize ? "+" : "" + }${version.file_metadata.size - previousSize})`, + ); + previousSize = version.file_metadata.size; + lastVersion = version; + console.error(` - application: ${JSON.stringify(version.application_id)}`); + } + if (previousSize != dfile.size) + console.error( + ` - mismatched sizes: DriveFile.size is ${dfile.size} but last Version.file_metadata is ${previousSize}`, + ); + if (lastVersion) { + const lastTimestamp = lastVersion.date_added; + if (lastTimestamp != dfile.last_modified) + console.error( + ` - mismatched FileVersion.date_added (${formatTS( + lastTimestamp, + )}) != DriveFile.last_modified (${formatTS(dfile.last_modified)}) - delta: ${ + (lastTimestamp - dfile.last_modified) / 1000 + }s`, + ); + if (lastTimestamp != dfile.last_version_cache.date_added) + console.error( + ` - mismatched FileVersion.date_added (${formatTS( + lastTimestamp, + )}) != DriveFile.last_version_cache.date_added (${formatTS( + dfile.last_version_cache.date_added, + )}) - delta: ${(lastTimestamp - dfile.last_version_cache.date_added) / 1000}s`, + ); + if (lastVersion.file_size != dfile.size) + console.error( + ` - mismatched FileVersion.file_size (${lastVersion.file_size}) != DriveFile.dfile.size (${dfile.size})`, + ); + } + } + if (!editedFiles.length) console.error(" (no matching DriveFiles)"); +} + +const command: yargs.CommandModule = { + command: "list", + describe: ` + List current DriveFile items that have an editing_session_key set + `.trim(), + + builder: { + all: { + type: "boolean", + alias: "a", + describe: "Include all DriveFiles (not just the ones with editing_session_keys)", + default: false, + }, + name: { + type: "string", + alias: "n", + describe: "Filter DriveFiles by name (must be exact)", + default: false, + }, + }, + handler: async argv => { + const args = argv as unknown as ListArguments; + await runWithPlatform("editing_session list", async ({ spinner: _spinner, platform }) => { + console.error("\n"); + await report(platform, args); + console.error("\n"); + }); + }, +}; +export default command; diff --git a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/parse.ts b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/parse.ts new file mode 100644 index 000000000..5329a13a3 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/parse.ts @@ -0,0 +1,34 @@ +import yargs from "yargs"; + +import { NonPlatformCommandYargsBuilder } from "../../utils/non-plaform-command-yargs-builder"; +import { EditingSessionKeyFormat } from "../../../services/documents/entities/drive-file"; + +interface ParseArguments { + editing_session_key: string; +} + +const command: yargs.CommandModule = { + command: "parse ", + describe: ` + Parse the provided editing_session_key and output json data (to stderr) + `.trim(), + + builder: { + ...NonPlatformCommandYargsBuilder, + }, + handler: async argv => { + const args = argv as unknown as ParseArguments; + const parsed = EditingSessionKeyFormat.parse(decodeURIComponent("" + args.editing_session_key)); + console.error( + JSON.stringify( + { + ageH: (new Date().getTime() - parsed.timestamp.getTime()) / (60 * 60 * 1000), + ...parsed, + }, + null, + 2, + ), + ); + }, +}; +export default command; diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts index 7d13f0953..a223035ee 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts @@ -47,9 +47,18 @@ export class PostgresQueryBuilder { values.push(...inClause); } } else { - const value = `${this.dataTransformer.toDbString(filter, columnsDefinition[key].type)}`; - whereClause += `${key} = $${idx++} AND `; - values.push(value); + const isANotEqualFilter = filter && Object.keys(filter).join("!") === "$ne"; + if (filter === null || (isANotEqualFilter && filter["$ne"] === null)) { + whereClause += `${key} IS${filter === null ? "" : " NOT"} NULL`; + } else { + const filterValue = isANotEqualFilter ? filter["$ne"] : filter; + const value = `${this.dataTransformer.toDbString( + filterValue, + columnsDefinition[key].type, + )}`; + whereClause += `${key} ${isANotEqualFilter ? "!=" : "="} $${idx++} AND `; + values.push(value); + } } }); } diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts index 250385834..e79df829c 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts @@ -43,6 +43,11 @@ export type FindOptions = { sort?: SortOption; }; +export type AtomicCompareAndSetResult = { + didSet: boolean; + currentValue: FieldValueType | null; +}; + /** * Repository to work with entities. Each entity type has its own repository instance. */ @@ -129,10 +134,7 @@ export default class Repository { fieldName: keyof EntityType, previousValue: FieldValueType | null, newValue: FieldValueType | null, - ): Promise<{ - didSet: boolean; - currentValue: FieldValueType | null; - }> { + ): Promise> { if (previousValue === newValue) throw new Error(`Previous and new values are identical: ${JSON.stringify(previousValue)}`); return this.connector.atomicCompareAndSet(entity, fieldName, previousValue, newValue); diff --git a/tdrive/backend/node/src/services/applications-api/index.ts b/tdrive/backend/node/src/services/applications-api/index.ts index 411b9230b..5853e7c8e 100644 --- a/tdrive/backend/node/src/services/applications-api/index.ts +++ b/tdrive/backend/node/src/services/applications-api/index.ts @@ -6,12 +6,30 @@ import WebServerAPI from "../../core/platform/services/webserver/provider"; import Application from "../applications/entities/application"; import web from "./web/index"; import { logger } from "../../core/platform/framework/logger"; +import { EditingSessionKeyFormat } from "../documents/entities/drive-file"; +import jwt from "jsonwebtoken"; + +export enum ApplicationEditingKeyStatus { + /** the key isn't known and maybe used for a new session */ + unknown = "unknown", + /** the key needed updating but is now invalid */ + updated = "updated", + /** the key was already used in a finished session and can't be used again */ + expired = "expired", + /** the key is valid and current and should be used again for the same file */ + live = "live", +} @Prefix("/api") export default class ApplicationsApiService extends TdriveService { version = "1"; name = "applicationsapi"; + private static default: ApplicationsApiService; + public static getDefault() { + return this.default; + } + public async doInit(): Promise { const fastify = this.context.getProvider("webserver").getServer(); fastify.register((instance, _opts, next) => { @@ -68,10 +86,95 @@ export default class ApplicationsApiService extends TdriveService { } } } - + ApplicationsApiService.default = this; return this; } + /** Get the configuration of a given `appId` or `undefined` if unknown */ + public getApplicationConfig(appId: string) { + const apps = config.get("applications.plugins") || []; + return apps.find(app => app.id === appId); + } + + /** Get the configuration of a given `appId` or throw an error if unknown */ + public requireApplicationConfig(appId: string) { + const app = this.getApplicationConfig(appId); + if (!app) throw new Error(`Unknown application.id ${JSON.stringify(appId)}`); + return app; + } + + /** Send a request to the plugin by its application id + * @param url Full URL that doesn't start with a `/` + */ + private async requestFromApplication( + method: "GET" | "POST" | "DELETE", + url: string, + appId: string, + data?: unknown, + ) { + const app = this.requireApplicationConfig(appId); + if (!app.internal_domain) + throw new Error(`application.id ${JSON.stringify(appId)} missing an internal_domain`); + const signature = jwt.sign( + { + ts: new Date().getTime(), + type: "tdriveToApplication", + application_id: appId, + }, + app.api.private_key, + ); + const domain = app.internal_domain.replace(/(\/$|^\/)/gm, ""); + const finalURL = `${domain}/${url}${ + url.indexOf("?") > -1 ? "&" : "?" + }token=${encodeURIComponent(signature)}`; + return axios.request({ + url: finalURL, + method: method, + data, + headers: { + Authorization: signature, + }, + maxRedirects: 0, + }); + } + + /** + * Check status of `editing_session_key` in the corresponding application. + * @param editingSessionKey {@see DriveFile.editing_session_key} to check + * @returns status of the provided key as far as the application knows + */ + async checkPendingEditingStatus(editingSessionKey: string): Promise { + const parsedKey = EditingSessionKeyFormat.parse(editingSessionKey); + const response = await this.requestFromApplication( + "POST", + "tdriveApi/1/session/" + encodeURIComponent(editingSessionKey) + "/check", + parsedKey.applicationId, + ); + if (response.status != 200 || response.data.error) + throw new Error( + `Application check key ${editingSessionKey} failed with HTTP ${ + response.status + }: ${JSON.stringify(response.data)}`, + ); + return (response.data.status as ApplicationEditingKeyStatus) || null; + } + + /** + * Change the filename in the external editing session + * @param editingSessionKey {@see DriveFile.editing_session_key} to change + * @param filename The new filename + */ + async renameEditingKeyFilename(editingSessionKey: string, filename: string): Promise { + const parsedKey = EditingSessionKeyFormat.parse(editingSessionKey); + const response = await this.requestFromApplication( + "POST", + `tdriveApi/1/session/${encodeURIComponent(editingSessionKey)}/title`, + parsedKey.applicationId, + { title: filename }, + ); + return !!response.data.done as boolean; + } + // TODO: remove api(): undefined { return undefined; diff --git a/tdrive/backend/node/src/services/documents/entities/drive-file.ts b/tdrive/backend/node/src/services/documents/entities/drive-file.ts index e3667a7d4..5a793abc6 100644 --- a/tdrive/backend/node/src/services/documents/entities/drive-file.ts +++ b/tdrive/backend/node/src/services/documents/entities/drive-file.ts @@ -1,8 +1,10 @@ import { Type } from "class-transformer"; +import { randomUUID } from "crypto"; import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators"; import { DriveFileAccessLevel, publicAccessLevel } from "../types"; import { FileVersion } from "./file-version"; import search from "./drive-file.search"; +import * as UUIDTools from "../../../utils/uuid"; export const TYPE = "drive_files"; export type DriveScope = "personal" | "shared"; @@ -93,8 +95,10 @@ export class DriveFile { /** * If this field is non-null, then an editing session is in progress (probably in OnlyOffice). - * Should be in the format `timestamp-appid-hexuuid` where `appid` and `timestamp` have no `-` - * characters. + * Use {@see EditingSessionKeyFormat} to generate and interpret it. + * Values should ensure that sorting lexicographically is chronological (assuming perfect clocks everywhere), + * and that the application, company and user that started the edit session are retrievable. + * It is not encrypted. */ @Type(() => String) @Column("editing_session_key", "string") @@ -120,6 +124,105 @@ export class DriveFile { scope: DriveScope; } +const OnlyOfficeSafeDocKeyBase64 = { + // base64 uses `+/`, but base64url uses `-_` instead. Both use `=` as padding, + // which conflicts with EditingSessionKeyFormat so using `.` instead. + fromBuffer(buffer: Buffer) { + return buffer.toString("base64url").replace(/=/g, "."); + }, + toBuffer(base64: string) { + return Buffer.from(base64.replace(/\./g, "="), "base64url"); + }, +}; + +function checkFieldValue(field: string, value: string, required: boolean = true) { + if (!required && !value) return; + if (!/^[0-9a-zA-Z_-]+$/m.test(value)) + throw new Error( + `Invalid ${field} value (${JSON.stringify( + value, + )}). Must be short and only alpha numeric or '_' and '-'`, + ); +} +/** + * Reference implementation for generating then parsing the {@link DriveFile.editing_session_key} field. + * + * Fields should be explicit, `instanceId` is for the case when we have multiple + * clients + */ +export const EditingSessionKeyFormat = { + // OnlyOffice key limits: 128 chars, [0-9a-zA-Z.=_-] + // See https://api.onlyoffice.com/editors/config/document#key + // This is specific to it, but the constraint seems strict enough + // that any other system needing such a unique identifier would find + // this compatible. This value must be ensured to be the strictest + // common denominator to all plugin/interop systems. Plugins that + // require something even stricter have the option of maintaining + // a look up table to an acceptable value. + generate( + applicationId: string, + instanceId: string, + companyId: string, + userId: string, + overrideTimeStamp?: Date, + ) { + checkFieldValue("applicationId", applicationId); + checkFieldValue("instanceId", instanceId, false); + const isoUTCDateNoSpecialCharsNoMS = (overrideTimeStamp ?? new Date()) + .toISOString() + .replace(/\..+$/, "") + .replace(/[ZT:-]/g, ""); + const userIdBuffer = UUIDTools.bufferFromUUIDString(userId) as unknown as Uint8Array; + const companyIdBuffer = UUIDTools.bufferFromUUIDString(companyId) as unknown as Uint8Array; + const entropyBuffer = UUIDTools.bufferFromUUIDString(randomUUID()) as unknown as Uint8Array; + const idsString = OnlyOfficeSafeDocKeyBase64.fromBuffer( + Buffer.concat([companyIdBuffer, userIdBuffer, entropyBuffer]), + ); + const newKey = [isoUTCDateNoSpecialCharsNoMS, applicationId, instanceId, idsString].join("="); + if (newKey.length > 128 || !/^[0-9a-zA-Z=_-]+$/m.test(newKey)) + throw new Error( + `Invalid generated editingSessionKey (${JSON.stringify( + newKey, + )}) string. Must be <128 chars, and only contain [0-9a-zA-z=_-]`, + ); + return newKey; + }, + + parse(editingSessionKey: string) { + const parts = editingSessionKey.split("="); + const expectedParts = 4; + if (parts.length !== expectedParts) + throw new Error( + `Invalid editingSessionKey (${JSON.stringify( + editingSessionKey, + )}). Expected ${expectedParts} parts`, + ); + const [timestampStr, applicationId, instanceId, idsOOBase64String] = parts; + const timestampMatch = timestampStr.match( + /^(?\d{4})(?\d\d)(?\d\d)(?\d\d)(?\d\d)(?\d\d)$/, + ); + if (!timestampMatch) + throw new Error( + `Invalid editingSessionKey (${JSON.stringify( + editingSessionKey, + )}). Didn't start with valid timestamp`, + ); + const { year, month, day, hour, minute, second } = timestampMatch.groups!; + const idsBuffer = OnlyOfficeSafeDocKeyBase64.toBuffer(idsOOBase64String); + const companyId = UUIDTools.formattedUUIDInBufferArray(idsBuffer, 0); + const userId = UUIDTools.formattedUUIDInBufferArray(idsBuffer, 1); + return { + timestamp: new Date( + Date.parse(`${[year, month, day].join("-")}T${[hour, minute, second].join(":")}Z`), + ), + applicationId, + instanceId, + companyId, + userId, + }; + }, +}; + export type AccessInformation = { public?: { token: string; diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index e6779ab4a..278816285 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -6,6 +6,7 @@ import { Pagination, } from "../../../core/platform/framework/api/crud-service"; import Repository, { + AtomicCompareAndSetResult, comparisonType, inType, } from "../../../core/platform/services/database/services/orm/repository/repository"; @@ -13,7 +14,7 @@ import { PublicFile } from "../../../services/files/entities/file"; import globalResolver from "../../../services/global-resolver"; import { hasCompanyAdminLevel } from "../../../utils/company"; import gr from "../../global-resolver"; -import { DriveFile, TYPE } from "../entities/drive-file"; +import { DriveFile, EditingSessionKeyFormat, TYPE } from "../entities/drive-file"; import { FileVersion, TYPE as FileVersionType } from "../entities/file-version"; import User, { TYPE as UserType } from "../../user/entities/user"; @@ -58,8 +59,10 @@ import { import archiver from "archiver"; import internal from "stream"; import config from "config"; -import { randomUUID } from "crypto"; +import { MultipartFile } from "@fastify/multipart"; +import { UploadOptions } from "src/services/files/types"; import { SortType } from "src/core/platform/services/search/api"; +import ApplicationsApiService, { ApplicationEditingKeyStatus } from "../../applications-api"; export class DocumentsService { version: "1"; @@ -546,7 +549,7 @@ export class DocumentsService { } const updatable = ["access_info", "name", "tags", "parent_id", "description", "is_in_trash"]; - + let renamedTo: string | undefined; for (const key of updatable) { if ((content as any)[key]) { if ( @@ -587,7 +590,7 @@ export class DocumentsService { } }); } else if (key === "name") { - item.name = await getItemName( + renamedTo = item.name = await getItemName( content.parent_id || item.parent_id, item.id, content.name, @@ -618,7 +621,17 @@ export class DocumentsService { await updateItemSize(oldParent, this.repository, context); } - + if (renamedTo && item.editing_session_key) + ApplicationsApiService.getDefault() + .renameEditingKeyFilename(item.editing_session_key, renamedTo) + .catch(err => { + logger.error("Error rename editing session to new name", { + err, + editing_session_key: item.editing_session_key, + renamedTo, + }); + /* Ignore errors, just throw it out there... */ + }); if (item.parent_id === this.TRASH) { //When moving to trash we recompute the access level to make them flat item.access_info = await makeStandaloneAccessLevel( @@ -915,6 +928,7 @@ export class DocumentsService { item.last_version_cache = driveItemVersion; item.size = driveItemVersion.file_size; + item.last_modified = driveItemVersion.date_added; await this.repository.save(item); @@ -958,6 +972,9 @@ export class DocumentsService { * with only that key provided. * @param id DriveFile ID of the document to begin editing * @param editorApplicationId Editor/Application/Plugin specific identifier + * @param appInstanceId For that `editorApplicationId` a unique identifier + * when multiple instances are running. Unused today - would need a mapping + * from `appInstanceId` to server host. * @param context * @returns An object in the format `{}` with the unique identifier for the * editing session @@ -965,34 +982,62 @@ export class DocumentsService { beginEditing = async ( id: string, editorApplicationId: string, + appInstanceId: string, context: DriveExecutionContext, ) => { - const isoUTCDateNoSpecialCharsNoMS = new Date() - .toISOString() - .replace(/\..+$/, "") - .replace(/[ZT:-]/g, ""); - const newKey = [ - isoUTCDateNoSpecialCharsNoMS, - editorApplicationId, - randomUUID().replace(/-+/g, ""), - ].join("-"); - // OnlyOffice key limits: 128 chars, [0-9a-zA-z=_-] - // This is specific to it, but the constraint seems strict enough - // that any other system needing such a unique identifier would find - // this compatible. This value must be ensured to be the strictest - // common denominator to all plugin/interop systems. Plugins that - // require something even stricter have the option of maintaining - // a look up table to an acceptable value. - if (newKey.length > 128 || !/^[0-9a-zA-Z=_]+$/m.test(editorApplicationId)) - CrudException.throwMe( - new Error('Invalid "editorApplicationId" string. Must be short and only alpha numeric'), - new CrudException("Invalid editorApplicationId", 400), - ); - if (!context) { this.logger.error("invalid execution context"); return null; } + if ( + !editorApplicationId || + !ApplicationsApiService.getDefault().getApplicationConfig(editorApplicationId) + ) { + logger.error(`Missing or invalid application ID: ${JSON.stringify(editorApplicationId)}`); + CrudException.throwMe( + new Error("Unknown appId"), + new CrudException("Missing or invalid application ID", 400), + ); + } + const spinLoopUntilEditable = async ( + provider: { + generateKey: () => string; + atomicSet: ( + key: string | null, + previous: string | null, + ) => Promise>; + getPluginKeyStatus: (key: string) => Promise; + }, + attemptCount = 8, + tarpitS = 1, + tarpitWorsenCoeff = 1.2, + ) => { + while (attemptCount-- > 0) { + const newKey = provider.generateKey(); + const swapResult = await provider.atomicSet(newKey, null); + logger.debug(`Begin edit try ${newKey}, got: ${JSON.stringify(swapResult)}`); + if (swapResult.didSet) return newKey; + if (!swapResult.currentValue) continue; // glitch in the matrix but ok because atomicCompareAndSet is not actually completely atomic + const existingStatus = await provider.getPluginKeyStatus(swapResult.currentValue); + logger.debug(`Begin edit get status of ${newKey}: ${JSON.stringify(existingStatus)}`); + switch (existingStatus) { + case ApplicationEditingKeyStatus.unknown: + case ApplicationEditingKeyStatus.live: + return swapResult.currentValue; + case ApplicationEditingKeyStatus.updated: + case ApplicationEditingKeyStatus.expired: + logger.debug(`Begin edit emptying previous ${swapResult.currentValue}`); + await provider.atomicSet(null, swapResult.currentValue); + break; + default: + throw new Error( + `Unexpected ApplicationEditingKeyStatus: ${JSON.stringify(existingStatus)}`, + ); + } + await new Promise(resolve => setTimeout(resolve, tarpitS * 1000)); + tarpitS *= tarpitWorsenCoeff; + } + }; const hasAccess = await checkAccess(id, null, "write", this.repository, context); if (!hasAccess) { @@ -1012,18 +1057,139 @@ export class DocumentsService { {}, context, ); - const result = await this.repository.atomicCompareAndSet( - driveFile, - "editing_session_key", - null, - newKey, - ); - return { editingSessionKey: result.currentValue }; + const editingSessionKey = await spinLoopUntilEditable({ + atomicSet: (key, previous) => + this.repository.atomicCompareAndSet(driveFile, "editing_session_key", previous, key), + generateKey: () => + EditingSessionKeyFormat.generate( + editorApplicationId, + appInstanceId, + context.company.id, + context.user.id, + ), + getPluginKeyStatus: key => + ApplicationsApiService.getDefault().checkPendingEditingStatus(key), + }); + return { editingSessionKey }; } catch (error) { logger.error({ error: `${error}` }, "Failed to begin editing Drive item"); CrudException.throwMe(error, new CrudException("Failed to begin editing Drive item", 500)); } }; + + /** + * End editing session either by providing a URL to a new file to create a version, + * or not, to just cancel the session. + * @param editing_session_key Editing key of the DriveFile + * @param file Multipart files from the incoming http request + * @param options Optional upload information from the request + * @param keepEditing If `true`, the file will be saved as a new version, + * and the DriveFile will keep its editing_session_key. If `true`, a file is required. + * @param userId When authentified by the root token of an application, this user + * will override the creator of this version + * @param context + */ + updateEditing = async ( + editing_session_key: string, + file: MultipartFile, + options: UploadOptions, + keepEditing: boolean, + userId: string | null, + context: CompanyExecutionContext, + ) => { + //TODO rethink the locking stuff shouldn't be just forgotten + //TODO Make this accept even if missing and act ok about it, + // store to dump folder or such + if (!context) { + this.logger.error("invalid execution context"); + return null; + } + if (!editing_session_key) { + this.logger.error("Invalid editing_session_key: " + JSON.stringify(editing_session_key)); + throw new CrudException("Invalid editing_session_key", 400); + } + try { + const parsedKey = EditingSessionKeyFormat.parse(editing_session_key); + context = { + ...context, + company: { id: parsedKey.companyId }, + }; + + if (context.user.id === context.user.application_id && context.user.application_id) { + context = { + ...context, + user: { + ...context.user, + id: userId || parsedKey.userId, + }, + }; + } + } catch (e) { + this.logger.error( + "Invalid editing_session_key value: " + JSON.stringify(editing_session_key), + e, + ); + throw new CrudException("Invalid editing_session_key", 400); + } + + const driveFile = await this.repository.findOne({ editing_session_key }, {}, context); + if (!driveFile) { + this.logger.error("Drive item not found by editing session key"); + throw new CrudException("Item not found by editing session key", 404); + } + + const hasAccess = await checkAccess(driveFile.id, driveFile, "write", this.repository, context); + if (!hasAccess) { + logger.error("user does not have access drive item " + driveFile.id); + CrudException.throwMe( + new Error("user does not have access to the drive item"), + new CrudException("user does not have access drive item", 401), + ); + } + + if (file) { + const fileEntity = await globalResolver.services.files.save(null, file, options, context); + + await globalResolver.services.documents.documents.createVersion( + driveFile.id, + { + drive_item_id: driveFile.id, + provider: "internal", + file_metadata: { + external_id: fileEntity.id, + source: "internal", + }, + }, + context, + ); + } else if (keepEditing) { + this.logger.error("Inconsistent endEditing call"); + throw new CrudException("Inconsistent endEditing call", 500); + } + + if (!keepEditing) { + try { + const result = await this.repository.atomicCompareAndSet( + driveFile, + "editing_session_key", + editing_session_key, + null, + ); + if (!result.didSet) + throw new Error( + `Couldn't set editing_session_key ${JSON.stringify( + editing_session_key, + )} on DriveFile ${JSON.stringify(driveFile.id)} because it is ${JSON.stringify( + result.currentValue, + )}`, + ); + } catch (error) { + logger.error({ error: `${error}` }, "Failed to cancel editing Drive item"); + CrudException.throwMe(error, new CrudException("Failed to cancel editing Drive item", 500)); + } + } + }; + downloadGetToken = async ( ids: string[], versionId: string | null, diff --git a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts index 1697d80cd..655e9a41d 100644 --- a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts +++ b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts @@ -341,10 +341,12 @@ export class DocumentsController { beginEditing = async ( request: FastifyRequest<{ Params: ItemRequestParams; - Body: { editorApplicationId: string }; + //TODO application id should be received from the token that we have during the login + Body: { editorApplicationId: string; instanceId: string }; }>, ) => { try { + //TODO create application execution context with the application identifier inside const context = getDriveExecutionContext(request); const { id } = request.params; @@ -355,6 +357,36 @@ export class DocumentsController { return await globalResolver.services.documents.documents.beginEditing( id, request.body.editorApplicationId, + request.body.instanceId || "", + context, + ); + } catch (error) { + logger.error({ error: `${error}` }, "Failed to begin editing Drive item"); + CrudException.throwMe(error, new CrudException("Failed to begin editing Drive item", 500)); + } + }; + + /** + * Finish an editing session by cancelling it. + */ + cancelEditing = async ( + request: FastifyRequest<{ + Params: ItemRequestByEditingSessionKeyParams; + Body: { editorApplicationId: string }; + }>, + ) => { + try { + const context = getDriveExecutionContext(request); + const { editing_session_key } = request.params; + + if (!editing_session_key) throw new CrudException("Missing editing_session_key", 400); + + return await globalResolver.services.documents.documents.updateEditing( + editing_session_key, + null, + null, + false, + null, context, ); } catch (error) { @@ -362,6 +394,57 @@ export class DocumentsController { CrudException.throwMe(error, new CrudException("Failed to begin editing Drive item", 500)); } }; + //TODO: will need a save under session key, but without ending the edit (for force saves) + /** + * Finish an editing session for a given `editing_session_key` by uploading the new version of the File. + * Unless the `keepEditing` query param is `true`, then just save and stay in editing mode. + */ + updateEditing = async ( + request: FastifyRequest<{ + Params: ItemRequestByEditingSessionKeyParams; + Querystring: { keepEditing?: string; userId?: string }; + Body: { + item: Partial; + version: Partial; + }; + }>, + ) => { + const { editing_session_key } = request.params; + if (!editing_session_key) throw new CrudException("Editing session key must be set", 400); + + const context = getDriveExecutionContext(request); + + if (request.isMultipart()) { + const file = await request.file(); + const q: Record = request.query; + const options: UploadOptions = { + totalChunks: parseInt(q.resumableTotalChunks || q.total_chunks) || 1, + totalSize: parseInt(q.resumableTotalSize || q.total_size) || 0, + chunkNumber: parseInt(q.resumableChunkNumber || q.chunk_number) || 1, + filename: q.resumableFilename || q.filename || file?.filename || undefined, + type: q.resumableType || q.type || file?.mimetype || undefined, + waitForThumbnail: !!q.thumbnail_sync, + ignoreThumbnails: false, + }; + return await globalResolver.services.documents.documents.updateEditing( + editing_session_key, + file, + options, + request.query.keepEditing == "true", + request.query.userId, + context, + ); + } else { + return await globalResolver.services.documents.documents.updateEditing( + editing_session_key, + null, + null, + true, + request.query.userId, + context, + ); + } + }; downloadGetToken = async ( request: FastifyRequest<{ diff --git a/tdrive/backend/node/src/services/documents/web/routes.ts b/tdrive/backend/node/src/services/documents/web/routes.ts index ed1e3e5da..25fde912a 100644 --- a/tdrive/backend/node/src/services/documents/web/routes.ts +++ b/tdrive/backend/node/src/services/documents/web/routes.ts @@ -5,6 +5,7 @@ import { createDocumentSchema, createVersionSchema, beginEditingSchema } from ". const baseUrl = "/companies/:company_id"; const serviceUrl = `${baseUrl}/item`; +const editingSessionBase = "/editing_session/:editing_session_key"; const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) => { const documentsController = new DocumentsController(); @@ -89,11 +90,25 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) fastify.route({ method: "GET", - url: `${serviceUrl}/editing_session/:editing_session_key`, + url: editingSessionBase, //TODO NONONO check authenticate*Optional* preValidation: [fastify.authenticateOptional], handler: documentsController.getByEditingSessionKey.bind(documentsController), }); + fastify.route({ + method: "POST", + url: editingSessionBase, + preValidation: [fastify.authenticateOptional], + handler: documentsController.updateEditing.bind(documentsController), + }); + + fastify.route({ + method: "DELETE", + url: editingSessionBase, + preValidation: [fastify.authenticateOptional], + handler: documentsController.cancelEditing.bind(documentsController), + }); + fastify.route({ method: "GET", url: `${serviceUrl}/download/token`, diff --git a/tdrive/backend/node/src/services/documents/web/schemas.ts b/tdrive/backend/node/src/services/documents/web/schemas.ts index 81a095d21..65384de78 100644 --- a/tdrive/backend/node/src/services/documents/web/schemas.ts +++ b/tdrive/backend/node/src/services/documents/web/schemas.ts @@ -115,6 +115,7 @@ export const beginEditingSchema = { type: "object", properties: { editorApplicationId: { type: "string" }, + appInstanceId: { type: "string" }, }, required: ["editorApplicationId"], }, diff --git a/tdrive/backend/node/src/utils/uuid.ts b/tdrive/backend/node/src/utils/uuid.ts index 31a73b91d..608ad964f 100644 --- a/tdrive/backend/node/src/utils/uuid.ts +++ b/tdrive/backend/node/src/utils/uuid.ts @@ -12,3 +12,35 @@ export function timeuuidToDate(time_str: string): number { return parseInt(time_string, 16); } + +/** Remove `-`s from a formatted UUID to get a hex a string */ +export function hexFromFormatted(uuid: string) { + const result = uuid.replace(/-+/g, ""); + if (result.length !== 32) + throw new Error(`Invalid UUID (${JSON.stringify(uuid)}). Wrong number of digits`); + return result; +} + +/** Add `-`s back in a hex string to make a formatted UUID */ +export function formattedFromHex(hex: string) { + const idMatch = hex.match( + /^([a-z0-f]{8})([a-z0-f]{4})([a-z0-f]{4})([a-z0-f]{4})([a-z0-f]{12})$/i, + ); + if (!idMatch) + throw new Error(`Invalid UUID hex (${JSON.stringify(hex)}). Wrong number of digits`); + const [, ...parts] = idMatch; + return parts.join("-"); +} + +/** Convert a UUID formatted or hex string into a binary buffer */ +export function bufferFromUUIDString(uuidOrHex: string) { + return Buffer.from(hexFromFormatted(uuidOrHex), "hex"); +} + +/** In a buffer with concatenated UUIDs in binary, extract the `index`th and return as formatted UUID */ +export function formattedUUIDInBufferArray(buffer: Buffer, index: number) { + if (buffer.length < (index + 1) * 16) + throw new Error(`Cannot get UUID ${JSON.stringify(index)} because the buffer is too small`); + const slice = buffer.subarray(index * 16, (index + 1) * 16); + return formattedFromHex(slice.toString("hex").toLowerCase()); +} diff --git a/tdrive/backend/node/test/e2e/common/user-api.ts b/tdrive/backend/node/test/e2e/common/user-api.ts index c41c6fc93..0ce15cde9 100644 --- a/tdrive/backend/node/test/e2e/common/user-api.ts +++ b/tdrive/backend/node/test/e2e/common/user-api.ts @@ -409,6 +409,40 @@ export default class UserApi { }); } + async updateEditingDocument( + editingSessionKey: string, + keepEditing: boolean = false, + userId: string | null = null, + ): Promise { + const fullPath = `${__dirname}/assets/${UserApi.ALL_FILES[0]}`; + const readable= Readable.from(fs.createReadStream(fullPath)); + const form = formAutoContent({ file: readable }); + form.headers["authorization"] = `Bearer ${this.jwt}`; + let queryString = keepEditing ? "keepEditing=true" : ""; + if (userId) + queryString += `${queryString.length ? "&" : ""}userId=${encodeURIComponent(userId)}`; + return await this.platform.app.inject({ + method: "POST", + url: `${UserApi.DOC_URL}/editing_session/${encodeURIComponent(editingSessionKey)}${queryString ? "?" : ""}${queryString}`, + headers: { + authorization: `Bearer ${this.jwt}` + }, + ...form, + }); + } + + async cancelEditingDocument( + editingSessionKey: string, + ): Promise { + return await this.platform.app.inject({ + method: "DELETE", + url: `${UserApi.DOC_URL}/editing_session/${editingSessionKey}`, + headers: { + authorization: `Bearer ${this.jwt}` + } + }); + } + async beginEditingDocumentExpectOk( driveFileId: string, editorApplicationId: string, @@ -486,7 +520,7 @@ export default class UserApi { async getDocumentByEditingKey(editing_session_key: string) { return await this.platform.app.inject({ method: "GET", - url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/editing_session/${encodeURIComponent(editing_session_key)}`, + url: `${UserApi.DOC_URL}/editing_session/${encodeURIComponent(editing_session_key)}`, headers: { authorization: `Bearer ${this.jwt}` } diff --git a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts index 8c5e6ce5f..3f9660499 100644 --- a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts +++ b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts @@ -1,9 +1,12 @@ -import { describe, beforeAll, beforeEach, it, expect, afterAll } from "@jest/globals"; +import { describe, beforeAll, beforeEach, it, expect, afterAll, jest } from "@jest/globals"; import { init, TestPlatform } from "../setup"; import UserApi from "../common/user-api"; import { DriveFile, TYPE as DriveFileType } from "../../../src/services/documents/entities/drive-file"; +import ApplicationsApiService, { ApplicationEditingKeyStatus } from "../../../src/services/applications-api"; +import { afterEach } from "node:test"; +import Application from "../../../src/services/applications/entities/application"; describe("the Drive's documents' editing session kind-of-lock", () => { let platform: TestPlatform | null; @@ -48,6 +51,12 @@ describe("the Drive's documents' editing session kind-of-lock", () => { parent_id: currentUserRoot, scope: "personal", }); + jest.spyOn(ApplicationsApiService.getDefault(), 'getApplicationConfig').mockImplementation((id) => id === "e2e_testing" ? {} as Application : undefined); + jest.spyOn(ApplicationsApiService.getDefault(), 'checkPendingEditingStatus').mockImplementation(async () => ApplicationEditingKeyStatus.unknown); + }); + + afterEach(() => { + jest.restoreAllMocks(); }); it("atomicCompareAndSet allows a single value at a time", async () => { @@ -110,8 +119,28 @@ describe("the Drive's documents' editing session kind-of-lock", () => { expect(temporaryDocument.id).toBe(foundDocumentResult.json().id); }); - it('can end an editing session on a document only once with the right key', async () => { + it('can cancel an editing session on a document only once with the right key', async () => { + //given const editingSessionKey = await currentUser.beginEditingDocumentExpectOk(temporaryDocument.id, 'e2e_testing'); + //when + const response = await currentUser.cancelEditingDocument(editingSessionKey); + //then + expect(response.statusCode).toBe(200); + const newSessionKey = await currentUser.beginEditingDocumentExpectOk(temporaryDocument.id, 'e2e_testing'); + expect(newSessionKey).not.toEqual(editingSessionKey); + }); + it('can end editing with a new version of document', async () => { + //given + const editingSessionKey = await currentUser.beginEditingDocumentExpectOk(temporaryDocument.id, 'e2e_testing'); + //when + const response = await currentUser.updateEditingDocument(editingSessionKey); + + //then + expect(response.statusCode).toBe(200); + const document = await currentUser.getDocumentOKCheck(temporaryDocument.id); + expect(document.versions.length).toEqual(2); }); + + }); diff --git a/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts b/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts index 62f7e846d..6145b1d95 100644 --- a/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts +++ b/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts @@ -105,6 +105,42 @@ describe('The PostgresQueryBuilder', () => { expect(query[1]).toEqual(options.$like.map(e=> `%${e[1]}%`)); }); + test('buildSelect query with IS NULL filter', async () => { + //given + const filters = { companu_id: null }; + + //when + const query = subj.buildSelect(TestDbEntity, filters, {}); + + //then + expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM \"test_table\" WHERE companu_id IS NULL ORDER BY id DESC LIMIT 100 OFFSET 0`); + expect(query[1]).toEqual([]); + }); + + test('buildSelect query with not equal filter', async () => { + //given + const filters = { company_id: { $ne: '123' } }; + + //when + const query = subj.buildSelect(TestDbEntity, filters, {}); + + //then + expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE company_id != $1 ORDER BY id DESC LIMIT 100 OFFSET 0`); + expect(query[1]).toEqual([ '123' ]); + }); + + test('buildSelect query with IS NOT NULL filter', async () => { + //given + const filters = { company_id: { $ne: null } }; + + //when + const query = subj.buildSelect(TestDbEntity, filters, {}); + + //then + expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE company_id IS NOT NULL ORDER BY id DESC LIMIT 100 OFFSET 0`); + expect(query[1]).toEqual([]); + }); + test('buildDelete query', async () => { //given const entity = newTestDbEntity(); diff --git a/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts b/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts new file mode 100644 index 000000000..c5e10cac3 --- /dev/null +++ b/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts @@ -0,0 +1,55 @@ +import "reflect-metadata"; +import { expect, jest, test, describe, beforeEach, afterEach } from "@jest/globals"; +import { randomUUID } from "crypto"; +import { EditingSessionKeyFormat } from "../../../../../src/services/documents/entities/drive-file"; + +describe('DriveFile EditingSessionKeyFormat', () => { + const mockAppId = 'tdrive_random_application_id'; + const mockCompanyId = randomUUID(); + const mockUserId = randomUUID(); + const mockInstanceId = "super-instance-id"; + const mockTimestamp = new Date(); + const mockTimestampWith0MS = mockTimestamp.getTime() - (mockTimestamp.getTime() % 1000); + + const checkIsUUID = (value: string) => { + expect(value).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i); + expect(value).not.toMatch(/^0{8}-0{4}-0{4}-0{4}-0{12}$/i); + }; + + const checkKeyIsOOCompatible = (key: string) => { + // OnlyOffice key limits: see https://api.onlyoffice.com/editors/config/document#key + expect(key).toMatch(/^[0-9a-zA-Z._=-]{1,128}$/); + }; + + test('generates a valid value that can be parsed', async () => { + checkIsUUID(mockUserId); + checkIsUUID(mockCompanyId); + const key = EditingSessionKeyFormat.generate(mockAppId, mockInstanceId, mockCompanyId, mockUserId, mockTimestamp); + checkKeyIsOOCompatible(key); + const parsed = EditingSessionKeyFormat.parse(key); + expect(parsed.applicationId).toBe(mockAppId); + expect(parsed.instanceId).toBe(mockInstanceId); + expect(parsed.userId).toBe(mockUserId); + expect(parsed.companyId).toBe(mockCompanyId); + expect(parsed.timestamp.getTime()).toBe(mockTimestampWith0MS); + }); + + test('generates unique values', async () => { + const key = EditingSessionKeyFormat.generate(mockAppId, mockInstanceId, mockCompanyId, mockUserId, mockTimestamp); + const key2 = EditingSessionKeyFormat.generate(mockAppId, mockInstanceId, mockCompanyId, mockUserId, mockTimestamp); + expect(key).not.toBe(key2); + }); + + test('checks the appId', async () => { + expect(() => { + EditingSessionKeyFormat.generate('invalid app id !', mockInstanceId, mockCompanyId, mockUserId); + }).toThrow('Invalid applicationId value'); + }); + + test('checks final length', async () => { + expect(() => { + const tooLongAppID = new Array(100).join('x'); + EditingSessionKeyFormat.generate(tooLongAppID, mockInstanceId, mockCompanyId, mockUserId); + }).toThrow('Must be <128 chars,'); + }); +}); \ No newline at end of file diff --git a/tdrive/connectors/onlyoffice-connector/readme.md b/tdrive/connectors/onlyoffice-connector/readme.md index 6811f33d7..cad453bd9 100644 --- a/tdrive/connectors/onlyoffice-connector/readme.md +++ b/tdrive/connectors/onlyoffice-connector/readme.md @@ -18,6 +18,7 @@ Documentation of the [API used with OnlyOffice is here](https://api.onlyoffice.c - The inline configuration provides a read and a write URL to the DocsAPI editor, these point to the routes of `OnlyOfficeController`. - When all the clients have disconnected from the editing session on the OnlyOffice document editing service, the OnlyOffice server will call the `callbackUrl` of this connector. - The connector then downloads the new file from Only Office, and creates a new version in Twake Drive. +- Periodically gets a list of forgotten files and updates the backend with them ## Configuration example diff --git a/tdrive/connectors/onlyoffice-connector/src/app.ts b/tdrive/connectors/onlyoffice-connector/src/app.ts index 65258ba9a..d85424a94 100644 --- a/tdrive/connectors/onlyoffice-connector/src/app.ts +++ b/tdrive/connectors/onlyoffice-connector/src/app.ts @@ -1,65 +1,49 @@ +import path from 'path'; import express from 'express'; -import { Routes } from '@interfaces/routes.interface'; -import { NODE_ENV } from '@config'; import cors from 'cors'; import { renderFile } from 'eta'; -import path from 'path'; -import errorMiddleware from './middlewares/error.middleware'; -import { SERVER_PORT, SERVER_PREFIX } from '@config'; -import logger from './lib/logger'; import cookieParser from 'cookie-parser'; -import apiService from './services/api.service'; -import onlyofficeService from './services/onlyoffice.service'; + +import { NODE_ENV, SERVER_BIND, SERVER_PORT } from '@config'; +import logger from './lib/logger'; +import errorMiddleware from './middlewares/error.middleware'; +import { mountRoutes } from './routes'; + +import forgottenProcessorService from './services/forgotten-processor.service'; class App { public app: express.Application; public env: string; public port: string | number; - constructor(routes: Routes[]) { + constructor() { this.app = express(); this.env = NODE_ENV; this.initViews(); this.initMiddlewares(); - this.initRoutes(routes); + this.initRoutes(); this.initErrorHandling(); + + forgottenProcessorService.makeSureItsLoaded(); } public listen = () => { - this.app.listen(parseInt(SERVER_PORT, 10), '0.0.0.0', () => { - logger.info(`🚀 App listening on port ${SERVER_PORT}`); + const binding_host = SERVER_BIND || '0.0.0.0'; + this.app.listen(parseInt(SERVER_PORT, 10), binding_host, () => { + logger.info(`🚀 App listening at http://${binding_host}:${SERVER_PORT}`); }); }; public getServer = () => this.app; - private initRoutes = (routes: Routes[]) => { + private initRoutes = () => { this.app.use((req, res, next) => { logger.info(`Received request: ${req.method} ${req.originalUrl} from ${req.header('user-agent')} (${req.ip})`); next(); }); - routes.forEach(route => { - this.app.use(SERVER_PREFIX, route.router); - }); - - this.app.get('/health', (_req, res) => { - Promise.all([onlyofficeService.getLatestVersion(), apiService.hasToken()]).then( - ([version, twakeDriveToken]) => res.status(version && twakeDriveToken ? 200 : 500).send({ version, twakeDriveToken }), - err => res.status(500).send(err), - ); - }); - - this.app.use( - SERVER_PREFIX.replace(/\/$/, '') + '/assets', - (req, res, next) => { - res.header('Access-Control-Allow-Origin', '*'); - res.header('Access-Control-Allow-Headers', 'X-Requested-With'); - next(); - }, - express.static(path.join(__dirname, '../assets')), - ); + mountRoutes(this.app); }; private initMiddlewares = () => { diff --git a/tdrive/connectors/onlyoffice-connector/src/config/index.ts b/tdrive/connectors/onlyoffice-connector/src/config/index.ts index ca0a3f5e1..9dc314361 100644 --- a/tdrive/connectors/onlyoffice-connector/src/config/index.ts +++ b/tdrive/connectors/onlyoffice-connector/src/config/index.ts @@ -4,6 +4,7 @@ config({ path: `.env.${process.env.NODE_ENV || 'development'}.local` }); export const { NODE_ENV, SERVER_PORT, + SERVER_BIND, SECRET_KEY, CREDENTIALS_ENDPOINT, ONLY_OFFICE_SERVER, @@ -11,4 +12,14 @@ export const { CREDENTIALS_SECRET, SERVER_PREFIX, SERVER_ORIGIN, + INSTANCE_ID, + OOCONNECTOR_HEALTH_SECRET, } = process.env; + +const secs = 1000, + mins = 60 * secs; + +export const twakeDriveTokenRefrehPeriodMS = 10 * mins; +export const onlyOfficeForgottenFilesCheckPeriodMS = 10 * mins; +export const onlyOfficeConnectivityCheckPeriodMS = 10 * mins; +export const onlyOfficeCallbackTimeoutMS = 10 * secs; diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts new file mode 100644 index 000000000..7ccb76ee5 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts @@ -0,0 +1,100 @@ +import { Request, Response } from 'express'; +import logger from '@/lib/logger'; +import { createSingleProcessorLock } from '@/lib/single-processor-lock'; +import onlyofficeService, { Callback, CommandError, ErrorCode } from '@/services/onlyoffice.service'; +import driveService from '@/services/drive.service'; +import forgottenProcessorService from '@/services/forgotten-processor.service'; +import { registerHealthProvider } from '@/services/health-providers.service'; + +interface RequestQuery { + editing_session_key: string; +} +interface RenameRequestBody { + title: string; +} +const keyCheckLock = createSingleProcessorLock<[status: number, body: unknown]>(); + +registerHealthProvider({ + async getHealthData() { + return { checks: { locks: keyCheckLock.getWorstStats() } }; + }, +}); + +/** + * These routes are called by Twake Drive backend, for ex. before editing or retreiving a file, + * if it has an editing_session_key still, get the status of that and force a resolution. + */ +export default class TwakeDriveBackendCallbackController { + /** + * Get status of an `editing_session_key` from OO, and return a URL to get the latest version, + * or an object with no `url` property, in which case the key is not known as forgotten by OO and should + * be considered lost after an admin alert. + * + * @returns + * - `{ status: 'unknown' }`: the key isn't known and maybe used for a new session + * - `{ status: 'updated' }`: the key needed updating but is now invalid + * - `{ status: 'expired' }`: the key was already used in a finished session and can't be used again + * - `{ status: 'live' }`: the key is valid and current and should be used again for the same file + * - `{ error: number }`: there was an error retreiving the status of the key, http status `!= 200` + */ + public async checkSessionStatus(req: Request, res: Response): Promise { + const [status, body] = await keyCheckLock.runWithLock(req.params.editing_session_key, async () => { + try { + const forgottenURL = await onlyofficeService.getForgotten(req.params.editing_session_key); + try { + await forgottenProcessorService.processForgottenFile(req.params.editing_session_key, forgottenURL); + } catch (error) { + logger.error(`processForgottenFile failed`, { error }); + return [502, { error: -57650 }]; + } + return [200, { status: 'updated' }]; + } catch (e) { + if (!(e instanceof CommandError && e.errorCode == ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND)) { + logger.error(`getForgotten failed`, { error: e }); + return [e instanceof CommandError ? 502 : 500, { error: -57651 }]; + } + } + const info = await onlyofficeService.getInfoAndWaitForCallbackUnsafe(req.params.editing_session_key); + if (info.error === ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND) { + // just start using it + return [200, { status: 'unknown' }]; + } + if (info.error !== undefined) { + logger.error(`getInfo failed`, { error: info }); + return [502, { error: -57652 }]; + } + switch (info.result.status) { + case Callback.Status.BEING_EDITED: + case Callback.Status.BEING_EDITED_BUT_IS_SAVED: + // use it as is + return [200, { status: 'live' }]; + + case Callback.Status.CLOSED_WITHOUT_CHANGES: + // just cancel it + return [200, { status: 'expired' }]; + + case Callback.Status.ERROR_FORCE_SAVING: + case Callback.Status.ERROR_SAVING: + return [502, { error: info.result.status }]; + + case Callback.Status.READY_FOR_SAVING: + // upload it, have to do it here for correct user stored in url in OO + await driveService.endEditing(req.params.editing_session_key, info.result.url); + return [200, { status: 'updated' }]; + + default: + throw new Error(`Unexpected callback status: ${JSON.stringify(info.result)}`); + } + }); + await res.status(status).send(body); + } + + public async updateSessionFilename(req: Request, res: Response): Promise { + try { + await onlyofficeService.meta(req.params.editing_session_key, req.body.title); + res.send({ ok: 1 }); + } catch (err) { + res.status(500).send({ error: -58650 }); + } + } +} diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts similarity index 76% rename from tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts rename to tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts index 61e6c8734..d2c82065d 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts @@ -1,13 +1,13 @@ import editorService from '@/services/editor.service'; import { NextFunction, Request, Response } from 'express'; -import { CREDENTIALS_SECRET, SERVER_ORIGIN, SERVER_PREFIX } from '@config'; +import { CREDENTIALS_SECRET } from '@config'; import jwt from 'jsonwebtoken'; import driveService from '@/services/drive.service'; import { DriveFileType } from '@/interfaces/drive.interface'; import fileService from '@/services/file.service'; -import { OfficeToken } from '@/interfaces/routes.interface'; +import { OfficeToken } from '@/interfaces/office-token.interface'; import logger from '@/lib/logger'; -import * as Utils from '@/utils'; +import { makeURLTo } from '@/routes'; interface RequestQuery { mode: string; @@ -22,6 +22,7 @@ interface RequestEditorQuery { office_token: string; company_id: string; file_id: string; + drive_file_id: string; } /** @@ -29,7 +30,7 @@ interface RequestEditorQuery { * The user is redirected from there to open directly the OnlyOffice edition server's web UI, with appropriate preview or not * and rights checks. */ -class IndexController { +class BrowserEditorController { /** * Opened by the user's browser, proxied through the Twake Drive backend. Checks access to the * file with the backend, then redirects the user to the `editor` method but directly on this @@ -74,25 +75,36 @@ class IndexController { throw new Error('You do not have access to this file'); } + let editingSessionKey = null; + if (!preview) { + editingSessionKey = await driveService.beginEditingSession(company_id, drive_file_id, token); + //TODO catch error and display to the user when we can't stopped editing + + //TODO Log error with format to be able to set up grafana alert fir such king of errors + } + const officeToken = jwt.sign( { user_id: user.id, //To verify that link is opened by the same user company_id, drive_file_id, + editing_session_key: editingSessionKey, file_id: file.id, - file_name: file.filename || file?.metadata?.name || '', + file_name: driveFile?.item?.name || file.filename || file.metadata?.name || '', preview: !!preview, } as OfficeToken, CREDENTIALS_SECRET, { + //one month, never expiring token expiresIn: 60 * 60 * 24 * 30, }, ); - res.redirect( - Utils.joinURL([SERVER_ORIGIN ?? '', SERVER_PREFIX, 'editor'], { + makeURLTo.editorAbsolute({ token, file_id, + drive_file_id, + editing_session_key: editingSessionKey, company_id, preview, office_token: officeToken, @@ -112,13 +124,16 @@ class IndexController { const { user } = req; const officeTokenPayload = jwt.verify(office_token, CREDENTIALS_SECRET) as OfficeToken; - const { preview, user_id, company_id, file_name, file_id, drive_file_id } = officeTokenPayload; + const { preview, user_id, company_id, file_name, file_id, drive_file_id, editing_session_key } = officeTokenPayload; if (user_id !== user.id) { throw new Error('You do not have access to this link'); } + if (!preview && !editing_session_key) { + throw new Error('Cant start editing without "editing session key"'); + } - const initResponse = await editorService.init(company_id, file_name, file_id, user, preview, drive_file_id || file_id); + const initResponse = await editorService.init(company_id, file_name, file_id, user, preview, drive_file_id); const inPageToken = jwt.sign( { @@ -130,7 +145,8 @@ class IndexController { res.render('index', { ...initResponse, - server: Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), + docId: preview ? file_id : editing_session_key, + server: makeURLTo.rootAbsolute(), token: inPageToken, }); } catch (error) { @@ -140,4 +156,4 @@ class IndexController { }; } -export default IndexController; +export default BrowserEditorController; diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts new file mode 100644 index 000000000..61ca91be3 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts @@ -0,0 +1,16 @@ +import { Request, Response } from 'express'; +import { getProviders } from '@/services/health-providers.service'; +import { OOCONNECTOR_HEALTH_SECRET } from '@/config'; + +/** + * Health response for devops operational purposes. Should not be exposed. + */ +export const HealthStatusController = { + async get(req: Request<{}, {}, {}, { secret: string }>, res: Response): Promise { + if (req.query.secret !== OOCONNECTOR_HEALTH_SECRET) return void res.status(404).send(`Cannot GET ${req.path}`); + const entries = await Promise.all(getProviders().map(p => p.getHealthData())); + let result = {}; + entries.forEach(entry => (result = { ...result, ...entry })); + res.send(result); + }, +}; diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 5c368082a..3a7ad59c2 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -1,18 +1,24 @@ import { CREDENTIALS_SECRET } from '@/config'; -import { OfficeToken } from '@/interfaces/routes.interface'; +import { OfficeToken } from '@/interfaces/office-token.interface'; import driveService from '@/services/drive.service'; import fileService from '@/services/file.service'; import logger from '@/lib/logger'; import * as OnlyOffice from '@/services/onlyoffice.service'; import { Request, Response, NextFunction } from 'express'; import jwt from 'jsonwebtoken'; +import * as Utils from '@/utils'; interface RequestQuery { company_id: string; file_id: string; + drive_file_id: string; token: string; } +interface RenameRequestBody { + name: string; +} + /** These expose a OnlyOffice document storage service methods, called by the OnlyOffice document editing service * to load and save files */ @@ -28,12 +34,12 @@ class OnlyOfficeController { const { token } = req.query; const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; - const { company_id, drive_file_id, file_id, in_page_token } = officeTokenPayload; - let fileId = file_id; + const { company_id, file_id, drive_file_id, in_page_token } = officeTokenPayload; // check token is an in_page_token if (!in_page_token) throw new Error('Invalid token, must be a in_page_token'); + let fileId = file_id; if (drive_file_id) { //Get the drive file const driveFile = await driveService.get({ @@ -45,6 +51,7 @@ class OnlyOfficeController { } } + if (!file_id) throw new Error(`File id is missing in the last version cache for ${JSON.stringify(file_id)}`); const file = await fileService.download({ company_id, file_id: fileId, @@ -74,57 +81,91 @@ class OnlyOfficeController { try { const { url, key } = req.body; const { token } = req.query; - logger.info('Save request', { key, url, token }); - + logger.info( + `OO callback status: ${Utils.getKeyForValueSafe(req.body.status, OnlyOffice.Callback.Status, 'Callback.Status')} - ${JSON.stringify( + req.body.userdata, + )}`, + req.body, + ); const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; - const { preview, company_id, file_id, /* user_id, */ drive_file_id, in_page_token } = officeTokenPayload; + const { preview, user_id, /* company_id, file_id, drive_file_id, */ in_page_token /* editing_session_key */ } = officeTokenPayload; // check token is an in_page_token and allow save - if (!in_page_token) throw new Error('Invalid token, must be a in_page_token'); - if (preview) throw new Error('Invalid token, must not be a preview token for save operation'); + if (!in_page_token) throw new Error('OO Callback invalid token, must be a in_page_token'); + if (preview) throw new Error('OO Callback invalid token, must not be a preview token for save operation'); switch (req.body.status) { case OnlyOffice.Callback.Status.BEING_EDITED: - case OnlyOffice.Callback.Status.BEING_EDITED_BUT_IS_SAVED: + // TODO this call back we recieve almost all the time, and here we save + // the user identifiers who start file editing and even control the amount of onlin users + // to have license constraint warning before OnlyOffice error about this // No-op break; + case OnlyOffice.Callback.Status.BEING_EDITED_BUT_IS_SAVED: + logger.info(`OO Callback force save for session ${key} for reason: ${OnlyOffice.Callback.ForceSaveTypeToString(req.body.forcesavetype)}`); + await driveService.addEditingSessionVersion(key, url, user_id); + break; + case OnlyOffice.Callback.Status.READY_FOR_SAVING: - const newVersionFile = await fileService.save({ - company_id, - file_id, - url, - create_new: true, - }); - - const version = await driveService.createVersion({ - company_id, - drive_file_id, - file_id: newVersionFile?.resource?.id, - }); - logger.info('New version created', version); - return respondToOO(); + logger.info(`OO Callback new version for session ${key} created`); + await driveService.endEditing(key, url, user_id); + break; case OnlyOffice.Callback.Status.CLOSED_WITHOUT_CHANGES: - // Save end of transaction + await driveService.cancelEditing(key); break; case OnlyOffice.Callback.Status.ERROR_SAVING: - // Save end of transaction - logger.info(`Error saving file ${req.body.url} (key: ${req.body.key})`); + // TODO: Save end of transaction ? + logger.error(`OO Callback with Status.ERROR_SAVING: ${req.body.url} (key: ${req.body.key})`); break; case OnlyOffice.Callback.Status.ERROR_FORCE_SAVING: // TODO: notify user ? - logger.info(`Error force saving (reason: ${req.body.forcesavetype}) file ${req.body.url} (key: ${req.body.key})`); - return void res.send({ error: 0 }); + logger.error( + `OO Callback with Status.ERROR_FORCE_SAVING (reason: ${OnlyOffice.Callback.ForceSaveTypeToString(req.body.forcesavetype)}) file ${ + req.body.url + } (key: ${req.body.key})`, + ); + break; default: - throw new Error(`Unexpected OO Callback status field: ${req.body.status}`); + throw new Error( + `OO Callback unexpected status field: ${OnlyOffice.Callback.StatusToString(req.body.status)} in ${JSON.stringify(req.body)}`, + ); } - return respondToOO(); + + // Ignore errors generated by pending request + // try-catch not needed because it is async + // there may be later reasons to wait for callbacks + // to process and eventually respond accordingly to + // OO an error for certain statuses + void OnlyOffice.default.ooCallbackCalled(req.body); // has to be single thread per key + + return respondToOO(0); } catch (error) { - next(error); + logger.error(`OO Callback root error`, { error }); + next(error || 'error'); + } + }; + + /** This route is called directly by the inline JS in the editor page, called by the client-side OO editor component */ + public rename = async (req: Request<{}, {}, RenameRequestBody, RequestQuery>, res: Response, next: NextFunction): Promise => { + try { + const { token } = req.query; + const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; + const { company_id, drive_file_id } = officeTokenPayload; + const { name } = req.body; + + if (!drive_file_id) throw new Error('OO Rename request missing drive_file_id'); + if (!name) throw new Error('OO Rename request missing name'); + + const result = await driveService.update({ company_id, drive_file_id, changes: { name } }); + res.send(result); + } catch (error) { + logger.error(`OO Rename request root error`, { error }); + next(error || 'error'); } }; } diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts index 9d274d668..4384d561e 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts @@ -1,11 +1,13 @@ export type DriveFileType = { access: 'manage' | 'write' | 'read' | 'none'; item: { + name: string; last_version_cache: { id: string; date_added: number; file_metadata: { external_id: string; + name?: string; }; }; }; @@ -19,4 +21,7 @@ export type DriveRequestParams = { export interface IDriveService { get: (params: DriveRequestParams) => Promise; createVersion: (params: { company_id: string; drive_file_id: string; file_id: string }) => Promise; + beginEditingSession: (company_id: string, drive_file_id: string) => Promise; + addEditingSessionVersion: (editing_session_key: string, url: string) => Promise; + endEditing: (editing_session_key: string, url: string) => Promise; } diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/editor.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/editor.interface.ts index 7aef5de3a..c23aedb4b 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/editor.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/editor.interface.ts @@ -17,7 +17,7 @@ export type EditConfigInitResult = { onlyoffice_server: string; color: string; company_id: string; - file_id: string; + drive_file_id: string; file_version_id: string; filename: string; file_type: string; @@ -33,6 +33,7 @@ export interface IEditorService { user: UserType, preview: boolean, file_id: string, + drive_file_id: string, ) => Promise; } diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/office-token.interface.ts similarity index 65% rename from tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts rename to tdrive/connectors/onlyoffice-connector/src/interfaces/office-token.interface.ts index 7a0295461..a52fc3371 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/office-token.interface.ts @@ -1,16 +1,10 @@ -import { Router } from 'express'; - -export interface Routes { - path?: string; - router: Router; -} - export interface OfficeToken { user_id: string; company_id: string; file_id: string; file_name: string; preview: boolean; + editing_session_key: string; drive_file_id?: string; in_page_token?: boolean; } diff --git a/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts b/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts new file mode 100644 index 000000000..26cd6387f --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts @@ -0,0 +1,106 @@ +import logger from './logger'; + +export type PendingRequestCallback = (timeout: boolean, result?: TResult) => Promise; + +/** Represents a single pending query that can be resolved with success or failure */ +class PendingRequest { + private readonly startedAt = new Date(); + private callbacks?: PendingRequestCallback[] = []; + + constructor(public readonly key: string, public readonly userdata: string, callback: PendingRequestCallback) { + this.addCallback(callback); + } + + public addCallback(callback: PendingRequestCallback) { + if (!this.callbacks) throw new Error("Cannot add callback to PendingRequest after it's been resolved"); + this.callbacks.push(callback); + return this; + } + + public getAge() { + return new Date().getTime() - this.startedAt.getTime(); + } + + public matches(key: string, userdata: string) { + return this.key === key && this.userdata === userdata; + } + + public async resolve(...cbArgs: Parameters>) { + if (!this.callbacks) throw new Error("Cannot resolve PendingRequest after it's already been resolved"); + const callbacks = this.callbacks; + this.callbacks = null; + return Promise.all(callbacks.map(fn => fn(...cbArgs))); + } +} + +/** + * Tracks a list of pending requests that are started with a pair + * of `key` and `userdata` strings. (Note. `userdata` is not the + * typical use of user data as a `void *`, but instead used to + * identify the specific request asynchroneously). + * Both must match. + * + * There is no timing garantee as to when expired requests' callbacks + * are executed. + */ +export class PendingRequestQueue { + private queue: PendingRequest[] = []; + + constructor(private readonly timeoutMs: number, niquystSamplingRatio = 4) { + setInterval(() => { + this.flush(); + }, timeoutMs / niquystSamplingRatio); + } + + /** + * Add a callback to be called when `gotResult` or `cancelPending` are + * called with identical `key` and `userdata` + * @param key First half of identifying string + * @param userdata Second half of identifying string + * @param callback When the request is resolved, succesfully or in error + */ + public enqueue(key: string, userdata: string, callback: PendingRequestCallback) { + const existing = this.queue.find(pending => pending.key === key && pending.userdata === userdata); + if (existing) return void existing.addCallback(callback); + this.queue.push(new PendingRequest(key, userdata, callback)); + } + + private remove(predicate: (request: PendingRequest) => boolean): PendingRequest[] { + // Warning: This operation must be synchroneous + const foundRequests = []; + this.queue = this.queue.filter(pending => { + if (!predicate(pending)) return true; + foundRequests.push(pending); + return false; + }); + return foundRequests; + } + + private async resolve(predicate: (request: PendingRequest) => boolean, ...cbArgs: Parameters>) { + const foundRequests = this.remove(predicate); + if (!foundRequests.length) return null; + return Promise.all(foundRequests.map(request => request.resolve(...cbArgs))); + } + + protected async flush() { + const expiredRequests = this.remove(request => request.getAge() > this.timeoutMs); + return Promise.all(expiredRequests.map(request => request.resolve(true))); + } + + public async gotResult(key: string, userdata: string, result?: TResult) { + const resolutions = this.resolve(request => request.matches(key, userdata), false, result); + if (!resolutions) + logger.error(`Got resolution on pending request that was unknown`, { + key, + userdata, + result, + }); + await this.flush(); + return resolutions; + } + + /** Equivalent to `gotResult` with an `undefined` result. (Callbacks are called) */ + public async cancelPending(key: string, userdata: string) { + return this.gotResult(key, userdata, undefined); + } +} diff --git a/tdrive/connectors/onlyoffice-connector/src/lib/polled-thingie-value.ts b/tdrive/connectors/onlyoffice-connector/src/lib/polled-thingie-value.ts index d4c2d8a13..27b95e36f 100644 --- a/tdrive/connectors/onlyoffice-connector/src/lib/polled-thingie-value.ts +++ b/tdrive/connectors/onlyoffice-connector/src/lib/polled-thingie-value.ts @@ -19,8 +19,8 @@ export class PolledThingieValue { private readonly getTheThingieValue: () => Promise>, private readonly intervalMs: number, ) { - this.run(); - setInterval(() => this.run(), this.intervalMs); + this.runIgnoringRejection(); + setInterval(() => this.runIgnoringRejection(), this.intervalMs); } protected setResult(value: undefined, error?: NotUndefined, ts?: number); @@ -52,6 +52,12 @@ export class PolledThingieValue { })); } + private runIgnoringRejection() { + return this.run().catch(() => { + /* active ignoring going on here */ + }); + } + public lastFailed() { return !!this.lastKoTimeMs; } diff --git a/tdrive/connectors/onlyoffice-connector/src/lib/single-processor-lock.ts b/tdrive/connectors/onlyoffice-connector/src/lib/single-processor-lock.ts new file mode 100644 index 000000000..108f3d6a7 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/lib/single-processor-lock.ts @@ -0,0 +1,161 @@ +import logger from './logger'; + +export class CannotSettleAlreadyReleasedLockError extends Error {} + +const debugLocks = false; + +/** Simplifies use of the Promise constructor function out of its scope */ +class ExplodedPromise { + public readonly startAtMs = new Date().getTime(); + public readonly promise: Promise; + private _waiters = 0; + private _resolve: (value: T) => void; + private _reject: (reason?: any) => void; + + constructor() { + this.promise = new Promise((resolve, reject) => { + // Works because this is garanteed to be called synchroneously + this._resolve = resolve; + this._reject = reject; + }); + } + private check() { + if (!this._resolve) throw new CannotSettleAlreadyReleasedLockError(`Promise is already settled`); + } + private finish() { + this._resolve = this._reject = null; + } + /** Increase waiter count and return new count */ + public addWaiter() { + return ++this._waiters; + } + get waiting() { + return this._waiters; + } + public resolve(...args: Parameters['_resolve']>): void { + this.check(); + this._resolve(...args); + this.finish(); + } + public reject(...args: Parameters['_reject']>): void { + this.check(); + this._reject(...args); + this.finish(); + } +} + +/** + * Use this instance to release the lock acquired by {@link SingleProcessorLock.acquire}. + * Only call either `resolve` or `reject` once or they will + * throw a {@link CannotSettleAlreadyReleasedLockError}. + */ +export interface LockReleaser { + /** To distinguish return from {@link LockWaiter} */ + didAcquire: true; + /** This is the same Promise that `resolve` and `reject` will settle, and waiters get */ + promise: LockWaiter['promise']; + /** Release the lock and resolve the promise of all the waiters */ + resolve: ExplodedPromise['resolve']; + /** Release the lock and reject the promise of all the waiters */ + reject: ExplodedPromise['reject']; +} + +/** Use this instance to wait if {@link SingleProcessorLock.acquire} was already processing that key */ +export interface LockWaiter { + /** To distinguish return from {@link LockReleaser} */ + didAcquire: false; + /** Number of waiters in the queue at the start of this one, including this one (ie. first to wait is `1`) */ + numberInQueue: number; + /** Promise that must be waited for after the acquire failed. Will settle as per called by the {@see LockReleaser} */ + promise: Promise; +} + +/** + * Create an synchronisation primitive that permits a single caller + * to process for a given `key`. Other callers with the same `key` + * will receive an `Promise` that should be waited for. When the + * processor caller releases the lock, the promise is settled accordingly. + */ +export function createSingleProcessorLock() { + const promisesByKey = new Map>(); + + /** + * The first caller for the given `key` is the processor that must settle the lock. The + * return value will be an instance of {@link LockReleaser} that can be tested because + * {@link LockReleaser.didAcquire} will be `true`. This caller has the responsibility to + * ensure a call to either {@link LockReleaser.resolve} or {@link LockReleaser.reject}. + * There is no other cleanup mecanism. + * + * All callers following them for the same `key` will get an instance of {@link LockWaiter}, + * that can be tested because {@link LockWaiter.didAcquire} will be `false`. They must + * `await` {@link LockWaiter.promise}. It will be settled by the processor when they call + * the methods of {@link LockReleaser}. must be ensured, there is no cleanup mecanism. + * + * @param key Unique key to lock on + */ + const acquire = (key: string): LockReleaser | LockWaiter => { + const existing = promisesByKey.get(key); + if (existing) { + const numberInQueue = existing.addWaiter(); + debugLocks && logger.debug(`LOCK blocked ${numberInQueue}: ${key}`); + return { + didAcquire: false, + numberInQueue, + promise: existing.promise, + }; + } + const processorPromise = new ExplodedPromise(); + promisesByKey.set(key, processorPromise); + debugLocks && logger.debug(`LOCK started: ${key}`); + return { + didAcquire: true, + promise: processorPromise.promise, + resolve(...args: Parameters['resolve']>) { + debugLocks && logger.debug(`LOCK resolved: ${key}`); + promisesByKey.delete(key); + processorPromise.resolve(...args); + }, + reject(...args: Parameters['reject']>) { + debugLocks && logger.debug(`LOCK rejected: ${key}`); + promisesByKey.delete(key); + processorPromise.reject(...args); + }, + }; + }; + return { + acquire, + + /** + * Attempt to acquire a lock. If succesful, settle the promise with the + * return value of `processor`. In both cases return the Promise. + */ + async runWithLock(key: string, processor: () => Promise): Promise { + const lock = acquire(key); + if (lock.didAcquire) + try { + lock.resolve(await processor()); + } catch (e) { + lock.reject(e); + } + return lock.promise; + }, + + /** Get statistics about pending locks */ + getWorstStats() { + const all = [...promisesByKey.values()]; + const now = new Date().getTime(); + return all.reduce( + ({ oldestMs, waiting, total }, cur) => ({ + oldestMs: Math.max(oldestMs, now - cur.startAtMs), + waiting: Math.max(waiting, cur.waiting), + total: total + 1, + }), + { + oldestMs: 0, + waiting: 0, + total: 0, + }, + ); + }, + }; +} diff --git a/tdrive/connectors/onlyoffice-connector/src/middlewares/auth.middleware.ts b/tdrive/connectors/onlyoffice-connector/src/middlewares/auth.middleware.ts index 8703da88f..5b727c41f 100644 --- a/tdrive/connectors/onlyoffice-connector/src/middlewares/auth.middleware.ts +++ b/tdrive/connectors/onlyoffice-connector/src/middlewares/auth.middleware.ts @@ -34,6 +34,21 @@ export default async (req: Request<{}, {}, {}, RequestQuery>, res: Response, nex return res.status(401).send({ message: 'invalid token' }); } + const authHeaderToken = req.header('authorization'); + try { + if (authHeaderToken) { + const fromTwakeDriveToken = jwt.verify(authHeaderToken, CREDENTIALS_SECRET) as Record; + // The following constant comes from tdrive/backend/node/src/services/applications-api/index.ts + // This is in the case when Twake Drive backend makes requests from the connector, + // but not on the behalf of a specific user, eg. when checking stale only office keys + if (fromTwakeDriveToken['type'] != 'tdriveToApplication') throw new Error(`Bad type in token ${JSON.stringify(fromTwakeDriveToken)}`); + return next(); + } + } catch (e) { + logger.error(`Invalid token in Authorization header from Twake Drive backend`, e); + return res.status(401).json({ message: 'invalid token' }); + } + const user = await userService.getCurrentUser(token); if (!user || !user.id) { diff --git a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts index ceb298092..e03576635 100644 --- a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts +++ b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts @@ -4,11 +4,14 @@ import { NextFunction, Request, Response } from 'express'; export default (error: Error & { status?: number }, req: Request, res: Response, next: NextFunction): void => { try { const status: number = error.status || 500; - const message: string = error.message || 'something went wrong'; + const message: string = error.message || 'Something went wrong'; logger.error(`[${req.method}] ${req.path} >> StatusCode:: ${status}, Message:: ${message}`, error.stack); - res.status(status).json({ message }); + res.status(status); + res.render('error', { + errorMessage: message, + }); } catch (error) { next(error); } diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts new file mode 100644 index 000000000..f2f85a395 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts @@ -0,0 +1,16 @@ +import TwakeDriveBackendCallbackController from '@/controllers/backend-callbacks.controller'; +import authMiddleware from '@/middlewares/auth.middleware'; +import type { Router } from 'express'; + +/** + * These routes are called by Twake Drive backend, for ex. before editing or retreiving a file, + * if it has an editing_session_key still, get the status of that and force a resolution. + */ +export const TwakeDriveBackendCallbackRoutes = { + mount(router: Router) { + const controller = new TwakeDriveBackendCallbackController(); + // Why post ? to garantee it is never cached and always ran + router.post('/session/:editing_session_key/check', authMiddleware, controller.checkSessionStatus.bind(controller)); + router.post('/session/:editing_session_key/title', authMiddleware, controller.updateSessionFilename.bind(controller)); + }, +}; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts new file mode 100644 index 000000000..f1e1a158c --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts @@ -0,0 +1,16 @@ +import BrowserEditorController from '@/controllers/browser-editor.controller'; +import authMiddleware from '@/middlewares/auth.middleware'; +import requirementsMiddleware from '@/middlewares/requirements.middleware'; +import type { Router } from 'express'; + +/** + * When the user previews or edits a file in Twake Drive, their browser is sent to these routes + * which return a webpage that instantiates the client side JS Only Office component. + */ +export const BrowserEditorRoutes = { + mount(router: Router) { + const controller = new BrowserEditorController(); + router.get('/', requirementsMiddleware, authMiddleware, controller.index.bind(controller)); + router.get('/editor', requirementsMiddleware, authMiddleware, controller.editor.bind(controller)); + }, +}; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts deleted file mode 100644 index 20fb86111..000000000 --- a/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts +++ /dev/null @@ -1,27 +0,0 @@ -import IndexController from '@/controllers/index.controller'; -import { Routes } from '@/interfaces/routes.interface'; -import authMiddleware from '@/middlewares/auth.middleware'; -import requirementsMiddleware from '@/middlewares/requirements.middleware'; -import { Router } from 'express'; - -/** - * When the user previews or edits a file in Twake Drive, their browser is sent to these routes - * which return a webpage that instantiates the client side JS Only Office component. - */ -class IndexRoute implements Routes { - public path = '/'; - public router = Router(); - public indexController: IndexController; - - constructor() { - this.indexController = new IndexController(); - this.initRoutes(); - } - - private initRoutes = () => { - this.router.get(this.path, requirementsMiddleware, authMiddleware, this.indexController.index); - this.router.get(this.path + 'editor', requirementsMiddleware, authMiddleware, this.indexController.editor); - }; -} - -export default IndexRoute; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/index.ts b/tdrive/connectors/onlyoffice-connector/src/routes/index.ts new file mode 100644 index 000000000..c930d71aa --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/routes/index.ts @@ -0,0 +1,55 @@ +import * as path from 'path'; +import { static as expressStatic, Application, Router } from 'express'; + +import * as Utils from '@/utils'; +import { SERVER_ORIGIN, SERVER_PREFIX } from '@config'; + +import { HealthStatusController } from '@/controllers/health-status.controller'; +import { TwakeDriveBackendCallbackRoutes } from './backend-callbacks.route'; +import { BrowserEditorRoutes } from './browser-editor.route'; +import { OnlyOfficeRoutes } from './onlyoffice.route'; + +function mountAssetsRoutes(router: Router) { + router.use( + '/assets', + (req, res, next) => { + res.header('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Headers', 'X-Requested-With'); + next(); + }, + expressStatic(path.join(__dirname, '../../assets')), + ); +} + +export function mountRoutes(app: Application) { + // Technical routes for ops. Should not be exposed. + app.get('/health', HealthStatusController.get); + + // These routes are forwarded through the Twake Drive front, back and here + const proxiedRouter = Router(); + BrowserEditorRoutes.mount(proxiedRouter); + OnlyOfficeRoutes.mount(proxiedRouter); + mountAssetsRoutes(proxiedRouter); + app.use(SERVER_PREFIX /* usually "plugins/onlyoffice" */, proxiedRouter); + + // These endpoints should only be accessible to the Twake Drive backend + const apiRouter = Router(); + TwakeDriveBackendCallbackRoutes.mount(apiRouter); + app.use('/tdriveApi/1', apiRouter); +} + +export const makeURLTo = { + rootAbsolute: () => Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), + assets: () => Utils.joinURL([SERVER_PREFIX, 'assets']), + editorAbsolute(params: { + token: string; + drive_file_id: string; + file_id: string; + editing_session_key: string; + company_id: string; + preview: string; + office_token: string; + }) { + return Utils.joinURL([SERVER_ORIGIN ?? '', SERVER_PREFIX, 'editor'], params); + }, +}; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts index 6bc3c879b..921157cd0 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts @@ -1,22 +1,16 @@ import OnlyOfficeController from '@/controllers/onlyoffice.controller'; -import { Routes } from '@/interfaces/routes.interface'; import requirementsMiddleware from '@/middlewares/requirements.middleware'; -import { Router } from 'express'; +import type { Router } from 'express'; -class OnlyOfficeRoute implements Routes { - public path = '/'; - public router = Router(); - public onlyOfficeController: OnlyOfficeController; - - constructor() { - this.onlyOfficeController = new OnlyOfficeController(); - this.initRoutes(); - } - - private initRoutes = () => { - this.router.get(`${this.path}:mode/read`, requirementsMiddleware, this.onlyOfficeController.read); - this.router.post(`${this.path}:mode/save`, requirementsMiddleware, this.onlyOfficeController.ooCallback); - }; -} - -export default OnlyOfficeRoute; +/** + * These routes are called by the Only Office server + * when reading a document or updating an editing session + */ +export const OnlyOfficeRoutes = { + mount(router: Router) { + const controller = new OnlyOfficeController(); + router.get(`/:mode/read`, requirementsMiddleware, controller.read.bind(controller)); + router.post(`/:mode/callback`, requirementsMiddleware, controller.ooCallback.bind(controller)); + router.post(`/:mode/rename`, requirementsMiddleware, controller.rename.bind(controller)); + }, +}; diff --git a/tdrive/connectors/onlyoffice-connector/src/server.ts b/tdrive/connectors/onlyoffice-connector/src/server.ts index 139be10f2..f743c935d 100644 --- a/tdrive/connectors/onlyoffice-connector/src/server.ts +++ b/tdrive/connectors/onlyoffice-connector/src/server.ts @@ -1,7 +1,5 @@ import App from '@/app'; -import IndexRoute from './routes/index.route'; -import OnlyOfficeRoute from './routes/onlyoffice.route'; -const app = new App([new IndexRoute(), new OnlyOfficeRoute()]); +const app = new App(); app.listen(); diff --git a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts index fd7bd96ed..92a75c3d9 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts @@ -5,28 +5,34 @@ import { IApiServiceApplicationTokenResponse, } from '@/interfaces/api.interface'; import axios, { Axios, AxiosRequestConfig, AxiosResponse } from 'axios'; -import { CREDENTIALS_ENDPOINT, CREDENTIALS_ID, CREDENTIALS_SECRET } from '@config'; +import { CREDENTIALS_ENDPOINT, CREDENTIALS_ID, CREDENTIALS_SECRET, twakeDriveTokenRefrehPeriodMS } from '@config'; import logger from '../lib/logger'; import * as Utils from '@/utils'; import { PolledThingieValue } from '@/lib/polled-thingie-value'; +import { IHealthProvider, registerHealthProvider } from './health-providers.service'; /** * Client for the Twake Drive backend API on behalf of the plugin (or provided token in parameters). * Periodically updates authorization and adds to requests. */ -class ApiService implements IApiService { - private readonly poller: PolledThingieValue; +class ApiService implements IApiService, IHealthProvider { + private readonly tokenPoller: PolledThingieValue; constructor() { - this.poller = new PolledThingieValue('Refresh Twake Drive token', async () => this.refreshToken(), 1000 * 60); //TODO: should be Every 10 minutes + this.tokenPoller = new PolledThingieValue('Refresh Twake Drive token', async () => await this.refreshToken(), twakeDriveTokenRefrehPeriodMS); + registerHealthProvider(this); } public async hasToken() { - return (await this.poller.latestValueWithTry()) !== undefined; + return (await this.tokenPoller.latestValueWithTry()) !== undefined; + } + + async getHealthData() { + return { TwakeDriveApi: { tokenAgeS: this.tokenPoller.latest()?.ageS ?? -1 } }; } private requireAxios() { - return this.poller.requireLatestValueWithTry('Token Kind 538 not ready'); + return this.tokenPoller.requireLatestValueWithTry('No Twake Drive app token.'); } public get = async (params: IApiServiceRequestParams): Promise => { @@ -49,20 +55,43 @@ class ApiService implements IApiService { return await axiosWithToken.get(url, config); }; + public delete = async (params: IApiServiceRequestParams): Promise => { + const { url, token, responseType, headers } = params; + + const axiosWithToken = await this.requireAxios(); + + const config: AxiosRequestConfig = {}; + + if (token) { + config['headers'] = { + Authorization: `Bearer ${token}`, + ...headers, + }; + } + + if (responseType) { + config['responseType'] = responseType; + } + return await axiosWithToken.delete(url, config); + }; + public post = async (params: IApiServiceRequestParams): Promise => { - const { url, payload, headers } = params; + const { url, token, payload, headers } = params; const axiosWithToken = await this.requireAxios(); + logger.info(`POST to Twake Drive ${url} - payload: ${payload}`); try { return await axiosWithToken.post(url, payload, { headers: { ...headers, + ...(token ? { Authorization: `Bearer ${token}` } : {}), }, }); } catch (error) { - logger.error('Failed to post to Twake drive: ', error.stack); + logger.error('Failed to post to Twake Drive: ', { error: error.stack }); this.refreshToken(); + throw error; } }; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index c070ceb91..ebb2bf658 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -1,8 +1,22 @@ import { DriveFileType, IDriveService } from '@/interfaces/drive.interface'; import apiService from './api.service'; import logger from '../lib/logger'; +import * as Utils from '../utils'; +import { Stream } from 'stream'; +import FormData from 'form-data'; +import { INSTANCE_ID } from '@/config'; -/** Client for Twake Drive's APIs dealing with `DriveItem`s, using {@see apiService} +/** Thrown when Twake Drive returns a 404 for a key */ +export class UnknownKeyInDriveError extends Error {} + +const makeTwakeDriveApiUrl = (paths: string[], params?: Utils.QueryParams) => Utils.joinURL(['/internal/services/documents/v1', ...paths], params); +const makeNonEditingSessionItemUrl = (company_id: string, drive_file_id: string, paths: string[] = [], params?: Utils.QueryParams) => + makeTwakeDriveApiUrl(['companies', company_id, 'item', drive_file_id, ...paths], params); +const makeEditingSessionItemUrl = (editing_session: string, params?: Utils.QueryParams) => + makeTwakeDriveApiUrl(['editing_session', editing_session], params); + +/** + * Client for Twake Drive's APIs dealing with `DriveItem`s, using {@see apiService} * to handle authorization */ class DriveService implements IDriveService { @@ -10,7 +24,7 @@ class DriveService implements IDriveService { try { const { company_id, drive_file_id } = params; const resource = await apiService.get({ - url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}`, + url: makeNonEditingSessionItemUrl(company_id, drive_file_id), token: params.user_token, }); @@ -22,6 +36,28 @@ class DriveService implements IDriveService { } }; + public update = async (params: { + company_id: string; + drive_file_id: string; + changes: Partial; + completeResult?: boolean; + }): Promise<(typeof params)['changes']> => { + try { + const { company_id, drive_file_id } = params; + const resource = await apiService.post<(typeof params)['changes'], ReturnType>({ + url: makeNonEditingSessionItemUrl(company_id, drive_file_id), + payload: params.changes, + }); + if (params.completeResult) return resource; + const result = {}; + Object.keys(params.changes).forEach(k => (result[k] = resource[k])); + return result; + } catch (error) { + logger.error('Failed to update file metadata: ', error.stack); + return Promise.reject(); + } + }; + public createVersion = async (params: { company_id: string; drive_file_id: string; @@ -29,8 +65,8 @@ class DriveService implements IDriveService { }): Promise => { try { const { company_id, drive_file_id, file_id } = params; - const resource = await apiService.post<{}, DriveFileType['item']['last_version_cache']>({ - url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}/version`, + return await apiService.post<{}, DriveFileType['item']['last_version_cache']>({ + url: makeNonEditingSessionItemUrl(company_id, drive_file_id, ['version']), payload: { drive_item_id: drive_file_id, provider: 'internal', @@ -40,13 +76,90 @@ class DriveService implements IDriveService { }, }, }); - - return resource; } catch (error) { logger.error('Failed to create version: ', error.stack); return Promise.reject(); } }; + + public async beginEditingSession(company_id: string, drive_file_id: string, user_token?: string) { + try { + const resource = await apiService.post<{}, { editingSessionKey: string }>({ + url: makeNonEditingSessionItemUrl(company_id, drive_file_id, ['editing_session']), + token: user_token, + payload: { + editorApplicationId: 'tdrive_onlyoffice', + appInstanceId: INSTANCE_ID ?? '', + }, + }); + if (resource?.editingSessionKey) { + return resource.editingSessionKey; + } else { + throw new Error(`Failed to obtain editing session key, response: ${JSON.stringify(resource)}`); + } + } catch (error) { + logger.error('Failed to begin editing session: ', error.stack); + throw error; + } + } + + public async cancelEditing(editing_session_key: string) { + try { + await apiService.delete<{}>({ + url: makeEditingSessionItemUrl(editing_session_key), + }); + } catch (error) { + logger.error('Failed to begin editing session: ', error.stack); + throw error; + //TODO make monitoring for such kind of errors + } + } + + public async addEditingSessionVersion(editing_session_key: string, url: string, userId?: string) { + return this.updateEditing(editing_session_key, url, true, userId); + } + + public async endEditing(editing_session_key: string, url: string, userId?: string) { + return this.updateEditing(editing_session_key, url, false, userId); + } + + private async updateEditing(editing_session_key: string, url: string, keepEditing: boolean, userId?: string) { + try { + if (!url) { + throw Error('no url found'); + } + + const newFile = await apiService.get({ + url, + responseType: 'stream', + }); + + const form = new FormData(); + + form.append('file', newFile); + + logger.info(`Saving file version to Twake Drive`, { editing_session_key, url }); + + await apiService.post({ + url: makeEditingSessionItemUrl(editing_session_key, { + keepEditing: keepEditing ? 'true' : null, + userId, + }), + payload: form, + headers: form.getHeaders(), + }); + } catch (error) { + if (error.response?.status === 404) { + logger.error('Forgotten OO document that Twake Drive doesnt know the key of.', { editing_session_key, url }); + throw new UnknownKeyInDriveError(`Unknown key ${JSON.stringify(editing_session_key)}`); + //TODO: Distinguish this case from a long disconnected browser tab waking up + //TODO make monitoring for such kind of errors + } else { + logger.error('Failed to end editing session: ', error.stack); + throw error; + } + } + } } export default new DriveService(); diff --git a/tdrive/connectors/onlyoffice-connector/src/services/editor.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/editor.service.ts index 5cd82f719..12794a0a1 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/editor.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/editor.service.ts @@ -10,7 +10,7 @@ class EditorService implements IEditorService { file_version_id: string, user: UserType, preview: boolean, - file_id: string, + drive_file_id: string, ): Promise => { const { color, mode: fileMode } = this.getFileMode(file_name); let [, extension] = Utils.splitFilename(file_name); @@ -18,7 +18,7 @@ class EditorService implements IEditorService { extension = extension.toLocaleLowerCase(); return { color, - file_id, + drive_file_id, file_version_id, file_type: extension, filename: file_name, diff --git a/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts new file mode 100644 index 000000000..de47e632c --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts @@ -0,0 +1,91 @@ +import logger from '@/lib/logger'; +import { onlyOfficeForgottenFilesCheckPeriodMS } from '@/config'; +import { PolledThingieValue } from '@/lib/polled-thingie-value'; +import { createSingleProcessorLock } from '@/lib/single-processor-lock'; + +import apiService from './api.service'; +import onlyofficeService from './onlyoffice.service'; +import driveService, { UnknownKeyInDriveError } from './drive.service'; +import { IHealthProvider, registerHealthProvider } from './health-providers.service'; + +/** + * Periodically poll the Only Office document server for forgotten + * files and try to upload to Twake Drive or get rid if unknown. + */ +class ForgottenProcessor implements IHealthProvider { + private readonly forgottenFilesPoller: PolledThingieValue; + public readonly forgottenSynchroniser = createSingleProcessorLock(); + private lastStart = 0; + + constructor() { + let skippedFirst = false; + this.forgottenFilesPoller = new PolledThingieValue( + 'Process forgotten files in OO', + async () => (skippedFirst ? await this.processForgottenFiles() : ((skippedFirst = true), -1)), + onlyOfficeForgottenFilesCheckPeriodMS, + ); + registerHealthProvider(this); + } + + public async getHealthData() { + const keys = await onlyofficeService.getForgottenList(); + return { + forgotten: { + timeSinceLastStartS: this.lastStart ? ~~((new Date().getTime() - this.lastStart) / 1000) : -1, + count: keys?.length ?? 0, + locks: this.forgottenSynchroniser.getWorstStats(), + }, + }; + } + + /** + * The point of this is to ensure this file is imported, + * which is needed for side-effect of starting this timer. + * The only other use being the health stuff it could easily + * be refactored out as unused. + */ + public makeSureItsLoaded() { + return 'yup this module is loaded !'; + } + + /** + * Try to upload the forgotten file, optionally delete it from OO, will return if it was + * (or should be) deleted. Does not throw unless the OO deletion itself threw. + */ + private async safeEndEditing(key: string, url: string, deleteForgotten: boolean) { + return this.forgottenSynchroniser.runWithLock(key, async () => { + let succeded = false; + try { + await driveService.endEditing(key, url); + succeded = true; + } catch (error) { + if (!(error instanceof UnknownKeyInDriveError)) + logger.error(`Error processing forgotten file by key ${JSON.stringify(key)}: ${error}`, { key, url, error }); + // Can't do much about it here, hope it goes in retry, but don't + // throw to keep processing + //TODO: Maybe make a date string, compare to key, if old enough, delete... + // this logic should probably in Twake Drive backend though + } + if (succeded && deleteForgotten) await onlyofficeService.deleteForgotten(key); + return succeded; + }); + } + + private async processForgottenFiles() { + this.lastStart = new Date().getTime(); + if (!(await apiService.hasToken())) return -1; + return await onlyofficeService.processForgotten((key, url) => + this.safeEndEditing(key, url, false /* `onlyofficeService.processForgotten` will do it */), + ); + } + + /** + * Attempt to upload the forgotten OO file, only throws if deleting it failed, + * returns wether it was deleted. + */ + public async processForgottenFile(key: string, url: string) { + return this.safeEndEditing(key, url, true); + } +} + +export default new ForgottenProcessor(); diff --git a/tdrive/connectors/onlyoffice-connector/src/services/health-providers.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/health-providers.service.ts new file mode 100644 index 000000000..da85ff623 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/services/health-providers.service.ts @@ -0,0 +1,13 @@ +const providers: IHealthProvider[] = []; + +export interface IHealthProvider { + getHealthData(): Promise<{ [key: string]: unknown }>; +} + +export function registerHealthProvider(provider: IHealthProvider) { + providers.push(provider); +} + +export function getProviders() { + return providers; +} diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index 72a9a161d..425cf6932 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -1,8 +1,11 @@ +import { randomUUID } from 'crypto'; import axios from 'axios'; -import { ONLY_OFFICE_SERVER } from '@config'; +import { ONLY_OFFICE_SERVER, onlyOfficeConnectivityCheckPeriodMS, onlyOfficeCallbackTimeoutMS } from '@config'; import { PolledThingieValue } from '@/lib/polled-thingie-value'; +import { PendingRequestQueue, PendingRequestCallback } from '@/lib/pending-request-matcher'; import logger from '@/lib/logger'; import * as Utils from '@/utils'; +import { IHealthProvider, registerHealthProvider } from './health-providers.service'; /** @see https://api.onlyoffice.com/editors/basic */ export enum ErrorCode { @@ -14,6 +17,12 @@ export enum ErrorCode { COMMAND_NOT_CORRECT = 5, INVALID_TOKEN = 6, } + +/** Error generated by this connector. Should be string to share field with {@link ErrorCode} */ +export enum ConnectorErrorCode { + CALLBACK_TIMEOUT = 'callback_timeout', +} + /** Return the name of the error code in the `ErrorCode` enum if recognised, or a descript string */ export const ErrorCodeFromValue = (value: number) => Utils.getKeyForValueSafe(value, ErrorCode, 'OnlyOffice.ErrorCode'); @@ -24,6 +33,12 @@ export namespace Callback { USER_CONNECTED = 1, USER_INITIATED_FORCE_SAVE = 2, } + export const ActionTypeToString = (value: number) => Utils.getKeyForValueSafe(value, ActionType, 'OnlyOffice.Callback.ActionType'); + + interface Action { + type: ActionType; + userid: string; + } enum ForceSaveType { FROM_COMMAND_SERVICE = 0, @@ -31,6 +46,7 @@ export namespace Callback { SERVER_TIMER = 2, FORM_SUBMITTED = 3, } + export const ForceSaveTypeToString = (value: number) => Utils.getKeyForValueSafe(value, ForceSaveType, 'OnlyOffice.Callback.ForceSaveType'); export enum Status { BEING_EDITED = 1, @@ -44,11 +60,8 @@ export namespace Callback { /** `url` and `forcesavetype` fields present with this status */ ERROR_FORCE_SAVING = 7, } + export const StatusToString = (value: number) => Utils.getKeyForValueSafe(value, Status, 'OnlyOffice.Callback.Status'); - interface Action { - type: ActionType; - userid: string; - } /** Parameters given to the callback by the editing service */ export interface Parameters { key: string; @@ -58,6 +71,16 @@ export namespace Callback { url?: string; actions?: Action[]; users?: string[]; + userdata?: string; + } +} + +/** For error responses from the {@see CommandService} */ +export class CommandError extends Error { + constructor(public readonly errorCode: ErrorCode, req: any, res: any) { + super( + `OnlyOffice command service error ${ErrorCodeFromValue(errorCode)} (${errorCode}): Requested ${JSON.stringify(req)} got ${JSON.stringify(res)}`, + ); } } @@ -76,28 +99,18 @@ namespace CommandService { error: Exclude; } - export class CommandError extends Error { - constructor(errorCode: ErrorCode, req: any, res: any) { - super( - `OnlyOffice command service error ${ErrorCodeFromValue(errorCode)} (${errorCode}): Requested ${JSON.stringify(req)} got ${JSON.stringify( - res, - )}`, - ); - } - } - - abstract class BaseRequest { - constructor(public c: string) {} + abstract class BaseRequest { + constructor(public readonly c: string) {} /** POST this OnlyOffice command, does not check the `error` field of the response */ async postUnsafe(): Promise { logger.silly(`OnlyOffice command ${this.c} sent: ${JSON.stringify(this)}`); - const result = await axios.post(`${ONLY_OFFICE_SERVER}coauthoring/CommandService.ashx`, this); - logger.info(`OnlyOffice command ${this.c} response: ${result.status}: ${JSON.stringify(result.data)}`); + const result = await axios.post(Utils.joinURL([ONLY_OFFICE_SERVER, 'coauthoring/CommandService.ashx']), this); + logger.info(`OnlyOffice command ${this.c} response ${result.status}: ${ErrorCodeFromValue(result.data.error)}: ${JSON.stringify(result.data)}`); return result.data as ErrorResponse | TSuccessResponse; } - /** POST this request, and return the result, or throw if the `errorCode` returned isn't 0 */ + /** POST this request, and return the result, or throws if the `errorCode` returned isn't `ErrorCode.SUCCESS` */ async post(): Promise { const result = await this.postUnsafe(); if (result.error === ErrorCode.SUCCESS) return result; @@ -121,7 +134,7 @@ namespace CommandService { key: string; } export class Request extends BaseRequest { - constructor(public key: string, public userdata: string = '') { + constructor(public readonly key: string, public readonly userdata: string = '') { super('forcesave'); } } @@ -133,7 +146,7 @@ namespace CommandService { url: string; } export class Request extends BaseRequest { - constructor(public key: string) { + constructor(public readonly key: string) { super('getForgotten'); } } @@ -155,30 +168,188 @@ namespace CommandService { key: string; } export class Request extends BaseRequest { - constructor(public key: string) { + constructor(public readonly key: string) { super('deleteForgotten'); } } } + + export namespace License { + export interface Response extends SuccessResponse { + // @see https://api.onlyoffice.com/editors/command/license + license: object; + server: object; + quota: object; + } + export class Request extends BaseRequest { + constructor() { + super('license'); + } + } + } + + export namespace Info { + export type Response = SuccessResponse; + export class Request extends BaseRequest { + constructor(public readonly key: string, public readonly userdata: string = '') { + super('info'); + } + } + } + + export namespace Meta { + export interface Response extends SuccessResponse { + key: string; + } + export class Request extends BaseRequest { + constructor(public readonly key: string, public readonly meta: { title: string }) { + super('meta'); + } + } + } +} + +/** + * This object holds possible outcomes for commands like `info` who's result + * is sent to the callback instead of replied to the request + */ +class CallbackResponseFromCommand { + /** + * If the `info` command returned an error code, this is it, or it can + * be an internal error to this connector (eg.: timeout waiting for callback). + * (If the `info` command returned {@link ErrorCode.SUCCESS} then this field will be `undefined`) + */ + public readonly error: Exclude | ConnectorErrorCode | undefined; + + public constructor(error: ErrorCode | ConnectorErrorCode, public readonly result?: Callback.Parameters) { + this.error = error === ErrorCode.SUCCESS ? undefined : error; + } } /** * Exposed OnlyOffice command service * @see https://api.onlyoffice.com/editors/command/ */ -class OnlyOfficeService { - private readonly poller: PolledThingieValue; +class OnlyOfficeService implements IHealthProvider { + private readonly poller: PolledThingieValue; + // Technically the timeout field is from the PendingRequestQueue but avoid 2 classes + private readonly pendingRequests = new PendingRequestQueue(onlyOfficeCallbackTimeoutMS); + constructor() { - this.poller = new PolledThingieValue('Connect to Only Office', () => this.getVersion(), 10 * 1000 * 60); + this.poller = new PolledThingieValue( + 'Connect to Only Office', + async () => { + logger.info('Only Office license status'); + return await this.getLicense(); + }, + onlyOfficeConnectivityCheckPeriodMS, + ); + registerHealthProvider(this); } - /** Get the latest Only Office version */ - public getLatestVersion() { + /** Get the latest Only Office licence status from polling. If the return is `undefined` + * it probably means there is a connection issue contacting the OnlyOffice server + * from the connector. + */ + public getLatestLicence() { return this.poller.latest(); } + + /** + * Iterate over all the forgotten files as returned by OnlyOffice, call the processor for each.. + * @param processor Handler to process the forgotten file (available at `url`). If `true` is returned, + * the file is deleted from the forgotten file list in OnlyOffice. If false is returned, the + * same forgotten file will reappear in a future batch + * @returns The number of files processed and deleted + */ + public async processForgotten(processor: (key: string, url: string) => Promise): Promise { + logger.info(`Begin to process forgotten files in OnlyOffice`); + //TODO: filter by instance id of the key + const forgottenFiles = await this.getForgottenList(); + if (forgottenFiles.length === 0) { + logger.info(`No forgotten files in OnlyOffice`); + return 0; + } + Utils.fisherYattesShuffleInPlace(forgottenFiles); + logger.info(`Forgotten files found: ${forgottenFiles.length}`); + let deleted = 0; + for (const forgottenFileKey of forgottenFiles) { + const forgottenFileURL = await this.getForgotten(forgottenFileKey); + logger.info(`Forgotten file about to process: ${JSON.stringify(forgottenFileKey)}`, { url: forgottenFileURL }); + const shouldDelete = await processor(forgottenFileKey, forgottenFileURL); + if (shouldDelete) { + logger.info(`Forgotten file about to be deleted: ${JSON.stringify(forgottenFileKey)}`); + await this.deleteForgotten(forgottenFileKey); + deleted++; + } + } + return deleted; + } + + /** Generates and returns a random UUID that has a matching pending task enqueued for */ + private enqueuePendingCallback(key: string, callback: PendingRequestCallback): string { + const userdata = randomUUID(); + this.pendingRequests.enqueue(key, userdata, callback); + return userdata; + } + + /** Called by the OnlyOffice controller when the OO document editing services uses our callback */ + async ooCallbackCalled(result: Callback.Parameters) { + if (!result.userdata) return; + logger.info('OO Callback pending request response received', result); + return this.pendingRequests.gotResult(result.key, result.userdata, new CallbackResponseFromCommand(ErrorCode.SUCCESS, result)); + } + + // Note that `async` is important in the functions below. While they avoid the overhead + // of `await`, the `async` is still required to catch the throw in `.post()` + /** Return the version string of OnlyOffice */ async getVersion(): Promise { return new CommandService.Version.Request().post().then(response => response.version); } + /** Return the version string of OnlyOffice */ + async getLicense(): Promise { + //TODO: When typing the response more fully, don't return the response object itself as here + return new CommandService.License.Request().post(); + } + + async getHealthData() { + return { OO: { license: this.poller.latest() } }; + } + + /** + * Requests a document status and the list of the identifiers of the users who opened the document for editing. + * The response will be sent to the callback handler. + * This method just sends the command. The response from the callback will be ignored if + * this called by itself + * + * *Warning*: returns non succesful error codes instead of throwing errors like the other + * methods. This is because the immediate response is likely to be itself usefull + * for detecting errors. + */ + async getInfoUnsafe(key: string, userdata: string): Promise { + return await new CommandService.Info.Request(key, userdata).postUnsafe().then(({ error }) => error); + } + + /** Send the info command, wait for the callback if warranted, and return the error or callback body */ + async getInfoAndWaitForCallbackUnsafe(key: string): Promise { + // const userdata = randomUUID(); + return new Promise((resolve, reject) => { + const userdata = this.enqueuePendingCallback(key, async (timeout, result) => { + // The callback has called, unless timeout = true, or result is undefined (cancelled request) + return resolve(timeout ? new CallbackResponseFromCommand(ConnectorErrorCode.CALLBACK_TIMEOUT) : result); + }); + void this.getInfoUnsafe(key, userdata).then( + response => { + // The command service responded to the `info` command request + if (response !== ErrorCode.SUCCESS) return this.pendingRequests.gotResult(key, userdata, new CallbackResponseFromCommand(response)); + // If it succeded, just wait for timeout or resolution by the callback + }, + error => { + this.pendingRequests.cancelPending(key, userdata).then(() => reject(error)); + }, + ); + }); + } /** Force a save in the editing session key provided. `userdata` will be forwarded to the callback */ async forceSave(key: string, userdata = ''): Promise { return new CommandService.ForceSave.Request(key, userdata).post().then(response => response.key); @@ -195,6 +366,14 @@ class OnlyOfficeService { async deleteForgotten(key: string): Promise { return new CommandService.DeleteForgotten.Request(key).post().then(response => response.key); } + /** + * Updates the meta information of the document for all collaborative editors. + * + * That's the official description. It send file renames to OO. + */ + async meta(key: string, title: string): Promise { + return new CommandService.Meta.Request(key, { title }).post().then(response => response.key); + } } export default new OnlyOfficeService(); diff --git a/tdrive/connectors/onlyoffice-connector/src/utils.ts b/tdrive/connectors/onlyoffice-connector/src/utils.ts index 094781722..36b731400 100644 --- a/tdrive/connectors/onlyoffice-connector/src/utils.ts +++ b/tdrive/connectors/onlyoffice-connector/src/utils.ts @@ -16,7 +16,7 @@ export type QueryParams = { [key: string]: string | number }; export function joinURL(path: string[], params?: QueryParams) { let joinedPath = path.map(x => x.replace(/(?:^\/+)+|(?:\/+$)/g, '')).join('/'); if (path[path.length - 1].endsWith('/')) joinedPath += '/'; - const paramEntries = Object.entries(params || {}); + const paramEntries = Object.entries(params || {}).filter(([, value]) => value !== undefined && value !== null); if (paramEntries.length === 0) return joinedPath; const query = paramEntries.map(p => p.map(encodeURIComponent).join('=')).join('&'); return joinedPath + (joinedPath.indexOf('?') > -1 ? '&' : '?') + query; @@ -44,3 +44,14 @@ export function splitFilename(filename: string): [string, string] { const extension = parts.pop(); return [parts.join('.'), extension]; } + +/** Shuffle an array in place, returns its parameter for convenience */ +export function fisherYattesShuffleInPlace(list: T[]): T[] { + let index = list.length; + while (index) { + const randomIndex = Math.floor(Math.random() * index); + index--; + [list[index], list[randomIndex]] = [list[randomIndex], list[index]]; + } + return list; +} diff --git a/tdrive/connectors/onlyoffice-connector/src/views/error.eta b/tdrive/connectors/onlyoffice-connector/src/views/error.eta new file mode 100644 index 000000000..8864a3e90 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/views/error.eta @@ -0,0 +1,212 @@ + + + + + + + + 500 Internal Server Error + + + + + + + +
+
+
+

500

+
+

We are sorry, internal server error!

+

Please, contact our support at :

+ support@twake.app +
+
+ + + diff --git a/tdrive/connectors/onlyoffice-connector/src/views/index.eta b/tdrive/connectors/onlyoffice-connector/src/views/index.eta index a88e8012e..fa1c3fd43 100644 --- a/tdrive/connectors/onlyoffice-connector/src/views/index.eta +++ b/tdrive/connectors/onlyoffice-connector/src/views/index.eta @@ -5,6 +5,7 @@ + Twake Drive @@ -22,18 +23,24 @@ $('#onlyoffice_container').html("
"); + const callbackQueryString = '?drive_file_id=<%= it.drive_file_id %>&company_id=<%= it.company_id %>&token=<%= it.token %>'; let doc = { title: "<%= it.filename %>", - url: `${window.baseURL}read?file_id=<%= it.file_id %>&company_id=<%= it.company_id %>&token=<%= it.token %>`, + url: `${window.baseURL}read${callbackQueryString}`, fileType: "<%= it.file_type %>", - key: "<%= it.file_version_id %>", - token: "<%= it.file_id %>", + key: "<%= it.docId %>", + token: "<%= it.drive_file_id %>", permissions: { download: true, edit: <%= it.editable %>, preview: <%= it.preview %>, } } + function updateTitle(title) { + const el = document.querySelector('head title'); + if (el) el.innerText = title + ' — Twake Drive'; + } + updateTitle(doc.title); window.docEditor = new DocsAPI.DocEditor('onlyoffice_container_instance', { scrollSensitivity: window.mode === 'text' ? 100 : 40, @@ -41,10 +48,28 @@ height: '100%', documentType: window.mode, document: doc, - token: "<%= it.file_id %>", + token: "<%= it.drive_file_id %>", type: screen.width < 600 ? 'mobile' : 'desktop', + events: { + onRequestRename: function (event) { + let name = event.data; + const prevExtension = /\.[^.]+$/.exec(doc.title); + if (prevExtension) name += prevExtension[0]; + $. + post(`${window.baseURL}rename${callbackQueryString}`, { name }). + done((changed) => { + if (changed.name != name) + window.docEditor.showMessage(`✅ Renamed to: ${changed.name}`); + }). + fail(() => window.docEditor.showMessage('🚨 Error renaming the file')); + }, + onMetaChange: function (event) { + const { title } = event.data; + if (title) updateTitle(title); + }, + }, editorConfig: { - callbackUrl: `${window.baseURL}save?file_id=<%= it.file_id %>&company_id=<%= it.company_id %>&token=<%= it.token %>`, + callbackUrl: `${window.baseURL}callback${callbackQueryString}`, lang: window.user.language, user: { id: window.user.id, @@ -52,7 +77,7 @@ }, customization: { chat: false, - compactToolbar: true, + compactToolbar: false, about: false, feedback: false, goback: { diff --git a/tdrive/docker-compose.dev.onlyoffice.yml b/tdrive/docker-compose.dev.onlyoffice.yml new file mode 100644 index 000000000..29b0380e9 --- /dev/null +++ b/tdrive/docker-compose.dev.onlyoffice.yml @@ -0,0 +1,62 @@ +version: "3.4" + +services: + + onlyoffice-rabbitmq: + image: rabbitmq:management + hostname: onlyoffice-rabbitmq + container_name: rabbitmq + environment: + - RABBITMQ_DEFAULT_USER=guest + - RABBITMQ_DEFAULT_PASS=guest + ports: + - "5672:5672" + - "15672:15672" + volumes: + - ./.docker-conf/rabbitmq/data/:/var/lib/rabbitmq/ + - ./.docker-conf/rabbitmq/log/:/var/log/rabbitmq + networks: + - tdrive_network + + onlyoffice-postgresql: + image: postgres:13 + hostname: onlyoffice-postgresql + environment: + - POSTGRES_DB=onlyoffice + - POSTGRES_USER=onlyoffice + - POSTGRES_PASSWORD=onlyoffice + ports: + - 5432:5432 + volumes: + - ./onlyoffice_postgres_data:/var/lib/postgresql/data + networks: + - tdrive_network + + onlyoffice: + image: docker.io/onlyoffice/documentserver + hostname: onlyoffice + ports: + - 8090:80 + networks: + - tdrive_network + environment: + - AMQP_URI=amqp://guest:guest@onlyoffice-rabbitmq + - DB_HOST=onlyoffice-postgresql + - DB_NAME=onlyoffice + - DB_PORT=5432 + - DB_TYPE=postgres + - DB_USER=onlyoffice + - JWT_ENABLED=false + - ALLOW_META_IP_ADDRESS=true + - ALLOW_PRIVATE_IP_ADDRESS=true + depends_on: + - onlyoffice-rabbitmq + - onlyoffice-postgresql + volumes: + - ./onlyoffice_data:/var/www/onlyoffice/Data + extra_hosts: + - "host.docker.internal:host-gateway" + +networks: + tdrive_network: + driver: bridge \ No newline at end of file diff --git a/tdrive/frontend/public/locales/en.json b/tdrive/frontend/public/locales/en.json index 0dd0aec96..fafac203d 100644 --- a/tdrive/frontend/public/locales/en.json +++ b/tdrive/frontend/public/locales/en.json @@ -20,10 +20,6 @@ "compenents.ConfirmTrashModalContent_move_to_trash": "Move to trash", "compenents.ConfirmTrashModalContent_move_to_trash_desc": "Click 'Move to trash' to move the selected items to the trash folder. You can restore them later from the trash.", "compenents.ConfirmTrashModalContent_to_trash": "to trash", - "compenents.ProprietiesModalContent_name": "Name", - "compenents.ProprietiesModalContent_place_holder": "Document or folder name", - "compenents.ProprietiesModalContent_rename": "Rename", - "compenents.ProprietiesModalContent_update_button": "Update name", "compenents.VersionModalContent_create": "Create version", "compenents.VersionModalContent_donwload": "Download", "compenents.VersionModalContent_version": "Versions of", @@ -100,7 +96,6 @@ "components.item_context_menu.to_trash_multiple": "Move selected items to trash", "components.item_context_menu.today": "Today", "components.item_context_menu.trash.empty": "Empty trash", - "components.item_context_menu.trash.exit": "Exit trash", "components.item_context_menu.versions": "Versions", "components.locked_features.locked_drive_popup.subtitle": "In the free version, you can store only 6GB.", "components.locked_features.locked_drive_popup.title": "You have just reached the drive limit", @@ -127,6 +122,10 @@ "components.open_desktop_popup.subtitle": "Opened in Tdrive app", "components.pending_file_list.estimation.approximations": "Waiting for time approximations...", "components.pending_file_list.estimation.end": "Will end", + "components.PropertiesModalContent_name": "Name", + "components.PropertiesModalContent_place_holder": "Document or folder name", + "components.PropertiesModalContent_rename": "Rename", + "components.PropertiesModalContent_update_button": "Update name", "components.public-link-access-level-create": "Anyone with the link will have access to", "components.public-link-access-level-delete": "Delete link", "components.public-link-access-level-update-subtitle": "Anyone with the link has access to", diff --git a/tdrive/frontend/public/locales/fr.json b/tdrive/frontend/public/locales/fr.json index 11adc2244..e2f320393 100644 --- a/tdrive/frontend/public/locales/fr.json +++ b/tdrive/frontend/public/locales/fr.json @@ -218,10 +218,10 @@ "components.SelectorModalContent_no_items": "Pas de fichier sélectionné", "components.SelectorModalContent_select": "sélectionné(s)", "components.SelectorModalContent_files": "fichier(s)", - "compenents.ProprietiesModalContent_rename": "Renommer", - "compenents.ProprietiesModalContent_name": "Nom", - "compenents.ProprietiesModalContent_place_holder": "Nom du fichier ou du document", - "compenents.ProprietiesModalContent_update_button": "Renommer", + "components.PropertiesModalContent_rename": "Renommer", + "components.PropertiesModalContent_name": "Nom", + "components.PropertiesModalContent_place_holder": "Nom du fichier ou du document", + "components.PropertiesModalContent_update_button": "Renommer", "compenents.ConfirmTrashModalContent_move": "Déplacer", "compenents.ConfirmTrashModalContent_to_trash": "vers la corbeille", "compenents.ConfirmTrashModalContent_items_to_trash": "éléments vers la corbeille", @@ -337,7 +337,6 @@ "components.item_context_menu.clear_selection": "Annuler la sélection", "components.item_context_menu.delete_multiple": "Supprimer", "components.item_context_menu.to_trash_multiple": "Supprimer", - "components.item_context_menu.trash.exit": "Quitter la corbeille", "components.item_context_menu.trash.empty": "Vider la corbeille", "components.item_context_menu.add_documents": "Ajouter un document ou un dossier", "components.item_context_menu.download_folder": "Télécharger le dossier", diff --git a/tdrive/frontend/public/locales/ru.json b/tdrive/frontend/public/locales/ru.json index f855dfc11..3098a8f98 100644 --- a/tdrive/frontend/public/locales/ru.json +++ b/tdrive/frontend/public/locales/ru.json @@ -20,10 +20,6 @@ "compenents.ConfirmTrashModalContent_move_to_trash": "Удалить", "compenents.ConfirmTrashModalContent_move_to_trash_desc": "Нажмите 'Удалить' чтобы переместить выделенные элементы в корзину. Вы сможете восстановить их позже.", "compenents.ConfirmTrashModalContent_to_trash": "в корзину", - "compenents.ProprietiesModalContent_name": "Имя", - "compenents.ProprietiesModalContent_place_holder": "Имя папки или документа", - "compenents.ProprietiesModalContent_rename": "Переименовать", - "compenents.ProprietiesModalContent_update_button": "Переименовать", "compenents.VersionModalContent_create": "Создать версию", "compenents.VersionModalContent_donwload": "Скачать", "compenents.VersionModalContent_version": "Версии ...", @@ -100,7 +96,6 @@ "components.item_context_menu.to_trash_multiple": "Удалить", "components.item_context_menu.today": "Сегодня", "components.item_context_menu.trash.empty": "Очистить корзину", - "components.item_context_menu.trash.exit": "Выйти из корзины", "components.item_context_menu.versions": "История изменений", "components.locked_features.locked_drive_popup.subtitle": "В бесплатной версии можно хранить только 6 ГБ.", "components.locked_features.locked_drive_popup.title": "Вы не можете загружать больше файлов", @@ -127,6 +122,10 @@ "components.open_desktop_popup.subtitle": "открыто в приложении Tdrive", "components.pending_file_list.estimation.approximations": "Примерное время ожидания...", "components.pending_file_list.estimation.end": "Закончится через", + "components.PropertiesModalContent_name": "Имя", + "components.PropertiesModalContent_place_holder": "Имя папки или документа", + "components.PropertiesModalContent_rename": "Переименовать", + "components.PropertiesModalContent_update_button": "Переименовать", "components.public-link-access-level-create": "Все кто имеет ссылку буду иметь доступ", "components.public-link-access-level-delete": "Удалить ссылку", "components.public-link-access-level-update-subtitle": "У каждого, кто имеет ссылку, есть доступ к", diff --git a/tdrive/frontend/public/locales/vn.json b/tdrive/frontend/public/locales/vn.json index b40aa0720..71528743e 100644 --- a/tdrive/frontend/public/locales/vn.json +++ b/tdrive/frontend/public/locales/vn.json @@ -233,10 +233,10 @@ "components.SelectorModalContent_no_items": "Không có mục nào được chọn", "components.SelectorModalContent_select": "Đã chọn", "components.SelectorModalContent_files": "tệp", - "compenents.ProprietiesModalContent_rename": "Đổi tên", - "compenents.ProprietiesModalContent_name": "Tên", - "compenents.ProprietiesModalContent_place_holder": "Tên tài liệu hoặc thư mục", - "compenents.ProprietiesModalContent_update_button": "Cập nhật tên", + "components.PropertiesModalContent_rename": "Đổi tên", + "components.PropertiesModalContent_name": "Tên", + "components.PropertiesModalContent_place_holder": "Tên tài liệu hoặc thư mục", + "components.PropertiesModalContent_update_button": "Cập nhật tên", "compenents.ConfirmTrashModalContent_move": "Di chuyển", "compenents.ConfirmTrashModalContent_to_trash": "vào thùng rác", "compenents.ConfirmTrashModalContent_items_to_trash": "mục vào thùng rác", @@ -326,7 +326,6 @@ "components.item_context_menu.clear_selection": "Xóa lựa chọn", "components.item_context_menu.delete_multiple": "Xóa", "components.item_context_menu.to_trash_multiple": "Di chuyển các mục đã chọn vào thùng rác", - "components.item_context_menu.trash.exit": "Thoát thùng rác", "components.item_context_menu.trash.empty": "Làm trống thùng rác", "components.item_context_menu.add_documents": "Thêm tài liệu hoặc thư mục", "components.item_context_menu.download_folder": "Tải xuống thư mục", diff --git a/tdrive/frontend/src/app/views/client/body/drive/context-menu.tsx b/tdrive/frontend/src/app/views/client/body/drive/context-menu.tsx index 1880d56a5..a909022b2 100644 --- a/tdrive/frontend/src/app/views/client/body/drive/context-menu.tsx +++ b/tdrive/frontend/src/app/views/client/body/drive/context-menu.tsx @@ -268,12 +268,6 @@ export const useOnBuildContextMenu = (children: DriveItem[], initialParentId?: s //Add parent related menus const newMenuActions: any[] = inTrash ? [ - { - type: 'menu', - text: Languages.t('components.item_context_menu.trash.exit'), - onClick: () => setParentId('root'), - }, - { type: 'separator' }, { type: 'menu', text: Languages.t('components.item_context_menu.trash.empty'), diff --git a/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx b/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx index e3c75e8b6..189d609aa 100644 --- a/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx +++ b/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx @@ -4,7 +4,7 @@ import { Input } from '@atoms/input/input-text'; import { Modal, ModalContent } from '@atoms/modal'; import { useDriveActions } from '@features/drive/hooks/use-drive-actions'; import { useDriveItem } from '@features/drive/hooks/use-drive-item'; -import { useEffect, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { atom, useRecoilState } from 'recoil'; import Languages from '@features/global/services/languages-service'; @@ -38,54 +38,79 @@ const PropertiesModalContent = ({ id, onClose }: { id: string; onClose: () => vo const { update } = useDriveActions(); const [loading, setLoading] = useState(false); const [name, setName] = useState(''); + const inputRef = useRef(null); useEffect(() => { refresh(id); }, []); + useEffect(() => { + setTimeout(() => { + if (inputRef.current) { + inputRef.current.focus(); + const lastDot = inputRef.current.value.lastIndexOf('.'); + const endRange = lastDot >= 0 ? lastDot : inputRef.current.value.length; + inputRef.current.setSelectionRange(0, endRange); + } + }, 100); + }, []); + useEffect(() => { if (!name) setName(item?.name || ''); }, [item?.name]); + const doSave = async () => { + setLoading(true); + if (item) { + let finalName = (name || '').trim(); + //TODO: Confirm rename if extension changed ? + if (!item?.is_directory) { + //TODO: Why do we trim extensions on folders ? + const lastDotIndex = finalName.lastIndexOf('.'); + if (lastDotIndex !== -1) { + const fileExtension = name.slice(lastDotIndex); + finalName = finalName.slice(0, lastDotIndex) + fileExtension; + } + } + await update({ name: finalName }, id, item.parent_id); + } + onClose(); + setLoading(false); + } + return ( setName(e.target.value)} - placeholder={Languages.t('compenents.ProprietiesModalContent_place_holder')} + onKeyUp={({ key }) => { + if (!loading) { + if (key === 'Enter') + doSave(); + else if (key === "Escape") + onClose(); + } + }} + placeholder={Languages.t('components.PropertiesModalContent_place_holder')} /> } />
);