From 688217dfdbb8ab550ba2154372f04f14f5e391e7 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 25 Aug 2024 22:26:27 +0200 Subject: [PATCH 01/52] =?UTF-8?q?=F0=9F=9A=A7=20WIP:=20backend:=20beginnin?= =?UTF-8?q?gs=20of=20a=20transactional=20editing=20for=20plugins=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/documents/services/index.ts | 74 +++++++++++++++++++ .../documents/web/controllers/documents.ts | 38 ++++++++++ .../node/src/services/documents/web/routes.ts | 14 ++++ .../src/services/onlyoffice.service.ts | 5 +- 4 files changed, 130 insertions(+), 1 deletion(-) diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 25d1a8c46..b8c4ae548 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -1025,6 +1025,80 @@ export class DocumentsService { CrudException.throwMe(error, new CrudException("Failed to begin editing Drive item", 500)); } }; + + /** + * End editing session either by providing a URL to a new file to create a version, + * or not, to just cancel the session. + * @param editing_session_key Editing key of the DriveFile + * @param newFileUrl Optional URL to a new version of the file to store + * @param options Optional upload information from the request + */ + endEditing = async ( + editing_session_key: string, + newFileUrl: string | null, + context: DriveExecutionContext, + ) => { + if (!context) { + this.logger.error("invalid execution context"); + return null; + } + if (!editing_session_key) { + this.logger.error("Invalid editing_session_key: " + JSON.stringify(editing_session_key)); + throw new CrudException("Invalid editing_session_key", 400); + } + + const driveFile = await this.repository.findOne({ editing_session_key }, {}, context); + if (!driveFile) { + this.logger.error("Drive item not found by editing session key"); + throw new CrudException("Item not found by editing session key", 404); + } + + const hasAccess = await checkAccess(driveFile.id, driveFile, "write", this.repository, context); + if (!hasAccess) { + logger.error("user does not have access drive item " + driveFile.id); + CrudException.throwMe( + new Error("user does not have access to the drive item"), + new CrudException("user does not have access drive item", 401), + ); + } + + if (newFileUrl) { + // const response = await axios.request({ + // url: domain + req.url, + // method: req.method as any, + // headers: _.omit(req.headers, "host", "content-length") as { + // [key: string]: string; + // }, + // data: req.body as any, + // responseType: "stream", + // maxRedirects: 0, + // validateStatus: null, + // }); + //TODO: save file into new version + } + + try { + const result = await this.repository.atomicCompareAndSet( + driveFile, + "editing_session_key", + editing_session_key, + null, + ); + if (!result.didSet) + throw new Error( + `Couldn't set editing_session_key ${JSON.stringify( + editing_session_key, + )} on DriveFile ${JSON.stringify(driveFile.id)} because it is ${JSON.stringify( + result.currentValue, + )}`, + ); + return { ok: true }; + } catch (error) { + logger.error({ error: `${error}` }, "Failed to cancel editing Drive item"); + CrudException.throwMe(error, new CrudException("Failed to cancel editing Drive item", 500)); + } + }; + downloadGetToken = async ( ids: string[], versionId: string | null, diff --git a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts index 973c07656..553417880 100644 --- a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts +++ b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts @@ -372,6 +372,44 @@ export class DocumentsController { } }; + /** + * Finish an editing session by cancelling it. + */ + cancelEditing = async ( + request: FastifyRequest<{ + Params: ItemRequestByEditingSessionKeyParams; + Body: { editorApplicationId: string }; + }>, + ) => { + try { + const context = getDriveExecutionContext(request); + const { editing_session_key } = request.params; + + if (!editing_session_key) throw new CrudException("Missing editing_session_key", 400); + + return await globalResolver.services.documents.documents.endEditing( + editing_session_key, + null, + context, + ); + } catch (error) { + logger.error({ error: `${error}` }, "Failed to begin editing Drive item"); + CrudException.throwMe(error, new CrudException("Failed to begin editing Drive item", 500)); + } + }; + //TODO: will need a save under session key, but without ending the edit (for force saves) + /** + * Finish an editing session for a given `editing_session_key` by uploading the new version of the File + */ + endEditing = async ( + request: FastifyRequest<{ + Params: ItemRequestByEditingSessionKeyParams; + Body: { editorApplicationId: string }; + }>, + ) => { + console.log(request); // make linter happy + }; + downloadGetToken = async ( request: FastifyRequest<{ Params: ItemRequestParams; diff --git a/tdrive/backend/node/src/services/documents/web/routes.ts b/tdrive/backend/node/src/services/documents/web/routes.ts index ed1e3e5da..be75de4da 100644 --- a/tdrive/backend/node/src/services/documents/web/routes.ts +++ b/tdrive/backend/node/src/services/documents/web/routes.ts @@ -94,6 +94,20 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) handler: documentsController.getByEditingSessionKey.bind(documentsController), }); + fastify.route({ + method: "POST", + url: `${serviceUrl}/editing_session/:editing_session_key`, + preValidation: [fastify.authenticateOptional], + handler: documentsController.endEditing.bind(documentsController), + }); + + fastify.route({ + method: "DELETE", + url: `${serviceUrl}/editing_session/:editing_session_key`, + preValidation: [fastify.authenticateOptional], + handler: documentsController.endEditing.bind(documentsController), + }); + fastify.route({ method: "GET", url: `${serviceUrl}/download/token`, diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index 72a9a161d..944c3796b 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -171,7 +171,10 @@ class OnlyOfficeService { constructor() { this.poller = new PolledThingieValue('Connect to Only Office', () => this.getVersion(), 10 * 1000 * 60); } - /** Get the latest Only Office version */ + /** Get the latest Only Office version from polling. If the return is `undefined` + * it probably means there is a connection issue contacting the OnlyOffice server + * from the connector. + */ public getLatestVersion() { return this.poller.latest(); } From ca3ebcc83d5ca2e6bfbcae4457a1ad17e8ea5134 Mon Sep 17 00:00:00 2001 From: Anton SHEPILOV Date: Sun, 25 Aug 2024 22:26:27 +0200 Subject: [PATCH 02/52] =?UTF-8?q?=F0=9F=9A=A7Changing=20file=20identifier?= =?UTF-8?q?=20with=20editing=5Fsession=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../documents/web/controllers/documents.ts | 2 + .../src/controllers/index.controller.ts | 11 ++++ .../src/controllers/onlyoffice.controller.ts | 5 +- .../src/services/drive.service.ts | 12 +++- tdrive/docker-compose.dev.onlyoffice.yml | 62 +++++++++++++++++++ 5 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 tdrive/docker-compose.dev.onlyoffice.yml diff --git a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts index 553417880..9cc4c37a9 100644 --- a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts +++ b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts @@ -350,10 +350,12 @@ export class DocumentsController { beginEditing = async ( request: FastifyRequest<{ Params: ItemRequestParams; + //TODO application id should be received from the token that we have during the login Body: { editorApplicationId: string }; }>, ) => { try { + //TODO create application execution context with the application identifier inside const context = getDriveExecutionContext(request); const { id } = request.params; diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts index 61e6c8734..31d9e11b2 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts @@ -74,17 +74,27 @@ class IndexController { throw new Error('You do not have access to this file'); } + let editingSessionId = null; + if (!preview) { + editingSessionId = driveService.beginEditing(drive_file_id); + //TODO catch error and display to the user when we can't stopped editing + + //TODO Log error with format to be able to set up grafana alert fir such king of errors + } + const officeToken = jwt.sign( { user_id: user.id, //To verify that link is opened by the same user company_id, drive_file_id, + editing_session_id: editingSessionId, file_id: file.id, file_name: file.filename || file?.metadata?.name || '', preview: !!preview, } as OfficeToken, CREDENTIALS_SECRET, { + //one month, never expiring token expiresIn: 60 * 60 * 24 * 30, }, ); @@ -93,6 +103,7 @@ class IndexController { Utils.joinURL([SERVER_ORIGIN ?? '', SERVER_PREFIX, 'editor'], { token, file_id, + editing_session_id: editingSessionId, company_id, preview, office_token: officeToken, diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 5c368082a..fcafc9ff4 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -74,7 +74,7 @@ class OnlyOfficeController { try { const { url, key } = req.body; const { token } = req.query; - logger.info('Save request', { key, url, token }); + logger.info('OO callback', req.body); const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; const { preview, company_id, file_id, /* user_id, */ drive_file_id, in_page_token } = officeTokenPayload; @@ -85,6 +85,9 @@ class OnlyOfficeController { switch (req.body.status) { case OnlyOffice.Callback.Status.BEING_EDITED: + // TODO this call back we recieve almost all the time, and here we save + // the user identifiers who start file editing and even control the amount of onlin users + // to have license constraint warning before OnlyOffice error about this case OnlyOffice.Callback.Status.BEING_EDITED_BUT_IS_SAVED: // No-op break; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index c070ceb91..dd21e3c03 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -2,7 +2,8 @@ import { DriveFileType, IDriveService } from '@/interfaces/drive.interface'; import apiService from './api.service'; import logger from '../lib/logger'; -/** Client for Twake Drive's APIs dealing with `DriveItem`s, using {@see apiService} +/** + * Client for Twake Drive's APIs dealing with `DriveItem`s, using {@see apiService} * to handle authorization */ class DriveService implements IDriveService { @@ -47,6 +48,15 @@ class DriveService implements IDriveService { return Promise.reject(); } }; + + public beginEditing(drive_file_id: string): string { + return ""; + } + + public endEditing(editing_session_id: string) { + return ""; + } + } export default new DriveService(); diff --git a/tdrive/docker-compose.dev.onlyoffice.yml b/tdrive/docker-compose.dev.onlyoffice.yml new file mode 100644 index 000000000..29b0380e9 --- /dev/null +++ b/tdrive/docker-compose.dev.onlyoffice.yml @@ -0,0 +1,62 @@ +version: "3.4" + +services: + + onlyoffice-rabbitmq: + image: rabbitmq:management + hostname: onlyoffice-rabbitmq + container_name: rabbitmq + environment: + - RABBITMQ_DEFAULT_USER=guest + - RABBITMQ_DEFAULT_PASS=guest + ports: + - "5672:5672" + - "15672:15672" + volumes: + - ./.docker-conf/rabbitmq/data/:/var/lib/rabbitmq/ + - ./.docker-conf/rabbitmq/log/:/var/log/rabbitmq + networks: + - tdrive_network + + onlyoffice-postgresql: + image: postgres:13 + hostname: onlyoffice-postgresql + environment: + - POSTGRES_DB=onlyoffice + - POSTGRES_USER=onlyoffice + - POSTGRES_PASSWORD=onlyoffice + ports: + - 5432:5432 + volumes: + - ./onlyoffice_postgres_data:/var/lib/postgresql/data + networks: + - tdrive_network + + onlyoffice: + image: docker.io/onlyoffice/documentserver + hostname: onlyoffice + ports: + - 8090:80 + networks: + - tdrive_network + environment: + - AMQP_URI=amqp://guest:guest@onlyoffice-rabbitmq + - DB_HOST=onlyoffice-postgresql + - DB_NAME=onlyoffice + - DB_PORT=5432 + - DB_TYPE=postgres + - DB_USER=onlyoffice + - JWT_ENABLED=false + - ALLOW_META_IP_ADDRESS=true + - ALLOW_PRIVATE_IP_ADDRESS=true + depends_on: + - onlyoffice-rabbitmq + - onlyoffice-postgresql + volumes: + - ./onlyoffice_data:/var/www/onlyoffice/Data + extra_hosts: + - "host.docker.internal:host-gateway" + +networks: + tdrive_network: + driver: bridge \ No newline at end of file From 3c156c4421d359783a4e3ed0651b56d4956aa74c Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 25 Aug 2024 22:26:28 +0200 Subject: [PATCH 03/52] =?UTF-8?q?=F0=9F=9A=A8=F0=9F=A9=B9=20ooconnector:?= =?UTF-8?q?=20pass=20linter,=20fix=20bug=20where=20ignored=20promises=20in?= =?UTF-8?q?=20poller=20caused=20crash=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/controllers/onlyoffice.controller.ts | 6 ++--- .../src/lib/polled-thingie-value.ts | 10 +++++-- .../src/services/drive.service.ts | 5 ++-- .../src/services/onlyoffice.service.ts | 27 +++++++++++-------- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index fcafc9ff4..1cfe7354c 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -85,9 +85,9 @@ class OnlyOfficeController { switch (req.body.status) { case OnlyOffice.Callback.Status.BEING_EDITED: - // TODO this call back we recieve almost all the time, and here we save - // the user identifiers who start file editing and even control the amount of onlin users - // to have license constraint warning before OnlyOffice error about this + // TODO this call back we recieve almost all the time, and here we save + // the user identifiers who start file editing and even control the amount of onlin users + // to have license constraint warning before OnlyOffice error about this case OnlyOffice.Callback.Status.BEING_EDITED_BUT_IS_SAVED: // No-op break; diff --git a/tdrive/connectors/onlyoffice-connector/src/lib/polled-thingie-value.ts b/tdrive/connectors/onlyoffice-connector/src/lib/polled-thingie-value.ts index d4c2d8a13..27b95e36f 100644 --- a/tdrive/connectors/onlyoffice-connector/src/lib/polled-thingie-value.ts +++ b/tdrive/connectors/onlyoffice-connector/src/lib/polled-thingie-value.ts @@ -19,8 +19,8 @@ export class PolledThingieValue { private readonly getTheThingieValue: () => Promise>, private readonly intervalMs: number, ) { - this.run(); - setInterval(() => this.run(), this.intervalMs); + this.runIgnoringRejection(); + setInterval(() => this.runIgnoringRejection(), this.intervalMs); } protected setResult(value: undefined, error?: NotUndefined, ts?: number); @@ -52,6 +52,12 @@ export class PolledThingieValue { })); } + private runIgnoringRejection() { + return this.run().catch(() => { + /* active ignoring going on here */ + }); + } + public lastFailed() { return !!this.lastKoTimeMs; } diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index dd21e3c03..7b6073931 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -50,13 +50,12 @@ class DriveService implements IDriveService { }; public beginEditing(drive_file_id: string): string { - return ""; + return ''; } public endEditing(editing_session_id: string) { - return ""; + return ''; } - } export default new DriveService(); diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index 944c3796b..39e52d5a9 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -25,6 +25,11 @@ export namespace Callback { USER_INITIATED_FORCE_SAVE = 2, } + interface Action { + type: ActionType; + userid: string; + } + enum ForceSaveType { FROM_COMMAND_SERVICE = 0, FORCE_SAVE_BUTTON_CLICKED = 1, @@ -45,10 +50,6 @@ export namespace Callback { ERROR_FORCE_SAVING = 7, } - interface Action { - type: ActionType; - userid: string; - } /** Parameters given to the callback by the editing service */ export interface Parameters { key: string; @@ -86,18 +87,18 @@ namespace CommandService { } } - abstract class BaseRequest { - constructor(public c: string) {} + abstract class BaseRequest { + constructor(public readonly c: string) {} /** POST this OnlyOffice command, does not check the `error` field of the response */ async postUnsafe(): Promise { logger.silly(`OnlyOffice command ${this.c} sent: ${JSON.stringify(this)}`); - const result = await axios.post(`${ONLY_OFFICE_SERVER}coauthoring/CommandService.ashx`, this); + const result = await axios.post(Utils.joinURL([ONLY_OFFICE_SERVER, 'coauthoring/CommandService.ashx']), this); logger.info(`OnlyOffice command ${this.c} response: ${result.status}: ${JSON.stringify(result.data)}`); return result.data as ErrorResponse | TSuccessResponse; } - /** POST this request, and return the result, or throw if the `errorCode` returned isn't 0 */ + /** POST this request, and return the result, or throws if the `errorCode` returned isn't `ErrorCode.SUCCESS` */ async post(): Promise { const result = await this.postUnsafe(); if (result.error === ErrorCode.SUCCESS) return result; @@ -121,7 +122,7 @@ namespace CommandService { key: string; } export class Request extends BaseRequest { - constructor(public key: string, public userdata: string = '') { + constructor(public readonly key: string, public readonly userdata: string = '') { super('forcesave'); } } @@ -133,7 +134,7 @@ namespace CommandService { url: string; } export class Request extends BaseRequest { - constructor(public key: string) { + constructor(public readonly key: string) { super('getForgotten'); } } @@ -155,7 +156,7 @@ namespace CommandService { key: string; } export class Request extends BaseRequest { - constructor(public key: string) { + constructor(public readonly key: string) { super('deleteForgotten'); } } @@ -178,6 +179,10 @@ class OnlyOfficeService { public getLatestVersion() { return this.poller.latest(); } + + // Note that `async` is important in the functions below. While they avoid the overhead + // of `await`, the `async` is still required to catch the throw in `.post()` + /** Return the version string of OnlyOffice */ async getVersion(): Promise { return new CommandService.Version.Request().post().then(response => response.version); From 7941ba2e1f8eb3b393b558ffdf6a25b143e5e07e Mon Sep 17 00:00:00 2001 From: Anton SHEPILOV Date: Sun, 25 Aug 2024 22:26:28 +0200 Subject: [PATCH 04/52] =?UTF-8?q?=F0=9F=92=84Added=20user=20friendly=20err?= =?UTF-8?q?or=20page=20instead=20of=20JSON=20response?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/middlewares/error.middleware.ts | 10 +- .../onlyoffice-connector/src/views/error.eta | 211 ++++++++++++++++++ 2 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 tdrive/connectors/onlyoffice-connector/src/views/error.eta diff --git a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts index ceb298092..49f1c3173 100644 --- a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts +++ b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts @@ -1,14 +1,20 @@ import logger from '@/lib/logger'; import { NextFunction, Request, Response } from 'express'; +import * as Utils from '@/utils'; +import { SERVER_ORIGIN, SERVER_PREFIX } from '@config'; export default (error: Error & { status?: number }, req: Request, res: Response, next: NextFunction): void => { try { const status: number = error.status || 500; - const message: string = error.message || 'something went wrong'; + const message: string = error.message || 'Something went wrong'; logger.error(`[${req.method}] ${req.path} >> StatusCode:: ${status}, Message:: ${message}`, error.stack); - res.status(status).json({ message }); + res.status(status); + res.render('error', { + server: Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), + errorMessage: message, + }); } catch (error) { next(error); } diff --git a/tdrive/connectors/onlyoffice-connector/src/views/error.eta b/tdrive/connectors/onlyoffice-connector/src/views/error.eta new file mode 100644 index 000000000..b82ed3c31 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/views/error.eta @@ -0,0 +1,211 @@ + + + + + + + + 404 HTML Tempate by Colorlib + + + + + + + +
+
+
+

500

+
+

We are sorry, internal server error!

+

Please, contact our support at support@twake.com.

+ Back To Homepage +
+
+ + + + + + From d94ec46e1df5be9fd92bb674f7826d16a73ae7cb Mon Sep 17 00:00:00 2001 From: Anton SHEPILOV Date: Sun, 25 Aug 2024 22:26:28 +0200 Subject: [PATCH 05/52] =?UTF-8?q?=E2=9C=A8Added=20editing=20session=20key?= =?UTF-8?q?=20to=20the=20connector=20services?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/controllers/index.controller.ts | 13 +++-- .../src/controllers/onlyoffice.controller.ts | 17 ++++-- .../src/interfaces/drive.interface.ts | 3 ++ .../src/interfaces/routes.interface.ts | 1 + .../src/routes/onlyoffice.route.ts | 2 +- .../src/services/api.service.ts | 20 +++++++ .../src/services/drive.service.ts | 53 +++++++++++++++++-- .../onlyoffice-connector/src/views/index.eta | 4 +- 8 files changed, 97 insertions(+), 16 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts index 31d9e11b2..248b9ba6e 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts @@ -74,9 +74,9 @@ class IndexController { throw new Error('You do not have access to this file'); } - let editingSessionId = null; + let editingSessionKey = null; if (!preview) { - editingSessionId = driveService.beginEditing(drive_file_id); + editingSessionKey = await driveService.beginEditingSession(company_id, drive_file_id); //TODO catch error and display to the user when we can't stopped editing //TODO Log error with format to be able to set up grafana alert fir such king of errors @@ -87,7 +87,7 @@ class IndexController { user_id: user.id, //To verify that link is opened by the same user company_id, drive_file_id, - editing_session_id: editingSessionId, + editing_session_key: editingSessionKey, file_id: file.id, file_name: file.filename || file?.metadata?.name || '', preview: !!preview, @@ -103,7 +103,7 @@ class IndexController { Utils.joinURL([SERVER_ORIGIN ?? '', SERVER_PREFIX, 'editor'], { token, file_id, - editing_session_id: editingSessionId, + editing_session_key: editingSessionKey, company_id, preview, office_token: officeToken, @@ -123,11 +123,14 @@ class IndexController { const { user } = req; const officeTokenPayload = jwt.verify(office_token, CREDENTIALS_SECRET) as OfficeToken; - const { preview, user_id, company_id, file_name, file_id, drive_file_id } = officeTokenPayload; + const { preview, user_id, company_id, file_name, file_id, drive_file_id, editing_session_key } = officeTokenPayload; if (user_id !== user.id) { throw new Error('You do not have access to this link'); } + if (!preview && !editing_session_key) { + throw new Error('Cant start editing without "editing session key"'); + } const initResponse = await editorService.init(company_id, file_name, file_id, user, preview, drive_file_id || file_id); diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 1cfe7354c..9c4aec793 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -28,7 +28,7 @@ class OnlyOfficeController { const { token } = req.query; const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; - const { company_id, drive_file_id, file_id, in_page_token } = officeTokenPayload; + const { company_id, drive_file_id, file_id, in_page_token, editing_session_key } = officeTokenPayload; let fileId = file_id; // check token is an in_page_token @@ -36,9 +36,9 @@ class OnlyOfficeController { if (drive_file_id) { //Get the drive file - const driveFile = await driveService.get({ + const driveFile = await driveService.getByEditingSessionKey({ company_id, - drive_file_id, + editing_session_key, }); if (driveFile) { fileId = driveFile?.item?.last_version_cache?.file_metadata?.external_id; @@ -77,7 +77,7 @@ class OnlyOfficeController { logger.info('OO callback', req.body); const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; - const { preview, company_id, file_id, /* user_id, */ drive_file_id, in_page_token } = officeTokenPayload; + const { preview, company_id, file_id, /* user_id, */ drive_file_id, in_page_token, editing_session_key } = officeTokenPayload; // check token is an in_page_token and allow save if (!in_page_token) throw new Error('Invalid token, must be a in_page_token'); @@ -93,6 +93,14 @@ class OnlyOfficeController { break; case OnlyOffice.Callback.Status.READY_FOR_SAVING: + const driveFile = await driveService.getByEditingSessionKey({ + company_id, + editing_session_key, + }); + if (!driveFile) { + throw new Error('Error getting drive files '); + } + const newVersionFile = await fileService.save({ company_id, file_id, @@ -106,6 +114,7 @@ class OnlyOfficeController { file_id: newVersionFile?.resource?.id, }); logger.info('New version created', version); + return respondToOO(); case OnlyOffice.Callback.Status.CLOSED_WITHOUT_CHANGES: diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts index 9d274d668..f17be95f7 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts @@ -19,4 +19,7 @@ export type DriveRequestParams = { export interface IDriveService { get: (params: DriveRequestParams) => Promise; createVersion: (params: { company_id: string; drive_file_id: string; file_id: string }) => Promise; + beginEditingSession: (company_id: string, drive_file_id: string) => Promise; + endEditing: (company_id: string, editing_session_key: string) => Promise; + getByEditingSessionKey: (params: { company_id: string; editing_session_key: string; user_token?: string }) => Promise; } diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts index 7a0295461..bced3c8d9 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts @@ -11,6 +11,7 @@ export interface OfficeToken { file_id: string; file_name: string; preview: boolean; + editing_session_key: string; drive_file_id?: string; in_page_token?: boolean; } diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts index 6bc3c879b..43aa36831 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts @@ -15,7 +15,7 @@ class OnlyOfficeRoute implements Routes { private initRoutes = () => { this.router.get(`${this.path}:mode/read`, requirementsMiddleware, this.onlyOfficeController.read); - this.router.post(`${this.path}:mode/save`, requirementsMiddleware, this.onlyOfficeController.ooCallback); + this.router.post(`${this.path}:mode/callback`, requirementsMiddleware, this.onlyOfficeController.ooCallback); }; } diff --git a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts index fd7bd96ed..9aff4fde9 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts @@ -49,6 +49,26 @@ class ApiService implements IApiService { return await axiosWithToken.get(url, config); }; + public delete = async (params: IApiServiceRequestParams): Promise => { + const { url, token, responseType, headers } = params; + + const axiosWithToken = await this.requireAxios(); + + const config: AxiosRequestConfig = {}; + + if (token) { + config['headers'] = { + Authorization: `Bearer ${token}`, + ...headers, + }; + } + + if (responseType) { + config['responseType'] = responseType; + } + return await axiosWithToken.delete(url, config); + }; + public post = async (params: IApiServiceRequestParams): Promise => { const { url, payload, headers } = params; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index 7b6073931..53c018ca8 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -49,13 +49,58 @@ class DriveService implements IDriveService { } }; - public beginEditing(drive_file_id: string): string { - return ''; + public async beginEditingSession(company_id: string, drive_file_id: string) { + try { + const resource = await apiService.post<{}, { editingSessionKey: string }>({ + url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}/editing_session`, + payload: { + editorApplicationId: 'mock_application_id', + }, + }); + if (resource?.editingSessionKey) { + return resource.editingSessionKey; + } else { + throw new Error(`Failed to obtain editing session key, response: ${JSON.stringify(resource)}`); + } + } catch (error) { + logger.error('Failed to begin editing session: ', error.stack); + throw error; + } } - public endEditing(editing_session_id: string) { - return ''; + public async endEditing(company_id: string, editing_session_key: string) { + try { + await apiService.delete<{}>({ + url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${editing_session_key}`, + }); + } catch (error) { + logger.error('Failed to begin editing session: ', error.stack); + throw error; + //TODO make monitoring for such kind of errors + } } + + /** + * Get the document information by the editing session key. Just simple call to the drive API + * /item/editing_session/${editing_session_key} + * @param params + */ + public getByEditingSessionKey = async (params: { + company_id: string; + editing_session_key: string; + user_token?: string; + }): Promise => { + try { + const { company_id, editing_session_key } = params; + return await apiService.get({ + url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${editing_session_key}`, + token: params.user_token, + }); + } catch (error) { + logger.error('Failed to fetch file metadata by editing session key: ', error.stack); + throw error; + } + }; } export default new DriveService(); diff --git a/tdrive/connectors/onlyoffice-connector/src/views/index.eta b/tdrive/connectors/onlyoffice-connector/src/views/index.eta index a88e8012e..2376cfae5 100644 --- a/tdrive/connectors/onlyoffice-connector/src/views/index.eta +++ b/tdrive/connectors/onlyoffice-connector/src/views/index.eta @@ -44,7 +44,7 @@ token: "<%= it.file_id %>", type: screen.width < 600 ? 'mobile' : 'desktop', editorConfig: { - callbackUrl: `${window.baseURL}save?file_id=<%= it.file_id %>&company_id=<%= it.company_id %>&token=<%= it.token %>`, + callbackUrl: `${window.baseURL}callback?file_id=<%= it.file_id %>&company_id=<%= it.company_id %>&token=<%= it.token %>`, lang: window.user.language, user: { id: window.user.id, @@ -52,7 +52,7 @@ }, customization: { chat: false, - compactToolbar: true, + compactToolbar: false, about: false, feedback: false, goback: { From 0914db9789d5377edbb135161b5f18522eada5b9 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 25 Aug 2024 22:26:28 +0200 Subject: [PATCH 06/52] =?UTF-8?q?=E2=9C=A8=20ooconnector:=20poll=20forgott?= =?UTF-8?q?en=20files,=20wip:=20extract=20code=20from=20oocallback=20to=20?= =?UTF-8?q?call=20here=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/onlyoffice-connector/readme.md | 1 + .../onlyoffice-connector/src/app.ts | 9 ++- .../onlyoffice-connector/src/config/index.ts | 4 ++ .../src/services/api.service.ts | 31 +++++++-- .../src/services/onlyoffice.service.ts | 63 +++++++++++++++++-- .../onlyoffice-connector/src/utils.ts | 11 ++++ 6 files changed, 107 insertions(+), 12 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/readme.md b/tdrive/connectors/onlyoffice-connector/readme.md index 6811f33d7..cad453bd9 100644 --- a/tdrive/connectors/onlyoffice-connector/readme.md +++ b/tdrive/connectors/onlyoffice-connector/readme.md @@ -18,6 +18,7 @@ Documentation of the [API used with OnlyOffice is here](https://api.onlyoffice.c - The inline configuration provides a read and a write URL to the DocsAPI editor, these point to the routes of `OnlyOfficeController`. - When all the clients have disconnected from the editing session on the OnlyOffice document editing service, the OnlyOffice server will call the `callbackUrl` of this connector. - The connector then downloads the new file from Only Office, and creates a new version in Twake Drive. +- Periodically gets a list of forgotten files and updates the backend with them ## Configuration example diff --git a/tdrive/connectors/onlyoffice-connector/src/app.ts b/tdrive/connectors/onlyoffice-connector/src/app.ts index 65258ba9a..c784dede7 100644 --- a/tdrive/connectors/onlyoffice-connector/src/app.ts +++ b/tdrive/connectors/onlyoffice-connector/src/app.ts @@ -45,8 +45,13 @@ class App { }); this.app.get('/health', (_req, res) => { - Promise.all([onlyofficeService.getLatestVersion(), apiService.hasToken()]).then( - ([version, twakeDriveToken]) => res.status(version && twakeDriveToken ? 200 : 500).send({ version, twakeDriveToken }), + Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken()]).then( + ([onlyOfficeLicense, twakeDriveToken]) => + res.status(onlyOfficeLicense && twakeDriveToken ? 200 : 500).send({ + uptime: process.uptime(), + onlyOfficeLicense, + twakeDriveToken, + }), err => res.status(500).send(err), ); }); diff --git a/tdrive/connectors/onlyoffice-connector/src/config/index.ts b/tdrive/connectors/onlyoffice-connector/src/config/index.ts index ca0a3f5e1..e25db1b8e 100644 --- a/tdrive/connectors/onlyoffice-connector/src/config/index.ts +++ b/tdrive/connectors/onlyoffice-connector/src/config/index.ts @@ -12,3 +12,7 @@ export const { SERVER_PREFIX, SERVER_ORIGIN, } = process.env; + +export const twakeDriveTokenRefrehPeriodMS = 10 * 60 * 1000; +export const onlyOfficeForgottenFilesCheckPeriodMS = 10 * 60 * 1000; +export const onlyOfficeConnectivityCheckPeriodMS = 10 * 60 * 1000; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts index 9aff4fde9..a509cf5cb 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts @@ -5,28 +5,49 @@ import { IApiServiceApplicationTokenResponse, } from '@/interfaces/api.interface'; import axios, { Axios, AxiosRequestConfig, AxiosResponse } from 'axios'; -import { CREDENTIALS_ENDPOINT, CREDENTIALS_ID, CREDENTIALS_SECRET } from '@config'; +import { + CREDENTIALS_ENDPOINT, + CREDENTIALS_ID, + CREDENTIALS_SECRET, + twakeDriveTokenRefrehPeriodMS, + onlyOfficeForgottenFilesCheckPeriodMS, +} from '@config'; import logger from '../lib/logger'; import * as Utils from '@/utils'; import { PolledThingieValue } from '@/lib/polled-thingie-value'; +import onlyofficeService from './onlyoffice.service'; /** * Client for the Twake Drive backend API on behalf of the plugin (or provided token in parameters). * Periodically updates authorization and adds to requests. */ class ApiService implements IApiService { - private readonly poller: PolledThingieValue; + private readonly tokenPoller: PolledThingieValue; + private readonly forgottenFilesPoller: PolledThingieValue; constructor() { - this.poller = new PolledThingieValue('Refresh Twake Drive token', async () => this.refreshToken(), 1000 * 60); //TODO: should be Every 10 minutes + this.tokenPoller = new PolledThingieValue('Refresh Twake Drive token', async () => await this.refreshToken(), twakeDriveTokenRefrehPeriodMS); + this.forgottenFilesPoller = new PolledThingieValue( + 'Process forgotten files in OO', + async () => await this.processForgottenFiles(), + onlyOfficeForgottenFilesCheckPeriodMS, + ); } public async hasToken() { - return (await this.poller.latestValueWithTry()) !== undefined; + return (await this.tokenPoller.latestValueWithTry()) !== undefined; } private requireAxios() { - return this.poller.requireLatestValueWithTry('Token Kind 538 not ready'); + return this.tokenPoller.requireLatestValueWithTry('No Twake Drive app token.'); + } + + private async processForgottenFiles() { + if (!this.tokenPoller.hasValue()) return -1; + return await onlyofficeService.processForgotten(async (/* key, url */) => { + //TODO: when endpoint decided, call here. See if accept HTTP 202 for ex. to avoid deleting. + return false; + }); } public get = async (params: IApiServiceRequestParams): Promise => { diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index 39e52d5a9..d0f8e07d5 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -1,5 +1,5 @@ import axios from 'axios'; -import { ONLY_OFFICE_SERVER } from '@config'; +import { ONLY_OFFICE_SERVER, onlyOfficeConnectivityCheckPeriodMS } from '@config'; import { PolledThingieValue } from '@/lib/polled-thingie-value'; import logger from '@/lib/logger'; import * as Utils from '@/utils'; @@ -161,6 +161,20 @@ namespace CommandService { } } } + + export namespace License { + export interface Response extends SuccessResponse { + // @see https://api.onlyoffice.com/editors/command/license + license: object; + server: object; + quota: object; + } + export class Request extends BaseRequest { + constructor() { + super('license'); + } + } + } } /** @@ -168,18 +182,52 @@ namespace CommandService { * @see https://api.onlyoffice.com/editors/command/ */ class OnlyOfficeService { - private readonly poller: PolledThingieValue; + private readonly poller: PolledThingieValue; + constructor() { - this.poller = new PolledThingieValue('Connect to Only Office', () => this.getVersion(), 10 * 1000 * 60); + this.poller = new PolledThingieValue( + 'Connect to Only Office', + async () => { + logger.info('Only Office license status'); + return await this.getLicense(); + }, + onlyOfficeConnectivityCheckPeriodMS, + ); } - /** Get the latest Only Office version from polling. If the return is `undefined` + /** Get the latest Only Office licence status from polling. If the return is `undefined` * it probably means there is a connection issue contacting the OnlyOffice server * from the connector. */ - public getLatestVersion() { + public getLatestLicence() { return this.poller.latest(); } + /** + * Iterate over all the forgotten files as returned by OnlyOffice, call the processor for each.. + * @param processor Handler to process the forgotten file (available at `url`). If `true` is returned, + * the file is deleted from the forgotten file list in OnlyOffice. If false is returned, the + * same forgotten file will reappear in a future batch + * @returns The number of files processed and deleted + */ + public async processForgotten(processor: (key: string, url: string) => Promise): Promise { + const forgottenFiles = await this.getForgottenList(); + if (forgottenFiles.length === 0) return 0; + Utils.fisherYattesShuffleInPlace(forgottenFiles); + logger.info(`Forgotten files found: ${forgottenFiles.length}`); + let deleted = 0; + for (const forgottenFileKey of forgottenFiles) { + const forgottenFileURL = await this.getForgotten(forgottenFileKey); + logger.info(`Forgotten file about to process: ${JSON.stringify(forgottenFileKey)}`, { url: forgottenFileURL }); + const shouldDelete = await processor(forgottenFileKey, forgottenFileURL); + if (shouldDelete) { + logger.info(`Forgotten file about to be deleted: ${JSON.stringify(forgottenFileKey)}`); + await this.deleteForgotten(forgottenFileKey); + deleted++; + } + } + return deleted; + } + // Note that `async` is important in the functions below. While they avoid the overhead // of `await`, the `async` is still required to catch the throw in `.post()` @@ -187,6 +235,11 @@ class OnlyOfficeService { async getVersion(): Promise { return new CommandService.Version.Request().post().then(response => response.version); } + /** Return the version string of OnlyOffice */ + async getLicense(): Promise { + //TODO: When typing the response more fully, don't return the response object itself as here + return new CommandService.License.Request().post(); + } /** Force a save in the editing session key provided. `userdata` will be forwarded to the callback */ async forceSave(key: string, userdata = ''): Promise { return new CommandService.ForceSave.Request(key, userdata).post().then(response => response.key); diff --git a/tdrive/connectors/onlyoffice-connector/src/utils.ts b/tdrive/connectors/onlyoffice-connector/src/utils.ts index 094781722..1fc1d55b3 100644 --- a/tdrive/connectors/onlyoffice-connector/src/utils.ts +++ b/tdrive/connectors/onlyoffice-connector/src/utils.ts @@ -44,3 +44,14 @@ export function splitFilename(filename: string): [string, string] { const extension = parts.pop(); return [parts.join('.'), extension]; } + +/** Shuffle an array in place, returns its parameter for convenience */ +export function fisherYattesShuffleInPlace(list: T[]): T[] { + let index = list.length; + while (index) { + const randomIndex = Math.floor(Math.random() * index); + index--; + [list[index], list[randomIndex]] = [list[randomIndex], list[index]]; + } + return list; +} From a692eabf3b7c8f53367fae632e0ee6b0fcb584a6 Mon Sep 17 00:00:00 2001 From: Anton SHEPILOV Date: Sun, 25 Aug 2024 22:26:28 +0200 Subject: [PATCH 07/52] =?UTF-8?q?=F0=9F=90=9BFixed=20document=20identifier?= =?UTF-8?q?=20for=20OnlyOffice=20in=20view=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/controllers/index.controller.ts | 1 + .../src/controllers/onlyoffice.controller.ts | 17 +++-------------- .../onlyoffice-connector/src/views/index.eta | 2 +- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts index 248b9ba6e..fd05b2af1 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts @@ -144,6 +144,7 @@ class IndexController { res.render('index', { ...initResponse, + docId: preview ? file_id : editing_session_key, server: Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), token: inPageToken, }); diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 9c4aec793..7bce33f67 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -28,26 +28,15 @@ class OnlyOfficeController { const { token } = req.query; const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; - const { company_id, drive_file_id, file_id, in_page_token, editing_session_key } = officeTokenPayload; - let fileId = file_id; + const { company_id, file_id, in_page_token } = officeTokenPayload; // check token is an in_page_token if (!in_page_token) throw new Error('Invalid token, must be a in_page_token'); - if (drive_file_id) { - //Get the drive file - const driveFile = await driveService.getByEditingSessionKey({ - company_id, - editing_session_key, - }); - if (driveFile) { - fileId = driveFile?.item?.last_version_cache?.file_metadata?.external_id; - } - } - + if (!file_id) throw new Error(`File id is missing in the last version cache for ${JSON.stringify(file_id)}`); const file = await fileService.download({ company_id, - file_id: fileId, + file_id: file_id, }); file.pipe(res); diff --git a/tdrive/connectors/onlyoffice-connector/src/views/index.eta b/tdrive/connectors/onlyoffice-connector/src/views/index.eta index 2376cfae5..6d9b9ec0d 100644 --- a/tdrive/connectors/onlyoffice-connector/src/views/index.eta +++ b/tdrive/connectors/onlyoffice-connector/src/views/index.eta @@ -26,7 +26,7 @@ title: "<%= it.filename %>", url: `${window.baseURL}read?file_id=<%= it.file_id %>&company_id=<%= it.company_id %>&token=<%= it.token %>`, fileType: "<%= it.file_type %>", - key: "<%= it.file_version_id %>", + key: "<%= it.docId %>", token: "<%= it.file_id %>", permissions: { download: true, From b6be90f8252ffec2e86ea631bd9fc2bfe6c4fe1c Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 25 Aug 2024 22:26:29 +0200 Subject: [PATCH 08/52] =?UTF-8?q?=F0=9F=9A=A7=20ooconnector:=20show=20forg?= =?UTF-8?q?otten=20files=20in=20health=20endpoint=20for=20debugging=20(#52?= =?UTF-8?q?5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdrive/connectors/onlyoffice-connector/src/app.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/app.ts b/tdrive/connectors/onlyoffice-connector/src/app.ts index c784dede7..58ebb0516 100644 --- a/tdrive/connectors/onlyoffice-connector/src/app.ts +++ b/tdrive/connectors/onlyoffice-connector/src/app.ts @@ -45,12 +45,13 @@ class App { }); this.app.get('/health', (_req, res) => { - Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken()]).then( - ([onlyOfficeLicense, twakeDriveToken]) => + Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken(), onlyofficeService.getForgottenList()]).then( + ([onlyOfficeLicense, twakeDriveToken, forgottenKeys]) => res.status(onlyOfficeLicense && twakeDriveToken ? 200 : 500).send({ uptime: process.uptime(), onlyOfficeLicense, twakeDriveToken, + forgottenKeys, }), err => res.status(500).send(err), ); From c66123f4503cc8039925c2fa826e4f8e2562cd90 Mon Sep 17 00:00:00 2001 From: Anton SHEPILOV Date: Sun, 25 Aug 2024 22:26:29 +0200 Subject: [PATCH 09/52] =?UTF-8?q?=E2=9C=A8Added=20support=20of=20the=20ses?= =?UTF-8?q?sion=20editing=20key=20for=20the=20OnlyOffice=20connector?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/documents/services/index.ts | 39 +++++++------ .../documents/web/controllers/documents.ts | 39 ++++++++++++- .../node/src/services/documents/web/routes.ts | 2 +- .../backend/node/test/e2e/common/user-api.ts | 30 ++++++++++ .../test/e2e/common/user-authorization.ts | 52 ++++++++++++++++++ .../e2e/documents/editing-session.spec.ts | 23 +++++++- .../src/controllers/onlyoffice.controller.ts | 23 +------- .../src/interfaces/drive.interface.ts | 5 +- .../src/services/drive.service.ts | 55 ++++++++++++++++--- 9 files changed, 216 insertions(+), 52 deletions(-) create mode 100644 tdrive/backend/node/test/e2e/common/user-authorization.ts diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index b8c4ae548..74aebab49 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -61,6 +61,8 @@ import archiver from "archiver"; import internal from "stream"; import config from "config"; import { randomUUID } from "crypto"; +import { MultipartFile } from "@fastify/multipart"; +import { UploadOptions } from "src/services/files/types"; export class DocumentsService { version: "1"; @@ -1030,13 +1032,15 @@ export class DocumentsService { * End editing session either by providing a URL to a new file to create a version, * or not, to just cancel the session. * @param editing_session_key Editing key of the DriveFile - * @param newFileUrl Optional URL to a new version of the file to store + * @param file Multipart files from the incoming http request * @param options Optional upload information from the request + * @param context */ endEditing = async ( editing_session_key: string, - newFileUrl: string | null, - context: DriveExecutionContext, + file: MultipartFile, + options: UploadOptions, + context: CompanyExecutionContext, ) => { if (!context) { this.logger.error("invalid execution context"); @@ -1062,19 +1066,21 @@ export class DocumentsService { ); } - if (newFileUrl) { - // const response = await axios.request({ - // url: domain + req.url, - // method: req.method as any, - // headers: _.omit(req.headers, "host", "content-length") as { - // [key: string]: string; - // }, - // data: req.body as any, - // responseType: "stream", - // maxRedirects: 0, - // validateStatus: null, - // }); - //TODO: save file into new version + if (file) { + const fileEntity = await globalResolver.services.files.save(null, file, options, context); + + await globalResolver.services.documents.documents.createVersion( + driveFile.id, + { + drive_item_id: driveFile.id, + provider: "internal", + file_metadata: { + external_id: fileEntity.id, + source: "internal", + }, + }, + context, + ); } try { @@ -1092,7 +1098,6 @@ export class DocumentsService { result.currentValue, )}`, ); - return { ok: true }; } catch (error) { logger.error({ error: `${error}` }, "Failed to cancel editing Drive item"); CrudException.throwMe(error, new CrudException("Failed to cancel editing Drive item", 500)); diff --git a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts index 9cc4c37a9..0084350e3 100644 --- a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts +++ b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts @@ -392,6 +392,7 @@ export class DocumentsController { return await globalResolver.services.documents.documents.endEditing( editing_session_key, null, + null, context, ); } catch (error) { @@ -406,10 +407,44 @@ export class DocumentsController { endEditing = async ( request: FastifyRequest<{ Params: ItemRequestByEditingSessionKeyParams; - Body: { editorApplicationId: string }; + Querystring: Record; + Body: { + item: Partial; + version: Partial; + }; }>, ) => { - console.log(request); // make linter happy + const { editing_session_key } = request.params; + if (!editing_session_key) throw new CrudException("Editing session key must be set", 400); + + const context = getDriveExecutionContext(request); + + if (request.isMultipart()) { + const file = await request.file(); + const q: Record = request.query; + const options: UploadOptions = { + totalChunks: parseInt(q.resumableTotalChunks || q.total_chunks) || 1, + totalSize: parseInt(q.resumableTotalSize || q.total_size) || 0, + chunkNumber: parseInt(q.resumableChunkNumber || q.chunk_number) || 1, + filename: q.resumableFilename || q.filename || file?.filename || undefined, + type: q.resumableType || q.type || file?.mimetype || undefined, + waitForThumbnail: !!q.thumbnail_sync, + ignoreThumbnails: false, + }; + return await globalResolver.services.documents.documents.endEditing( + editing_session_key, + file, + options, + context, + ); + } else { + return await globalResolver.services.documents.documents.endEditing( + editing_session_key, + null, + null, + context, + ); + } }; downloadGetToken = async ( diff --git a/tdrive/backend/node/src/services/documents/web/routes.ts b/tdrive/backend/node/src/services/documents/web/routes.ts index be75de4da..48a4caebc 100644 --- a/tdrive/backend/node/src/services/documents/web/routes.ts +++ b/tdrive/backend/node/src/services/documents/web/routes.ts @@ -105,7 +105,7 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) method: "DELETE", url: `${serviceUrl}/editing_session/:editing_session_key`, preValidation: [fastify.authenticateOptional], - handler: documentsController.endEditing.bind(documentsController), + handler: documentsController.cancelEditing.bind(documentsController), }); fastify.route({ diff --git a/tdrive/backend/node/test/e2e/common/user-api.ts b/tdrive/backend/node/test/e2e/common/user-api.ts index c41c6fc93..c0186901d 100644 --- a/tdrive/backend/node/test/e2e/common/user-api.ts +++ b/tdrive/backend/node/test/e2e/common/user-api.ts @@ -409,6 +409,36 @@ export default class UserApi { }); } + async endEditingDocument( + editingSessionKey: string + ): Promise { + const fullPath = `${__dirname}/assets/${UserApi.ALL_FILES[0]}`; + const readable= Readable.from(fs.createReadStream(fullPath)); + const form = formAutoContent({ file: readable }); + form.headers["authorization"] = `Bearer ${this.jwt}`; + + return await this.platform.app.inject({ + method: "POST", + url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/editing_session/${editingSessionKey}`, + headers: { + authorization: `Bearer ${this.jwt}` + }, + ...form, + }); + } + + async cancelEditingDocument( + editingSessionKey: string, + ): Promise { + return await this.platform.app.inject({ + method: "DELETE", + url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/editing_session/${editingSessionKey}`, + headers: { + authorization: `Bearer ${this.jwt}` + } + }); + } + async beginEditingDocumentExpectOk( driveFileId: string, editorApplicationId: string, diff --git a/tdrive/backend/node/test/e2e/common/user-authorization.ts b/tdrive/backend/node/test/e2e/common/user-authorization.ts new file mode 100644 index 000000000..8a36ac19c --- /dev/null +++ b/tdrive/backend/node/test/e2e/common/user-authorization.ts @@ -0,0 +1,52 @@ +// import { OidcJwtVerifier } from "../../../src/services/console/clients/remote-jwks-verifier"; +// +// export class UserAuthorization { +// /** +// * Just send the login requests without any validation and login response assertion +// */ +// public async login(session?: string) { +// if (session !== undefined) { +// this.session = session; +// } else { +// this.session = uuidv1(); +// } +// const payload = { +// claims: { +// sub: this.user.id, +// first_name: this.user.first_name, +// sid: this.session, +// }, +// }; +// const verifierMock = jest.spyOn(OidcJwtVerifier.prototype, "verifyIdToken"); +// verifierMock.mockImplementation(() => { +// return Promise.resolve(payload); // Return the predefined payload +// }); +// return await this.api.post("/internal/services/console/v1/login", { +// oidc_id_token: "sample_oidc_token", +// }); +// } +// +// public async logout() { +// const payload = { +// claims: { +// iss: "tdrive_lemonldap", +// sub: this.user.id, +// sid: this.session, +// aud: "your-audience", +// iat: Math.floor(Date.now() / 1000), +// jti: "jwt-id", +// events: { +// "http://schemas.openid.net/event/backchannel-logout": {}, +// }, +// } +// }; +// const verifierMock = jest.spyOn(OidcJwtVerifier.prototype, "verifyLogoutToken"); +// verifierMock.mockImplementation(() => { +// return Promise.resolve(payload); // Return the predefined payload +// }); +// +// return await this.api.post("/internal/services/console/v1/backchannel_logout", { +// logout_token: "logout_token_rsa256", +// }); +// } +// } \ No newline at end of file diff --git a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts index 8c5e6ce5f..845ec9c94 100644 --- a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts +++ b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts @@ -4,6 +4,7 @@ import { init, TestPlatform } from "../setup"; import UserApi from "../common/user-api"; import { DriveFile, TYPE as DriveFileType } from "../../../src/services/documents/entities/drive-file"; +import exp = require("node:constants"); describe("the Drive's documents' editing session kind-of-lock", () => { let platform: TestPlatform | null; @@ -110,8 +111,28 @@ describe("the Drive's documents' editing session kind-of-lock", () => { expect(temporaryDocument.id).toBe(foundDocumentResult.json().id); }); - it('can end an editing session on a document only once with the right key', async () => { + it('can cancel an editing session on a document only once with the right key', async () => { + //given const editingSessionKey = await currentUser.beginEditingDocumentExpectOk(temporaryDocument.id, 'e2e_testing'); + //when + const response = await currentUser.cancelEditingDocument(editingSessionKey); + //then + expect(response.statusCode).toBe(200); + const newSessionKey = await currentUser.beginEditingDocumentExpectOk(temporaryDocument.id, 'e2e_testing'); + expect(newSessionKey).not.toEqual(editingSessionKey); + }); + + it('can end editing with a new version of document', async () => { + //given + const editingSessionKey = await currentUser.beginEditingDocumentExpectOk(temporaryDocument.id, 'e2e_testing'); + //when + const response = await currentUser.endEditingDocument(editingSessionKey); + //then + expect(response.statusCode).toBe(200); + const document = await currentUser.getDocumentOKCheck(temporaryDocument.id); + expect(document.versions.length).toEqual(2); }); + + }); diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 7bce33f67..fcd16c846 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -82,28 +82,9 @@ class OnlyOfficeController { break; case OnlyOffice.Callback.Status.READY_FOR_SAVING: - const driveFile = await driveService.getByEditingSessionKey({ - company_id, - editing_session_key, - }); - if (!driveFile) { - throw new Error('Error getting drive files '); - } - - const newVersionFile = await fileService.save({ - company_id, - file_id, - url, - create_new: true, - }); - - const version = await driveService.createVersion({ - company_id, - drive_file_id, - file_id: newVersionFile?.resource?.id, - }); - logger.info('New version created', version); + await driveService.endEditing(company_id, editing_session_key, url); + logger.info(`New version for session ${editing_session_key} created`); return respondToOO(); case OnlyOffice.Callback.Status.CLOSED_WITHOUT_CHANGES: diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts index f17be95f7..c7b94f3ac 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts @@ -6,6 +6,7 @@ export type DriveFileType = { date_added: number; file_metadata: { external_id: string; + name?: string; }; }; }; @@ -20,6 +21,6 @@ export interface IDriveService { get: (params: DriveRequestParams) => Promise; createVersion: (params: { company_id: string; drive_file_id: string; file_id: string }) => Promise; beginEditingSession: (company_id: string, drive_file_id: string) => Promise; - endEditing: (company_id: string, editing_session_key: string) => Promise; - getByEditingSessionKey: (params: { company_id: string; editing_session_key: string; user_token?: string }) => Promise; + endEditing: (company_id: string, editing_session_key: string, file_source_url: string) => Promise; + getByEditingSessionKey: (params: { company_id: string; editing_session_key: string; user_token?: string }) => Promise; } diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index 53c018ca8..240463538 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -1,6 +1,8 @@ import { DriveFileType, IDriveService } from '@/interfaces/drive.interface'; import apiService from './api.service'; import logger from '../lib/logger'; +import { Stream } from 'stream'; +import FormData from 'form-data'; /** * Client for Twake Drive's APIs dealing with `DriveItem`s, using {@see apiService} @@ -30,7 +32,7 @@ class DriveService implements IDriveService { }): Promise => { try { const { company_id, drive_file_id, file_id } = params; - const resource = await apiService.post<{}, DriveFileType['item']['last_version_cache']>({ + return await apiService.post<{}, DriveFileType['item']['last_version_cache']>({ url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}/version`, payload: { drive_item_id: drive_file_id, @@ -41,8 +43,6 @@ class DriveService implements IDriveService { }, }, }); - - return resource; } catch (error) { logger.error('Failed to create version: ', error.stack); return Promise.reject(); @@ -68,10 +68,49 @@ class DriveService implements IDriveService { } } - public async endEditing(company_id: string, editing_session_key: string) { + public async cancelEditing(company_id: string, editing_session_key) { try { await apiService.delete<{}>({ - url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${editing_session_key}`, + url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${encodeURIComponent(editing_session_key)}`, + }); + } catch (error) { + logger.error('Failed to begin editing session: ', error.stack); + throw error; + //TODO make monitoring for such kind of errors + } + } + + public async endEditing(company_id: string, editing_session_key: string, url: string) { + try { + if (!url) { + throw Error('no url found'); + } + + const originalFile = await this.getByEditingSessionKey({ company_id, editing_session_key }); + + if (!originalFile) { + throw Error('original file not found'); + } + + const newFile = await apiService.get({ + url, + responseType: 'stream', + }); + + const form = new FormData(); + + const filename = encodeURIComponent(originalFile.last_version_cache.file_metadata.name); + + form.append('file', newFile, { + filename, + }); + + logger.info('Saving file version to Twake Drive: ', filename); + + await apiService.post({ + url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${encodeURIComponent(editing_session_key)}`, + payload: form, + headers: form.getHeaders(), }); } catch (error) { logger.error('Failed to begin editing session: ', error.stack); @@ -89,11 +128,11 @@ class DriveService implements IDriveService { company_id: string; editing_session_key: string; user_token?: string; - }): Promise => { + }): Promise => { try { const { company_id, editing_session_key } = params; - return await apiService.get({ - url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${editing_session_key}`, + return await apiService.get({ + url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${encodeURIComponent(editing_session_key)}`, token: params.user_token, }); } catch (error) { From 3b81a21e083c28e608edfa1f6c11a55ca95f7392 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 25 Aug 2024 22:26:29 +0200 Subject: [PATCH 10/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20=20backend:=20move?= =?UTF-8?q?=20editing=5Fsession=5Fkey=20generation=20and=20parsing=20to=20?= =?UTF-8?q?specific=20implementation=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Documentation/docs/plugins.md | 2 +- .../services/documents/entities/drive-file.ts | 81 ++++++++++++++++++- .../src/services/documents/services/index.ts | 32 ++------ 3 files changed, 88 insertions(+), 27 deletions(-) diff --git a/Documentation/docs/plugins.md b/Documentation/docs/plugins.md index ba83a027b..53fbf43b4 100644 --- a/Documentation/docs/plugins.md +++ b/Documentation/docs/plugins.md @@ -43,7 +43,7 @@ Links in top right drop down application grid can be configured in the `applicat Plugins are defined as entries under the `applications.plugins` array, with at least the following properties: -- `id` is mandatory and must be unique to each plugin +- `id` is mandatory and must be unique to each plugin. It must be rather short, and alphanumeric with `_`s only. - `internal_domain` is the internal domain of the plugin's server, it must be the same as the one in the docker-compose.yml file for ex. or resolvable and accessible to the Twake Drive backend server. - `external_prefix` is the external URL prefix of the plugin exposed by the backend and proxied to the `internal_domain`. - `api.private_key` is the shared secret used by the plugin's server to authentify to the backend diff --git a/tdrive/backend/node/src/services/documents/entities/drive-file.ts b/tdrive/backend/node/src/services/documents/entities/drive-file.ts index e3667a7d4..d6f01ccf8 100644 --- a/tdrive/backend/node/src/services/documents/entities/drive-file.ts +++ b/tdrive/backend/node/src/services/documents/entities/drive-file.ts @@ -1,4 +1,5 @@ import { Type } from "class-transformer"; +import { randomUUID } from "crypto"; import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators"; import { DriveFileAccessLevel, publicAccessLevel } from "../types"; import { FileVersion } from "./file-version"; @@ -93,8 +94,9 @@ export class DriveFile { /** * If this field is non-null, then an editing session is in progress (probably in OnlyOffice). - * Should be in the format `timestamp-appid-hexuuid` where `appid` and `timestamp` have no `-` - * characters. + * Use {@see EditingSessionKeyFormat} to generate and interpret it. + * Values should ensure that sorting lexicographically is chronological (assuming perfect clocks everywhere), + * and that the application and user that started the edit session are retrievable. */ @Type(() => String) @Column("editing_session_key", "string") @@ -120,6 +122,81 @@ export class DriveFile { scope: DriveScope; } +/** Reference implementation for generating then parsing the {@link DriveFile.editing_session_key} field */ +export const EditingSessionKeyFormat = { + // OnlyOffice key limits: 128 chars, [0-9a-zA-z=_-] + // This is specific to it, but the constraint seems strict enough + // that any other system needing such a unique identifier would find + // this compatible. This value must be ensured to be the strictest + // common denominator to all plugin/interop systems. Plugins that + // require something even stricter have the option of maintaining + // a look up table to an acceptable value. + generate(applicationId: string, userId: string) { + if (!/^[0-9a-zA-Z_-]+$/m.test(applicationId)) + throw new Error( + `Invalid applicationId string (${JSON.stringify( + applicationId, + )}). Must be short and only alpha numeric`, + ); + const isoUTCDateNoSpecialCharsNoMS = new Date() + .toISOString() + .replace(/\..+$/, "") + .replace(/[ZT:-]/g, ""); + const newKey = [ + isoUTCDateNoSpecialCharsNoMS, + applicationId, + userId.replace(/-+/g, ""), + randomUUID().replace(/-+/g, ""), + ].join("="); + if (newKey.length > 128 || !/^[0-9a-zA-Z=_-]+$/m.test(newKey)) + throw new Error( + `Invalid generated editingSessionKey (${JSON.stringify( + newKey, + )}) string. Must be <128 chars, and only contain [0-9a-zA-z=_-]`, + ); + return newKey; + }, + + parse(editingSessionKey: string) { + const parts = editingSessionKey.split("="); + const expectedParts = 4; + if (parts.length !== expectedParts) + throw new Error( + `Invalid editingSessionKey (${JSON.stringify( + editingSessionKey, + )}). Expected ${expectedParts} parts`, + ); + const [timestampStr, appId, userId, _random] = parts; + const timestampMatch = timestampStr.match( + /^(?\d{4})(?\d\d)(?\d\d)(?\d\d)(?\d\d)(?\d\d)$/, + ); + if (!timestampMatch) + throw new Error( + `Invalid editingSessionKey (${JSON.stringify( + editingSessionKey, + )}). Didn't start with valid timestamp`, + ); + const { year, month, day, hour, minute, second } = timestampMatch.groups!; + const userIdMatch = userId.match( + /^([a-z0-f]{8})([a-z0-f]{4})([a-z0-f]{4})([a-z0-f]{4})([a-z0-f]{12})$/i, + ); + if (!userIdMatch) + throw new Error( + `Invalid editingSessionKey (${JSON.stringify( + editingSessionKey, + )}). UserID has wrong number of digits`, + ); + const [, userIdPart1, userIdPart2, userIdPart3, userIdPart4, userIdPart5] = userIdMatch; + return { + timestamp: new Date( + Date.parse(`${[year, month, day].join("-")}T${[hour, minute, second].join(":")}Z`), + ), + applicationId: appId, + userId: [userIdPart1, userIdPart2, userIdPart3, userIdPart4, userIdPart5].join("-"), + }; + }, +}; + export type AccessInformation = { public?: { token: string; diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 74aebab49..476e9a2b2 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -13,7 +13,7 @@ import { PublicFile } from "../../../services/files/entities/file"; import globalResolver from "../../../services/global-resolver"; import { hasCompanyAdminLevel } from "../../../utils/company"; import gr from "../../global-resolver"; -import { DriveFile, TYPE } from "../entities/drive-file"; +import { DriveFile, EditingSessionKeyFormat, TYPE } from "../entities/drive-file"; import { FileVersion, TYPE as FileVersionType } from "../entities/file-version"; import User, { TYPE as UserType } from "../../user/entities/user"; @@ -60,7 +60,6 @@ import { import archiver from "archiver"; import internal from "stream"; import config from "config"; -import { randomUUID } from "crypto"; import { MultipartFile } from "@fastify/multipart"; import { UploadOptions } from "src/services/files/types"; @@ -970,33 +969,18 @@ export class DocumentsService { editorApplicationId: string, context: DriveExecutionContext, ) => { - const isoUTCDateNoSpecialCharsNoMS = new Date() - .toISOString() - .replace(/\..+$/, "") - .replace(/[ZT:-]/g, ""); - const newKey = [ - isoUTCDateNoSpecialCharsNoMS, - editorApplicationId, - randomUUID().replace(/-+/g, ""), - ].join("-"); - // OnlyOffice key limits: 128 chars, [0-9a-zA-z=_-] - // This is specific to it, but the constraint seems strict enough - // that any other system needing such a unique identifier would find - // this compatible. This value must be ensured to be the strictest - // common denominator to all plugin/interop systems. Plugins that - // require something even stricter have the option of maintaining - // a look up table to an acceptable value. - if (newKey.length > 128 || !/^[0-9a-zA-Z=_]+$/m.test(editorApplicationId)) - CrudException.throwMe( - new Error('Invalid "editorApplicationId" string. Must be short and only alpha numeric'), - new CrudException("Invalid editorApplicationId", 400), - ); - if (!context) { this.logger.error("invalid execution context"); return null; } + let newKey: string; + try { + newKey = EditingSessionKeyFormat.generate(editorApplicationId, context.user.id); + } catch (e) { + CrudException.throwMe(e, new CrudException("Error generating new editing_session_key", 500)); + } + const hasAccess = await checkAccess(id, null, "write", this.repository, context); if (!hasAccess) { logger.error("user does not have access drive item " + id); From 9b0dcc106870dbbbfc9aaa899701e76096a49a96 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 25 Aug 2024 22:26:29 +0200 Subject: [PATCH 11/52] =?UTF-8?q?=E2=9C=85=E2=99=BB=EF=B8=8F=F0=9F=A9=B9?= =?UTF-8?q?=20backend:=20add=20company=5Fid=20to=20editing=5Fsession=5Fkey?= =?UTF-8?q?=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/documents/entities/drive-file.ts | 55 +++++++++++-------- .../src/services/documents/services/index.ts | 6 +- tdrive/backend/node/src/utils/uuid.ts | 32 +++++++++++ .../drive-file-editing-sessions-key.test.ts | 53 ++++++++++++++++++ .../onlyoffice-connector/src/app.ts | 2 +- 5 files changed, 123 insertions(+), 25 deletions(-) create mode 100644 tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts diff --git a/tdrive/backend/node/src/services/documents/entities/drive-file.ts b/tdrive/backend/node/src/services/documents/entities/drive-file.ts index d6f01ccf8..f5bfe9d62 100644 --- a/tdrive/backend/node/src/services/documents/entities/drive-file.ts +++ b/tdrive/backend/node/src/services/documents/entities/drive-file.ts @@ -4,6 +4,7 @@ import { Column, Entity } from "../../../core/platform/services/database/service import { DriveFileAccessLevel, publicAccessLevel } from "../types"; import { FileVersion } from "./file-version"; import search from "./drive-file.search"; +import * as UUIDTools from "../../../utils/uuid"; export const TYPE = "drive_files"; export type DriveScope = "personal" | "shared"; @@ -96,7 +97,8 @@ export class DriveFile { * If this field is non-null, then an editing session is in progress (probably in OnlyOffice). * Use {@see EditingSessionKeyFormat} to generate and interpret it. * Values should ensure that sorting lexicographically is chronological (assuming perfect clocks everywhere), - * and that the application and user that started the edit session are retrievable. + * and that the application, company and user that started the edit session are retrievable. + * It is not encrypted. */ @Type(() => String) @Column("editing_session_key", "string") @@ -122,32 +124,45 @@ export class DriveFile { scope: DriveScope; } +const OnlyOfficeSafeDocKeyBase64 = { + // base64 uses `+/`, but base64url uses `-_` instead. Both use `=` as padding, + // which conflicts with EditingSessionKeyFormat so using `.` instead. + fromBuffer(buffer: Buffer) { + return buffer.toString("base64url").replace(/=/g, "."); + }, + toBuffer(base64: string) { + return Buffer.from(base64.replace(/\./g, "="), "base64url"); + }, +}; + /** Reference implementation for generating then parsing the {@link DriveFile.editing_session_key} field */ export const EditingSessionKeyFormat = { - // OnlyOffice key limits: 128 chars, [0-9a-zA-z=_-] + // OnlyOffice key limits: 128 chars, [0-9a-zA-Z.=_-] + // See https://api.onlyoffice.com/editors/config/document#key // This is specific to it, but the constraint seems strict enough // that any other system needing such a unique identifier would find // this compatible. This value must be ensured to be the strictest // common denominator to all plugin/interop systems. Plugins that // require something even stricter have the option of maintaining // a look up table to an acceptable value. - generate(applicationId: string, userId: string) { + generate(applicationId: string, companyId: string, userId: string, overrideTimeStamp?: Date) { if (!/^[0-9a-zA-Z_-]+$/m.test(applicationId)) throw new Error( `Invalid applicationId string (${JSON.stringify( applicationId, )}). Must be short and only alpha numeric`, ); - const isoUTCDateNoSpecialCharsNoMS = new Date() + const isoUTCDateNoSpecialCharsNoMS = (overrideTimeStamp ?? new Date()) .toISOString() .replace(/\..+$/, "") .replace(/[ZT:-]/g, ""); - const newKey = [ - isoUTCDateNoSpecialCharsNoMS, - applicationId, - userId.replace(/-+/g, ""), - randomUUID().replace(/-+/g, ""), - ].join("="); + const userIdBuffer = UUIDTools.bufferFromUUIDString(userId) as unknown as Uint8Array; + const companyIdBuffer = UUIDTools.bufferFromUUIDString(companyId) as unknown as Uint8Array; + const entropyBuffer = UUIDTools.bufferFromUUIDString(randomUUID()) as unknown as Uint8Array; + const idsString = OnlyOfficeSafeDocKeyBase64.fromBuffer( + Buffer.concat([companyIdBuffer, userIdBuffer, entropyBuffer]), + ); + const newKey = [isoUTCDateNoSpecialCharsNoMS, applicationId, idsString].join("="); if (newKey.length > 128 || !/^[0-9a-zA-Z=_-]+$/m.test(newKey)) throw new Error( `Invalid generated editingSessionKey (${JSON.stringify( @@ -159,14 +174,14 @@ export const EditingSessionKeyFormat = { parse(editingSessionKey: string) { const parts = editingSessionKey.split("="); - const expectedParts = 4; + const expectedParts = 3; if (parts.length !== expectedParts) throw new Error( `Invalid editingSessionKey (${JSON.stringify( editingSessionKey, )}). Expected ${expectedParts} parts`, ); - const [timestampStr, appId, userId, _random] = parts; + const [timestampStr, appId, idsOOBase64String] = parts; const timestampMatch = timestampStr.match( /^(?\d{4})(?\d\d)(?\d\d)(?\d\d)(?\d\d)(?\d\d)$/, ); @@ -177,22 +192,16 @@ export const EditingSessionKeyFormat = { )}). Didn't start with valid timestamp`, ); const { year, month, day, hour, minute, second } = timestampMatch.groups!; - const userIdMatch = userId.match( - /^([a-z0-f]{8})([a-z0-f]{4})([a-z0-f]{4})([a-z0-f]{4})([a-z0-f]{12})$/i, - ); - if (!userIdMatch) - throw new Error( - `Invalid editingSessionKey (${JSON.stringify( - editingSessionKey, - )}). UserID has wrong number of digits`, - ); - const [, userIdPart1, userIdPart2, userIdPart3, userIdPart4, userIdPart5] = userIdMatch; + const idsBuffer = OnlyOfficeSafeDocKeyBase64.toBuffer(idsOOBase64String); + const companyId = UUIDTools.formattedUUIDInBufferArray(idsBuffer, 0); + const userId = UUIDTools.formattedUUIDInBufferArray(idsBuffer, 1); return { timestamp: new Date( Date.parse(`${[year, month, day].join("-")}T${[hour, minute, second].join(":")}Z`), ), applicationId: appId, - userId: [userIdPart1, userIdPart2, userIdPart3, userIdPart4, userIdPart5].join("-"), + companyId, + userId, }; }, }; diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 476e9a2b2..dabc3778c 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -976,7 +976,11 @@ export class DocumentsService { let newKey: string; try { - newKey = EditingSessionKeyFormat.generate(editorApplicationId, context.user.id); + newKey = EditingSessionKeyFormat.generate( + editorApplicationId, + context.company.id, + context.user.id, + ); } catch (e) { CrudException.throwMe(e, new CrudException("Error generating new editing_session_key", 500)); } diff --git a/tdrive/backend/node/src/utils/uuid.ts b/tdrive/backend/node/src/utils/uuid.ts index 31a73b91d..608ad964f 100644 --- a/tdrive/backend/node/src/utils/uuid.ts +++ b/tdrive/backend/node/src/utils/uuid.ts @@ -12,3 +12,35 @@ export function timeuuidToDate(time_str: string): number { return parseInt(time_string, 16); } + +/** Remove `-`s from a formatted UUID to get a hex a string */ +export function hexFromFormatted(uuid: string) { + const result = uuid.replace(/-+/g, ""); + if (result.length !== 32) + throw new Error(`Invalid UUID (${JSON.stringify(uuid)}). Wrong number of digits`); + return result; +} + +/** Add `-`s back in a hex string to make a formatted UUID */ +export function formattedFromHex(hex: string) { + const idMatch = hex.match( + /^([a-z0-f]{8})([a-z0-f]{4})([a-z0-f]{4})([a-z0-f]{4})([a-z0-f]{12})$/i, + ); + if (!idMatch) + throw new Error(`Invalid UUID hex (${JSON.stringify(hex)}). Wrong number of digits`); + const [, ...parts] = idMatch; + return parts.join("-"); +} + +/** Convert a UUID formatted or hex string into a binary buffer */ +export function bufferFromUUIDString(uuidOrHex: string) { + return Buffer.from(hexFromFormatted(uuidOrHex), "hex"); +} + +/** In a buffer with concatenated UUIDs in binary, extract the `index`th and return as formatted UUID */ +export function formattedUUIDInBufferArray(buffer: Buffer, index: number) { + if (buffer.length < (index + 1) * 16) + throw new Error(`Cannot get UUID ${JSON.stringify(index)} because the buffer is too small`); + const slice = buffer.subarray(index * 16, (index + 1) * 16); + return formattedFromHex(slice.toString("hex").toLowerCase()); +} diff --git a/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts b/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts new file mode 100644 index 000000000..89498de0c --- /dev/null +++ b/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts @@ -0,0 +1,53 @@ +import "reflect-metadata"; +import { expect, jest, test, describe, beforeEach, afterEach } from "@jest/globals"; +import { randomUUID } from "crypto"; +import { EditingSessionKeyFormat } from "../../../../../src/services/documents/entities/drive-file"; + +describe('DriveFile EditingSessionKeyFormat', () => { + const mockAppId = 'tdrive_random_application_id'; + const mockCompanyId = randomUUID(); + const mockUserId = randomUUID(); + const mockTimestamp = new Date(); + const mockTimestampWith0MS = mockTimestamp.getTime() - (mockTimestamp.getTime() % 1000); + + const checkIsUUID = (value: string) => { + expect(value).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i); + expect(value).not.toMatch(/^0{8}-0{4}-0{4}-0{4}-0{12}$/i); + }; + + const checkKeyIsOOCompatible = (key: string) => { + // OnlyOffice key limits: see https://api.onlyoffice.com/editors/config/document#key + expect(key).toMatch(/^[0-9a-zA-Z._=-]{1,128}$/); + }; + + test('generates a valid value that can be parsed', async () => { + checkIsUUID(mockUserId); + checkIsUUID(mockCompanyId); + const key = EditingSessionKeyFormat.generate(mockAppId, mockCompanyId, mockUserId, mockTimestamp); + checkKeyIsOOCompatible(key); + const parsed = EditingSessionKeyFormat.parse(key); + expect(parsed.applicationId).toBe(mockAppId); + expect(parsed.userId).toBe(mockUserId); + expect(parsed.companyId).toBe(mockCompanyId); + expect(parsed.timestamp.getTime()).toBe(mockTimestampWith0MS); + }); + + test('generates unique values', async () => { + const key = EditingSessionKeyFormat.generate(mockAppId, mockCompanyId, mockUserId, mockTimestamp); + const key2 = EditingSessionKeyFormat.generate(mockAppId, mockCompanyId, mockUserId, mockTimestamp); + expect(key).not.toBe(key2); + }); + + test('checks the appId', async () => { + expect(() => { + EditingSessionKeyFormat.generate('invalid app id !', mockCompanyId, mockUserId); + }).toThrow('Invalid applicationId string'); + }); + + test('checks final length', async () => { + expect(() => { + const tooLongAppID = new Array(100).join('x'); + EditingSessionKeyFormat.generate(tooLongAppID, mockCompanyId, mockUserId); + }).toThrow('Must be <128 chars,'); + }); +}); \ No newline at end of file diff --git a/tdrive/connectors/onlyoffice-connector/src/app.ts b/tdrive/connectors/onlyoffice-connector/src/app.ts index 58ebb0516..738fb0cf2 100644 --- a/tdrive/connectors/onlyoffice-connector/src/app.ts +++ b/tdrive/connectors/onlyoffice-connector/src/app.ts @@ -41,7 +41,7 @@ class App { }); routes.forEach(route => { - this.app.use(SERVER_PREFIX, route.router); + this.app.use(route.path ?? '/', route.router); }); this.app.get('/health', (_req, res) => { From 483b1b0688319d1dfe32fbbfd566c15b987d851d Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 25 Aug 2024 22:26:30 +0200 Subject: [PATCH 12/52] =?UTF-8?q?=E2=9C=A8=20backend:=20Add=20callback=20t?= =?UTF-8?q?o=20application=20to=20check=20editing=20key=20status=20=20(#52?= =?UTF-8?q?5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/applications-api/index.ts | 68 +++++++++++++++++++ .../backend-callbacks.controller.ts | 48 +++++++++++++ .../src/middlewares/auth.middleware.ts | 15 ++++ .../src/routes/backend-callbacks.route.ts | 21 ++++++ .../src/routes/index.route.ts | 7 +- .../src/routes/onlyoffice.route.ts | 7 +- .../onlyoffice-connector/src/server.ts | 3 +- .../src/services/onlyoffice.service.ts | 19 +++--- 8 files changed, 171 insertions(+), 17 deletions(-) create mode 100644 tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts create mode 100644 tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts diff --git a/tdrive/backend/node/src/services/applications-api/index.ts b/tdrive/backend/node/src/services/applications-api/index.ts index 411b9230b..312566cb6 100644 --- a/tdrive/backend/node/src/services/applications-api/index.ts +++ b/tdrive/backend/node/src/services/applications-api/index.ts @@ -6,6 +6,8 @@ import WebServerAPI from "../../core/platform/services/webserver/provider"; import Application from "../applications/entities/application"; import web from "./web/index"; import { logger } from "../../core/platform/framework/logger"; +import { EditingSessionKeyFormat } from "../documents/entities/drive-file"; +import jwt from "jsonwebtoken"; @Prefix("/api") export default class ApplicationsApiService extends TdriveService { @@ -72,6 +74,72 @@ export default class ApplicationsApiService extends TdriveService { return this; } + /** Send a request to the plugin by its application id + * @param url Full URL that doesn't start with a `/` + */ + private async requestFromApplication( + method: "GET" | "POST" | "DELETE", + url: string, + appId: string, + ) { + const apps = config.get("applications.plugins") || []; + const app = apps.find(app => app.id === appId); + if (!app) throw new Error(`Unknown application.id ${JSON.stringify(appId)}`); + if (!app.internal_domain) + throw new Error(`application.id ${JSON.stringify(appId)} missing an internal_domain`); + const signature = jwt.sign( + { + ts: new Date().getTime(), + type: "tdriveToApplication", + application_id: appId, + }, + app.api.private_key, + ); + const domain = app.internal_domain.replace(/(\/$|^\/)/gm, ""); + const finalURL = `${domain}/${url}${ + url.indexOf("?") > -1 ? "&" : "?" + }token=${encodeURIComponent(signature)}`; + return axios.request({ + url: finalURL, + method: method, + headers: { + Authorization: signature, + }, + maxRedirects: 0, + }); + } + + /** + * Check status of `editing_session_key` in the corresponding application. + * @param editingSessionKey {@see DriveFile.editing_session_key} to check + * @returns a URL string if there is a pending version to add, `null` + * if the key is unknown. + */ + async checkPendingEditingStatus(editingSessionKey: string): Promise { + const parsedKey = EditingSessionKeyFormat.parse(editingSessionKey); + const response = await this.requestFromApplication( + "POST", + "tdriveApi/1/session/" + encodeURIComponent(editingSessionKey) + "/check", + parsedKey.applicationId, + ); + return (response.data.url as string) || null; + } + + /** + * Remove any reference to the `editing_session_key` in the plugin + * @param editingSessionKey {@see DriveFile.editing_session_key} to delete + * @returns `true` if the key was deleted + */ + async deleteEditingKey(editingSessionKey: string): Promise { + const parsedKey = EditingSessionKeyFormat.parse(editingSessionKey); + const response = await this.requestFromApplication( + "DELETE", + "tdriveApi/1/session/" + encodeURIComponent(editingSessionKey), + parsedKey.applicationId, + ); + return !!response.data.done as boolean; + } + // TODO: remove api(): undefined { return undefined; diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts new file mode 100644 index 000000000..45a8801aa --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts @@ -0,0 +1,48 @@ +import { NextFunction, Request, Response } from 'express'; +import logger from '@/lib/logger'; +import onlyofficeService, { CommandError, ErrorCode } from '@/services/onlyoffice.service'; + +interface RequestQuery { + editing_session_key: string; +} + +async function ignoreMissingKeyErrorButNoneElse(res: Response, call: () => Promise): Promise { + try { + await call(); + } catch (e) { + if (e instanceof CommandError && e.errorCode == ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND) { + return void (await res.send({ info: 'Unknown editing_session_key' })); + } + logger.error('Running OO command for TDrive backend', e); + return void (await res.sendStatus(500)); + } +} + +/** + * These routes are called by Twake Drive backend, for ex. before editing or retreiving a file, + * if it has an editing_session_key still, get the status of that and force a resolution. + */ +export default class TwakeDriveBackendCallbackController { + /** + * Get status of an `editing_session_key` from OO, and return a URL to get the latest version, + * or an object with no `url` property, in which case the key is not known as forgotten by OO and should + * be considered lost after an admin alert. + */ + public async checkSessionStatus(req: Request, res: Response): Promise { + //TODO: Find a way to check if the key is live (`info` uses the callback url) + await ignoreMissingKeyErrorButNoneElse(res, async () => { + await res.send({ url: await onlyofficeService.getForgotten(req.params.editing_session_key) }); + }); + } + + /** + * Force deletion of the provided `editing_session_key` in the OO document server. + * If the key was succesfully deleted, the `done` property in the response body will be true. + */ + public async deleteSessionKey(req: Request, res: Response): Promise { + await ignoreMissingKeyErrorButNoneElse(res, async () => { + await onlyofficeService.deleteForgotten(req.params.editing_session_key); + await res.send({ done: true }); + }); + } +} diff --git a/tdrive/connectors/onlyoffice-connector/src/middlewares/auth.middleware.ts b/tdrive/connectors/onlyoffice-connector/src/middlewares/auth.middleware.ts index 8703da88f..5b727c41f 100644 --- a/tdrive/connectors/onlyoffice-connector/src/middlewares/auth.middleware.ts +++ b/tdrive/connectors/onlyoffice-connector/src/middlewares/auth.middleware.ts @@ -34,6 +34,21 @@ export default async (req: Request<{}, {}, {}, RequestQuery>, res: Response, nex return res.status(401).send({ message: 'invalid token' }); } + const authHeaderToken = req.header('authorization'); + try { + if (authHeaderToken) { + const fromTwakeDriveToken = jwt.verify(authHeaderToken, CREDENTIALS_SECRET) as Record; + // The following constant comes from tdrive/backend/node/src/services/applications-api/index.ts + // This is in the case when Twake Drive backend makes requests from the connector, + // but not on the behalf of a specific user, eg. when checking stale only office keys + if (fromTwakeDriveToken['type'] != 'tdriveToApplication') throw new Error(`Bad type in token ${JSON.stringify(fromTwakeDriveToken)}`); + return next(); + } + } catch (e) { + logger.error(`Invalid token in Authorization header from Twake Drive backend`, e); + return res.status(401).json({ message: 'invalid token' }); + } + const user = await userService.getCurrentUser(token); if (!user || !user.id) { diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts new file mode 100644 index 000000000..527eb02e7 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts @@ -0,0 +1,21 @@ +import TwakeDriveBackendCallbackController from '@/controllers/backend-callbacks.controller'; +import { Routes } from '@/interfaces/routes.interface'; +import authMiddleware from '@/middlewares/auth.middleware'; +import { Router } from 'express'; + +/** + * These routes are called by Twake Drive backend, for ex. before editing or retreiving a file, + * if it has an editing_session_key still, get the status of that and force a resolution. + */ +export default class TwakeDriveBackendCallbackRoutes implements Routes { + public readonly router = Router(); + public readonly path = '/tdriveApi/1'; + private readonly controller: TwakeDriveBackendCallbackController; + + constructor() { + this.controller = new TwakeDriveBackendCallbackController(); + // Why post ? to garantee it is never cached and always ran + this.router.post('/session/:editing_session_key/check', authMiddleware, this.controller.checkSessionStatus); + this.router.delete('/session/:editing_session_key', authMiddleware, this.controller.deleteSessionKey); + } +} diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts index 20fb86111..3e99c7198 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts @@ -3,13 +3,14 @@ import { Routes } from '@/interfaces/routes.interface'; import authMiddleware from '@/middlewares/auth.middleware'; import requirementsMiddleware from '@/middlewares/requirements.middleware'; import { Router } from 'express'; +import { SERVER_PREFIX } from '@config'; /** * When the user previews or edits a file in Twake Drive, their browser is sent to these routes * which return a webpage that instantiates the client side JS Only Office component. */ class IndexRoute implements Routes { - public path = '/'; + public path = SERVER_PREFIX; public router = Router(); public indexController: IndexController; @@ -19,8 +20,8 @@ class IndexRoute implements Routes { } private initRoutes = () => { - this.router.get(this.path, requirementsMiddleware, authMiddleware, this.indexController.index); - this.router.get(this.path + 'editor', requirementsMiddleware, authMiddleware, this.indexController.editor); + this.router.get('/', requirementsMiddleware, authMiddleware, this.indexController.index); + this.router.get('/editor', requirementsMiddleware, authMiddleware, this.indexController.editor); }; } diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts index 43aa36831..504c2c7ef 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts @@ -2,9 +2,10 @@ import OnlyOfficeController from '@/controllers/onlyoffice.controller'; import { Routes } from '@/interfaces/routes.interface'; import requirementsMiddleware from '@/middlewares/requirements.middleware'; import { Router } from 'express'; +import { SERVER_PREFIX } from '@config'; class OnlyOfficeRoute implements Routes { - public path = '/'; + public path = SERVER_PREFIX; public router = Router(); public onlyOfficeController: OnlyOfficeController; @@ -14,8 +15,8 @@ class OnlyOfficeRoute implements Routes { } private initRoutes = () => { - this.router.get(`${this.path}:mode/read`, requirementsMiddleware, this.onlyOfficeController.read); - this.router.post(`${this.path}:mode/callback`, requirementsMiddleware, this.onlyOfficeController.ooCallback); + this.router.get(`:mode/read`, requirementsMiddleware, this.onlyOfficeController.read); + this.router.post(`:mode/callback`, requirementsMiddleware, this.onlyOfficeController.ooCallback); }; } diff --git a/tdrive/connectors/onlyoffice-connector/src/server.ts b/tdrive/connectors/onlyoffice-connector/src/server.ts index 139be10f2..0c86358e3 100644 --- a/tdrive/connectors/onlyoffice-connector/src/server.ts +++ b/tdrive/connectors/onlyoffice-connector/src/server.ts @@ -1,7 +1,8 @@ import App from '@/app'; import IndexRoute from './routes/index.route'; import OnlyOfficeRoute from './routes/onlyoffice.route'; +import TwakeDriveBackendCallbacksRoutes from './routes/backend-callbacks.route'; -const app = new App([new IndexRoute(), new OnlyOfficeRoute()]); +const app = new App([new IndexRoute(), new OnlyOfficeRoute(), new TwakeDriveBackendCallbacksRoutes()]); app.listen(); diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index d0f8e07d5..887b63632 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -62,6 +62,15 @@ export namespace Callback { } } +/** For error responses from the {@see CommandService} */ +export class CommandError extends Error { + constructor(public readonly errorCode: ErrorCode, req: any, res: any) { + super( + `OnlyOffice command service error ${ErrorCodeFromValue(errorCode)} (${errorCode}): Requested ${JSON.stringify(req)} got ${JSON.stringify(res)}`, + ); + } +} + /** * Helpers to define the protocol of the OnlyOffice editor service command API * @see https://api.onlyoffice.com/editors/command/ @@ -77,16 +86,6 @@ namespace CommandService { error: Exclude; } - export class CommandError extends Error { - constructor(errorCode: ErrorCode, req: any, res: any) { - super( - `OnlyOffice command service error ${ErrorCodeFromValue(errorCode)} (${errorCode}): Requested ${JSON.stringify(req)} got ${JSON.stringify( - res, - )}`, - ); - } - } - abstract class BaseRequest { constructor(public readonly c: string) {} From afe6562804aadce3756e3927c2ab48b22093a8d0 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Tue, 3 Sep 2024 21:03:19 +0200 Subject: [PATCH 13/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20backend:=20adding=20?= =?UTF-8?q?freeform=20instanceid=20to=20identify=20multiple=20instances=20?= =?UTF-8?q?of=20plugin=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/documents/entities/drive-file.ts | 33 ++++++++++++------- .../src/services/documents/services/index.ts | 2 ++ .../documents/web/controllers/documents.ts | 3 +- .../drive-file-editing-sessions-key.test.ts | 14 ++++---- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/tdrive/backend/node/src/services/documents/entities/drive-file.ts b/tdrive/backend/node/src/services/documents/entities/drive-file.ts index f5bfe9d62..e4e029a59 100644 --- a/tdrive/backend/node/src/services/documents/entities/drive-file.ts +++ b/tdrive/backend/node/src/services/documents/entities/drive-file.ts @@ -135,6 +135,14 @@ const OnlyOfficeSafeDocKeyBase64 = { }, }; +function checkFieldValue(field: string, value: string) { + if (!/^[0-9a-zA-Z_-]+$/m.test(value)) + throw new Error( + `Invalid ${field} value (${JSON.stringify( + value, + )}). Must be short and only alpha numeric or '_' and '-'`, + ); +} /** Reference implementation for generating then parsing the {@link DriveFile.editing_session_key} field */ export const EditingSessionKeyFormat = { // OnlyOffice key limits: 128 chars, [0-9a-zA-Z.=_-] @@ -145,13 +153,15 @@ export const EditingSessionKeyFormat = { // common denominator to all plugin/interop systems. Plugins that // require something even stricter have the option of maintaining // a look up table to an acceptable value. - generate(applicationId: string, companyId: string, userId: string, overrideTimeStamp?: Date) { - if (!/^[0-9a-zA-Z_-]+$/m.test(applicationId)) - throw new Error( - `Invalid applicationId string (${JSON.stringify( - applicationId, - )}). Must be short and only alpha numeric`, - ); + generate( + applicationId: string, + instanceId: string, + companyId: string, + userId: string, + overrideTimeStamp?: Date, + ) { + checkFieldValue("applicationId", applicationId); + checkFieldValue("instanceId", instanceId); const isoUTCDateNoSpecialCharsNoMS = (overrideTimeStamp ?? new Date()) .toISOString() .replace(/\..+$/, "") @@ -162,7 +172,7 @@ export const EditingSessionKeyFormat = { const idsString = OnlyOfficeSafeDocKeyBase64.fromBuffer( Buffer.concat([companyIdBuffer, userIdBuffer, entropyBuffer]), ); - const newKey = [isoUTCDateNoSpecialCharsNoMS, applicationId, idsString].join("="); + const newKey = [isoUTCDateNoSpecialCharsNoMS, applicationId, instanceId, idsString].join("="); if (newKey.length > 128 || !/^[0-9a-zA-Z=_-]+$/m.test(newKey)) throw new Error( `Invalid generated editingSessionKey (${JSON.stringify( @@ -174,14 +184,14 @@ export const EditingSessionKeyFormat = { parse(editingSessionKey: string) { const parts = editingSessionKey.split("="); - const expectedParts = 3; + const expectedParts = 4; if (parts.length !== expectedParts) throw new Error( `Invalid editingSessionKey (${JSON.stringify( editingSessionKey, )}). Expected ${expectedParts} parts`, ); - const [timestampStr, appId, idsOOBase64String] = parts; + const [timestampStr, applicationId, instanceId, idsOOBase64String] = parts; const timestampMatch = timestampStr.match( /^(?\d{4})(?\d\d)(?\d\d)(?\d\d)(?\d\d)(?\d\d)$/, ); @@ -199,7 +209,8 @@ export const EditingSessionKeyFormat = { timestamp: new Date( Date.parse(`${[year, month, day].join("-")}T${[hour, minute, second].join(":")}Z`), ), - applicationId: appId, + applicationId, + instanceId, companyId, userId, }; diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index dabc3778c..61e720d85 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -967,6 +967,7 @@ export class DocumentsService { beginEditing = async ( id: string, editorApplicationId: string, + appInstanceId: string, context: DriveExecutionContext, ) => { if (!context) { @@ -978,6 +979,7 @@ export class DocumentsService { try { newKey = EditingSessionKeyFormat.generate( editorApplicationId, + appInstanceId, context.company.id, context.user.id, ); diff --git a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts index 0084350e3..e794117cc 100644 --- a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts +++ b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts @@ -351,7 +351,7 @@ export class DocumentsController { request: FastifyRequest<{ Params: ItemRequestParams; //TODO application id should be received from the token that we have during the login - Body: { editorApplicationId: string }; + Body: { editorApplicationId: string; instanceId: string }; }>, ) => { try { @@ -366,6 +366,7 @@ export class DocumentsController { return await globalResolver.services.documents.documents.beginEditing( id, request.body.editorApplicationId, + request.body.instanceId || "", context, ); } catch (error) { diff --git a/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts b/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts index 89498de0c..c5e10cac3 100644 --- a/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts +++ b/tdrive/backend/node/test/unit/core/services/documents/drive-file-editing-sessions-key.test.ts @@ -7,6 +7,7 @@ describe('DriveFile EditingSessionKeyFormat', () => { const mockAppId = 'tdrive_random_application_id'; const mockCompanyId = randomUUID(); const mockUserId = randomUUID(); + const mockInstanceId = "super-instance-id"; const mockTimestamp = new Date(); const mockTimestampWith0MS = mockTimestamp.getTime() - (mockTimestamp.getTime() % 1000); @@ -23,31 +24,32 @@ describe('DriveFile EditingSessionKeyFormat', () => { test('generates a valid value that can be parsed', async () => { checkIsUUID(mockUserId); checkIsUUID(mockCompanyId); - const key = EditingSessionKeyFormat.generate(mockAppId, mockCompanyId, mockUserId, mockTimestamp); + const key = EditingSessionKeyFormat.generate(mockAppId, mockInstanceId, mockCompanyId, mockUserId, mockTimestamp); checkKeyIsOOCompatible(key); const parsed = EditingSessionKeyFormat.parse(key); expect(parsed.applicationId).toBe(mockAppId); + expect(parsed.instanceId).toBe(mockInstanceId); expect(parsed.userId).toBe(mockUserId); expect(parsed.companyId).toBe(mockCompanyId); expect(parsed.timestamp.getTime()).toBe(mockTimestampWith0MS); }); test('generates unique values', async () => { - const key = EditingSessionKeyFormat.generate(mockAppId, mockCompanyId, mockUserId, mockTimestamp); - const key2 = EditingSessionKeyFormat.generate(mockAppId, mockCompanyId, mockUserId, mockTimestamp); + const key = EditingSessionKeyFormat.generate(mockAppId, mockInstanceId, mockCompanyId, mockUserId, mockTimestamp); + const key2 = EditingSessionKeyFormat.generate(mockAppId, mockInstanceId, mockCompanyId, mockUserId, mockTimestamp); expect(key).not.toBe(key2); }); test('checks the appId', async () => { expect(() => { - EditingSessionKeyFormat.generate('invalid app id !', mockCompanyId, mockUserId); - }).toThrow('Invalid applicationId string'); + EditingSessionKeyFormat.generate('invalid app id !', mockInstanceId, mockCompanyId, mockUserId); + }).toThrow('Invalid applicationId value'); }); test('checks final length', async () => { expect(() => { const tooLongAppID = new Array(100).join('x'); - EditingSessionKeyFormat.generate(tooLongAppID, mockCompanyId, mockUserId); + EditingSessionKeyFormat.generate(tooLongAppID, mockInstanceId, mockCompanyId, mockUserId); }).toThrow('Must be <128 chars,'); }); }); \ No newline at end of file From 9bc9dcafeefa809bc309ce845013c4bb121f9653 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Wed, 4 Sep 2024 03:47:59 +0200 Subject: [PATCH 14/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20oo-connector:=20refa?= =?UTF-8?q?ctored=20router=20system=20to=20something=20more=20coherent=20(?= =?UTF-8?q?#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../onlyoffice-connector/src/app.ts | 15 +++---- .../onlyoffice-connector/src/config/index.ts | 2 + ...roller.ts => browser-editor.controller.ts} | 13 +++--- .../src/middlewares/error.middleware.ts | 5 +-- .../src/routes/backend-callbacks.route.ts | 21 ++++----- .../src/routes/browser-editor.route.ts | 16 +++++++ .../src/routes/index.route.ts | 28 ------------ .../onlyoffice-connector/src/routes/index.ts | 43 +++++++++++++++++++ .../src/routes/onlyoffice.route.ts | 34 ++++++--------- .../onlyoffice-connector/src/server.ts | 5 +-- 10 files changed, 98 insertions(+), 84 deletions(-) rename tdrive/connectors/onlyoffice-connector/src/controllers/{index.controller.ts => browser-editor.controller.ts} (94%) create mode 100644 tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts delete mode 100644 tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts create mode 100644 tdrive/connectors/onlyoffice-connector/src/routes/index.ts diff --git a/tdrive/connectors/onlyoffice-connector/src/app.ts b/tdrive/connectors/onlyoffice-connector/src/app.ts index 738fb0cf2..95e2d37df 100644 --- a/tdrive/connectors/onlyoffice-connector/src/app.ts +++ b/tdrive/connectors/onlyoffice-connector/src/app.ts @@ -5,24 +5,25 @@ import cors from 'cors'; import { renderFile } from 'eta'; import path from 'path'; import errorMiddleware from './middlewares/error.middleware'; -import { SERVER_PORT, SERVER_PREFIX } from '@config'; +import { SERVER_PORT } from '@config'; import logger from './lib/logger'; import cookieParser from 'cookie-parser'; import apiService from './services/api.service'; import onlyofficeService from './services/onlyoffice.service'; +import { makeURLTo, mountRoutes } from './routes'; class App { public app: express.Application; public env: string; public port: string | number; - constructor(routes: Routes[]) { + constructor() { this.app = express(); this.env = NODE_ENV; this.initViews(); this.initMiddlewares(); - this.initRoutes(routes); + this.initRoutes(); this.initErrorHandling(); } @@ -34,15 +35,13 @@ class App { public getServer = () => this.app; - private initRoutes = (routes: Routes[]) => { + private initRoutes = () => { this.app.use((req, res, next) => { logger.info(`Received request: ${req.method} ${req.originalUrl} from ${req.header('user-agent')} (${req.ip})`); next(); }); - routes.forEach(route => { - this.app.use(route.path ?? '/', route.router); - }); + mountRoutes(this.app); this.app.get('/health', (_req, res) => { Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken(), onlyofficeService.getForgottenList()]).then( @@ -58,7 +57,7 @@ class App { }); this.app.use( - SERVER_PREFIX.replace(/\/$/, '') + '/assets', + makeURLTo.assets(), (req, res, next) => { res.header('Access-Control-Allow-Origin', '*'); res.header('Access-Control-Allow-Headers', 'X-Requested-With'); diff --git a/tdrive/connectors/onlyoffice-connector/src/config/index.ts b/tdrive/connectors/onlyoffice-connector/src/config/index.ts index e25db1b8e..43d33300c 100644 --- a/tdrive/connectors/onlyoffice-connector/src/config/index.ts +++ b/tdrive/connectors/onlyoffice-connector/src/config/index.ts @@ -16,3 +16,5 @@ export const { export const twakeDriveTokenRefrehPeriodMS = 10 * 60 * 1000; export const onlyOfficeForgottenFilesCheckPeriodMS = 10 * 60 * 1000; export const onlyOfficeConnectivityCheckPeriodMS = 10 * 60 * 1000; + +export const SERVER_TDRIVE_API_PREFIX = '/tdriveApi/1'; diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts similarity index 94% rename from tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts rename to tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts index fd05b2af1..c2779403e 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/index.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts @@ -1,13 +1,13 @@ import editorService from '@/services/editor.service'; import { NextFunction, Request, Response } from 'express'; -import { CREDENTIALS_SECRET, SERVER_ORIGIN, SERVER_PREFIX } from '@config'; +import { CREDENTIALS_SECRET } from '@config'; import jwt from 'jsonwebtoken'; import driveService from '@/services/drive.service'; import { DriveFileType } from '@/interfaces/drive.interface'; import fileService from '@/services/file.service'; import { OfficeToken } from '@/interfaces/routes.interface'; import logger from '@/lib/logger'; -import * as Utils from '@/utils'; +import { makeURLTo } from '@/routes'; interface RequestQuery { mode: string; @@ -29,7 +29,7 @@ interface RequestEditorQuery { * The user is redirected from there to open directly the OnlyOffice edition server's web UI, with appropriate preview or not * and rights checks. */ -class IndexController { +class BrowserEditorController { /** * Opened by the user's browser, proxied through the Twake Drive backend. Checks access to the * file with the backend, then redirects the user to the `editor` method but directly on this @@ -98,9 +98,8 @@ class IndexController { expiresIn: 60 * 60 * 24 * 30, }, ); - res.redirect( - Utils.joinURL([SERVER_ORIGIN ?? '', SERVER_PREFIX, 'editor'], { + makeURLTo.editorAbsolute({ token, file_id, editing_session_key: editingSessionKey, @@ -145,7 +144,7 @@ class IndexController { res.render('index', { ...initResponse, docId: preview ? file_id : editing_session_key, - server: Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), + server: makeURLTo.rootAbsolute(), token: inPageToken, }); } catch (error) { @@ -155,4 +154,4 @@ class IndexController { }; } -export default IndexController; +export default BrowserEditorController; diff --git a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts index 49f1c3173..d49eaa17c 100644 --- a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts +++ b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts @@ -1,7 +1,6 @@ import logger from '@/lib/logger'; import { NextFunction, Request, Response } from 'express'; -import * as Utils from '@/utils'; -import { SERVER_ORIGIN, SERVER_PREFIX } from '@config'; +import { makeURLTo } from '@/routes'; export default (error: Error & { status?: number }, req: Request, res: Response, next: NextFunction): void => { try { @@ -12,7 +11,7 @@ export default (error: Error & { status?: number }, req: Request, res: Response, res.status(status); res.render('error', { - server: Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), + server: makeURLTo.rootAbsolute(), errorMessage: message, }); } catch (error) { diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts index 527eb02e7..f5b409279 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts @@ -1,21 +1,16 @@ import TwakeDriveBackendCallbackController from '@/controllers/backend-callbacks.controller'; -import { Routes } from '@/interfaces/routes.interface'; import authMiddleware from '@/middlewares/auth.middleware'; -import { Router } from 'express'; +import type { Router } from 'express'; /** * These routes are called by Twake Drive backend, for ex. before editing or retreiving a file, * if it has an editing_session_key still, get the status of that and force a resolution. */ -export default class TwakeDriveBackendCallbackRoutes implements Routes { - public readonly router = Router(); - public readonly path = '/tdriveApi/1'; - private readonly controller: TwakeDriveBackendCallbackController; - - constructor() { - this.controller = new TwakeDriveBackendCallbackController(); +export const TwakeDriveBackendCallbackRoutes = { + mount(router: Router) { + const controller = new TwakeDriveBackendCallbackController(); // Why post ? to garantee it is never cached and always ran - this.router.post('/session/:editing_session_key/check', authMiddleware, this.controller.checkSessionStatus); - this.router.delete('/session/:editing_session_key', authMiddleware, this.controller.deleteSessionKey); - } -} + router.post('/session/:editing_session_key/check', authMiddleware, controller.checkSessionStatus); + router.delete('/session/:editing_session_key', authMiddleware, controller.deleteSessionKey); + }, +}; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts new file mode 100644 index 000000000..4f02de215 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts @@ -0,0 +1,16 @@ +import BrowserEditorController from '@/controllers/browser-editor.controller'; +import authMiddleware from '@/middlewares/auth.middleware'; +import requirementsMiddleware from '@/middlewares/requirements.middleware'; +import type { Router } from 'express'; + +/** + * When the user previews or edits a file in Twake Drive, their browser is sent to these routes + * which return a webpage that instantiates the client side JS Only Office component. + */ +export const BrowserEditorRoutes = { + mount(router: Router) { + const controller = new BrowserEditorController(); + router.get('/', requirementsMiddleware, authMiddleware, controller.index); + router.get('/editor', requirementsMiddleware, authMiddleware, controller.editor); + }, +}; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts deleted file mode 100644 index 3e99c7198..000000000 --- a/tdrive/connectors/onlyoffice-connector/src/routes/index.route.ts +++ /dev/null @@ -1,28 +0,0 @@ -import IndexController from '@/controllers/index.controller'; -import { Routes } from '@/interfaces/routes.interface'; -import authMiddleware from '@/middlewares/auth.middleware'; -import requirementsMiddleware from '@/middlewares/requirements.middleware'; -import { Router } from 'express'; -import { SERVER_PREFIX } from '@config'; - -/** - * When the user previews or edits a file in Twake Drive, their browser is sent to these routes - * which return a webpage that instantiates the client side JS Only Office component. - */ -class IndexRoute implements Routes { - public path = SERVER_PREFIX; - public router = Router(); - public indexController: IndexController; - - constructor() { - this.indexController = new IndexController(); - this.initRoutes(); - } - - private initRoutes = () => { - this.router.get('/', requirementsMiddleware, authMiddleware, this.indexController.index); - this.router.get('/editor', requirementsMiddleware, authMiddleware, this.indexController.editor); - }; -} - -export default IndexRoute; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/index.ts b/tdrive/connectors/onlyoffice-connector/src/routes/index.ts new file mode 100644 index 000000000..758ee1a7c --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/routes/index.ts @@ -0,0 +1,43 @@ +import { Application, Router } from 'express'; + +import * as Utils from '@/utils'; + +import { TwakeDriveBackendCallbackRoutes } from './backend-callbacks.route'; +import { BrowserEditorRoutes } from './browser-editor.route'; +import { OnlyOfficeRoutes } from './onlyoffice.route'; + +import { SERVER_ORIGIN, SERVER_PREFIX, SERVER_TDRIVE_API_PREFIX } from '@config'; + +export function mountRoutes(app: Application) { + // These routes are forwarded through the Twake Drive front, back and here + const proxiedRouter = Router(); + BrowserEditorRoutes.mount(proxiedRouter); + OnlyOfficeRoutes.mount(proxiedRouter); + console.log('Mounting at ' + SERVER_PREFIX); + app.use(SERVER_PREFIX, proxiedRouter); + + // These endpoints should only be accessible to the Twake Drive backend + const apiRouter = Router(); + console.log('Mounting at ' + SERVER_TDRIVE_API_PREFIX); + TwakeDriveBackendCallbackRoutes.mount(apiRouter); + app.use(SERVER_TDRIVE_API_PREFIX, apiRouter); +} + +export const makeURLTo = { + rootAbsolute: () => Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), + assets: () => Utils.joinURL([SERVER_PREFIX, 'assets']), + editorAbsolute(params: { token: string; file_id: string; editing_session_key: string; company_id: string; preview: string; office_token: string }) { + return Utils.joinURL([SERVER_ORIGIN ?? '', SERVER_PREFIX, 'editor'], params); + }, +}; + +// export function makeURLToEditor2() { +// const initResponse = await editorService.init(company_id, file_name, file_id, user, preview, drive_file_id || file_id); + +// res.render('index', { +// ...initResponse, +// docId: preview ? file_id : editing_session_key, +// server: Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), +// token: inPageToken, +// }); +// } diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts index 504c2c7ef..0392cc3ce 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts @@ -1,23 +1,15 @@ import OnlyOfficeController from '@/controllers/onlyoffice.controller'; -import { Routes } from '@/interfaces/routes.interface'; import requirementsMiddleware from '@/middlewares/requirements.middleware'; -import { Router } from 'express'; -import { SERVER_PREFIX } from '@config'; - -class OnlyOfficeRoute implements Routes { - public path = SERVER_PREFIX; - public router = Router(); - public onlyOfficeController: OnlyOfficeController; - - constructor() { - this.onlyOfficeController = new OnlyOfficeController(); - this.initRoutes(); - } - - private initRoutes = () => { - this.router.get(`:mode/read`, requirementsMiddleware, this.onlyOfficeController.read); - this.router.post(`:mode/callback`, requirementsMiddleware, this.onlyOfficeController.ooCallback); - }; -} - -export default OnlyOfficeRoute; +import type { Router } from 'express'; + +/** + * These routes are called by the Only Office server + * when reading a document or updating an editing session + */ +export const OnlyOfficeRoutes = { + mount(router: Router) { + const controller = new OnlyOfficeController(); + router.get(`/:mode/read`, requirementsMiddleware, controller.read); + router.post(`/:mode/callback`, requirementsMiddleware, controller.ooCallback); + }, +}; diff --git a/tdrive/connectors/onlyoffice-connector/src/server.ts b/tdrive/connectors/onlyoffice-connector/src/server.ts index 0c86358e3..f743c935d 100644 --- a/tdrive/connectors/onlyoffice-connector/src/server.ts +++ b/tdrive/connectors/onlyoffice-connector/src/server.ts @@ -1,8 +1,5 @@ import App from '@/app'; -import IndexRoute from './routes/index.route'; -import OnlyOfficeRoute from './routes/onlyoffice.route'; -import TwakeDriveBackendCallbacksRoutes from './routes/backend-callbacks.route'; -const app = new App([new IndexRoute(), new OnlyOfficeRoute(), new TwakeDriveBackendCallbacksRoutes()]); +const app = new App(); app.listen(); From 996c7a9a61a1b3681f440c013b29f80c3af2000b Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 5 Sep 2024 13:27:40 +0200 Subject: [PATCH 15/52] =?UTF-8?q?=F0=9F=92=84=20oo-connector:=20Fixed=20er?= =?UTF-8?q?ror=20page=20to=20show=20500,=20correct=20e-mail,=20removed=20G?= =?UTF-8?q?A=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/middlewares/error.middleware.ts | 1 - .../onlyoffice-connector/src/views/error.eta | 37 +++++++++---------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts index d49eaa17c..b452275a5 100644 --- a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts +++ b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts @@ -11,7 +11,6 @@ export default (error: Error & { status?: number }, req: Request, res: Response, res.status(status); res.render('error', { - server: makeURLTo.rootAbsolute(), errorMessage: message, }); } catch (error) { diff --git a/tdrive/connectors/onlyoffice-connector/src/views/error.eta b/tdrive/connectors/onlyoffice-connector/src/views/error.eta index b82ed3c31..d3aa93103 100644 --- a/tdrive/connectors/onlyoffice-connector/src/views/error.eta +++ b/tdrive/connectors/onlyoffice-connector/src/views/error.eta @@ -5,7 +5,7 @@ - 404 HTML Tempate by Colorlib + 500 Internal Server Error @@ -186,26 +199,10 @@

500

We are sorry, internal server error!

-

Please, contact our support at support@twake.com.

- Back To Homepage +

Please, contact our support at :

+ support@twake.app - - - From 66525d7e482542f49af712020f2cb021589e16ca Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sat, 14 Sep 2024 22:29:50 +0200 Subject: [PATCH 16/52] =?UTF-8?q?=F0=9F=90=9B=20backend:=20fix=20bug=20wit?= =?UTF-8?q?h=20postgres=20support=20when=20uploading=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdrive/backend/node/src/services/documents/services/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 28f32018b..e6779ab4a 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -227,7 +227,7 @@ export class DocumentsService { if (options?.pagination) { const { page_token, limitStr } = options.pagination; const pageNumber = - dbType === "mongodb" ? parseInt(page_token) : parseInt(page_token) / parseInt(limitStr) + 1; + dbType === "mongodb" ? parseInt(page_token) : parseInt(page_token) / parseInt(limitStr); pagination = new Pagination(`${pageNumber}`, `${limitStr}`, false); } @@ -441,7 +441,7 @@ export class DocumentsService { ); // TODO: notify the user a document has been added to the directory shared with them try { - if (driveItem.parent_id !== "root" && driveItem.parent_id !== "trash") { + if (!isVirtualFolder(driveItem.parent_id)) { const parentItem = await this.repository.findOne( { id: driveItem.parent_id, From 53b34a0962836d8fde9a1b96bc6fbd2242d6086e Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sat, 14 Sep 2024 23:58:01 +0200 Subject: [PATCH 17/52] =?UTF-8?q?=E2=9C=A8=20backend:=20Adding=20support?= =?UTF-8?q?=20for=20select=20based=20on=20null=20values=20and=20negation?= =?UTF-8?q?=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../postgres/postgres-query-builder.ts | 15 ++++++-- .../postgres/postgres-query-builder.test.ts | 36 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts index 7d13f0953..441ff0b08 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts @@ -47,9 +47,18 @@ export class PostgresQueryBuilder { values.push(...inClause); } } else { - const value = `${this.dataTransformer.toDbString(filter, columnsDefinition[key].type)}`; - whereClause += `${key} = $${idx++} AND `; - values.push(value); + const isANotEqualFilter = filter && Object.keys(filter).join("") === "$ne"; + if (filter === null || (isANotEqualFilter && filter["$ne"] === null)) { + whereClause += `${key} IS${filter === null ? "" : " NOT"} NULL`; + } else { + const filterValue = isANotEqualFilter ? filter["$ne"] : filter; + const value = `${this.dataTransformer.toDbString( + filterValue, + columnsDefinition[key].type, + )}`; + whereClause += `${key} ${isANotEqualFilter ? "!=" : "="} $${idx++} AND `; + values.push(value); + } } }); } diff --git a/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts b/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts index 62f7e846d..6145b1d95 100644 --- a/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts +++ b/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres-query-builder.test.ts @@ -105,6 +105,42 @@ describe('The PostgresQueryBuilder', () => { expect(query[1]).toEqual(options.$like.map(e=> `%${e[1]}%`)); }); + test('buildSelect query with IS NULL filter', async () => { + //given + const filters = { companu_id: null }; + + //when + const query = subj.buildSelect(TestDbEntity, filters, {}); + + //then + expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM \"test_table\" WHERE companu_id IS NULL ORDER BY id DESC LIMIT 100 OFFSET 0`); + expect(query[1]).toEqual([]); + }); + + test('buildSelect query with not equal filter', async () => { + //given + const filters = { company_id: { $ne: '123' } }; + + //when + const query = subj.buildSelect(TestDbEntity, filters, {}); + + //then + expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE company_id != $1 ORDER BY id DESC LIMIT 100 OFFSET 0`); + expect(query[1]).toEqual([ '123' ]); + }); + + test('buildSelect query with IS NOT NULL filter', async () => { + //given + const filters = { company_id: { $ne: null } }; + + //when + const query = subj.buildSelect(TestDbEntity, filters, {}); + + //then + expect(normalizeWhitespace(query[0] as string)).toBe(`SELECT * FROM "test_table" WHERE company_id IS NOT NULL ORDER BY id DESC LIMIT 100 OFFSET 0`); + expect(query[1]).toEqual([]); + }); + test('buildDelete query', async () => { //given const entity = newTestDbEntity(); From 1e16e179b017792037eb29b7eeb51242cb2362cc Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 15 Sep 2024 00:03:49 +0200 Subject: [PATCH 18/52] =?UTF-8?q?=E2=9C=A8=20backend=20cli:=20add=20editin?= =?UTF-8?q?g=5Fsession=20list=20viewer=20command=20(and=20minor=20instance?= =?UTF-8?q?Id=20fixes)=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/cli/cmds/editing_session_cmds/list.ts | 54 +++++++++++++++++++ .../services/documents/entities/drive-file.ts | 12 +++-- .../src/services/documents/services/index.ts | 2 + 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts diff --git a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts new file mode 100644 index 000000000..98c4bd484 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts @@ -0,0 +1,54 @@ +import yargs from "yargs"; + +import runWithPlatform from "../../lib/run-with-platform"; +import { TdrivePlatform } from "../../../core/platform/platform"; +import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; +import { + DriveFile, + EditingSessionKeyFormat, + TYPE, +} from "../../../services/documents/entities/drive-file"; + +async function report(platform: TdrivePlatform) { + const drivesRepo = await platform + .getProvider("database") + .getRepository(TYPE, DriveFile); + const editedFiled = (await drivesRepo.find({ editing_session_key: { $ne: null } })).getEntities(); + console.error("DriveFiles with non null editing_session_key (url encoded):"); + console.error(""); + editedFiled.forEach(dfile => { + console.error(`- ${dfile.name} (${dfile.id}) has key:`); + const parsed = EditingSessionKeyFormat.parse(dfile.editing_session_key); + console.error(` - URL encoded: ${encodeURIComponent(dfile.editing_session_key)}`); + console.error(` - applicationId: ${parsed.applicationId}`); + console.error(` - companyId: ${parsed.companyId}`); + console.error(` - instanceId: ${parsed.instanceId}`); + console.error( + ` - userId: ${parsed.userId} (${ + parsed.userId === dfile.creator ? "same as creator ID" : "not the creator" + })`, + ); + console.error( + ` - timestamp: ${parsed.timestamp.toISOString()} (${Math.floor( + (new Date().getTime() - parsed.timestamp.getTime()) / 1000, + )}s ago)`, + ); + }); + if (!editedFiled.length) console.error(" (no DriveFile currently has an editing_session_key)"); +} + +const command: yargs.CommandModule = { + command: "list", + describe: ` + List current DriveFile items that have an editing_session_key set + `.trim(), + builder: {}, + handler: async _argv => { + await runWithPlatform("editing_session list", async ({ spinner: _spinner, platform }) => { + console.error("\n"); + await report(platform); + console.error("\n"); + }); + }, +}; +export default command; diff --git a/tdrive/backend/node/src/services/documents/entities/drive-file.ts b/tdrive/backend/node/src/services/documents/entities/drive-file.ts index e4e029a59..5a793abc6 100644 --- a/tdrive/backend/node/src/services/documents/entities/drive-file.ts +++ b/tdrive/backend/node/src/services/documents/entities/drive-file.ts @@ -135,7 +135,8 @@ const OnlyOfficeSafeDocKeyBase64 = { }, }; -function checkFieldValue(field: string, value: string) { +function checkFieldValue(field: string, value: string, required: boolean = true) { + if (!required && !value) return; if (!/^[0-9a-zA-Z_-]+$/m.test(value)) throw new Error( `Invalid ${field} value (${JSON.stringify( @@ -143,7 +144,12 @@ function checkFieldValue(field: string, value: string) { )}). Must be short and only alpha numeric or '_' and '-'`, ); } -/** Reference implementation for generating then parsing the {@link DriveFile.editing_session_key} field */ +/** + * Reference implementation for generating then parsing the {@link DriveFile.editing_session_key} field. + * + * Fields should be explicit, `instanceId` is for the case when we have multiple + * clients + */ export const EditingSessionKeyFormat = { // OnlyOffice key limits: 128 chars, [0-9a-zA-Z.=_-] // See https://api.onlyoffice.com/editors/config/document#key @@ -161,7 +167,7 @@ export const EditingSessionKeyFormat = { overrideTimeStamp?: Date, ) { checkFieldValue("applicationId", applicationId); - checkFieldValue("instanceId", instanceId); + checkFieldValue("instanceId", instanceId, false); const isoUTCDateNoSpecialCharsNoMS = (overrideTimeStamp ?? new Date()) .toISOString() .replace(/\..+$/, "") diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index cf8be32a1..89027fc2e 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -959,6 +959,8 @@ export class DocumentsService { * with only that key provided. * @param id DriveFile ID of the document to begin editing * @param editorApplicationId Editor/Application/Plugin specific identifier + * @param appInstanceId For that `editorApplicationId` a unique identifier + * when multiple instances are running. Unused today. * @param context * @returns An object in the format `{}` with the unique identifier for the * editing session From 22f04a5eebe9af8ccf27330df0f301fc8428c608 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 15 Sep 2024 00:33:36 +0200 Subject: [PATCH 19/52] =?UTF-8?q?=F0=9F=90=9B=20ooconnector:=20minor=20err?= =?UTF-8?q?or=20page=20dark=20mode=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdrive/connectors/onlyoffice-connector/src/views/error.eta | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tdrive/connectors/onlyoffice-connector/src/views/error.eta b/tdrive/connectors/onlyoffice-connector/src/views/error.eta index d3aa93103..8864a3e90 100644 --- a/tdrive/connectors/onlyoffice-connector/src/views/error.eta +++ b/tdrive/connectors/onlyoffice-connector/src/views/error.eta @@ -177,6 +177,10 @@ } @media (prefers-color-scheme: dark) { + body { + background-color: black; + } + .notfound .notfound-404 h1 { color: #444444; } From 1d81c94d479eeab568d3063529f815c2136dc2f7 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 15 Sep 2024 01:19:05 +0200 Subject: [PATCH 20/52] =?UTF-8?q?=F0=9F=A9=B9=20ooconnector:=20using=20use?= =?UTF-8?q?r=20token=20for=20post=20to=20backend=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../onlyoffice-connector/src/config/index.ts | 11 +++++++---- .../src/controllers/browser-editor.controller.ts | 2 +- .../onlyoffice-connector/src/services/api.service.ts | 3 ++- .../src/services/drive.service.ts | 4 ++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/config/index.ts b/tdrive/connectors/onlyoffice-connector/src/config/index.ts index 43d33300c..a312ed134 100644 --- a/tdrive/connectors/onlyoffice-connector/src/config/index.ts +++ b/tdrive/connectors/onlyoffice-connector/src/config/index.ts @@ -4,6 +4,7 @@ config({ path: `.env.${process.env.NODE_ENV || 'development'}.local` }); export const { NODE_ENV, SERVER_PORT, + SERVER_BIND, SECRET_KEY, CREDENTIALS_ENDPOINT, ONLY_OFFICE_SERVER, @@ -13,8 +14,10 @@ export const { SERVER_ORIGIN, } = process.env; -export const twakeDriveTokenRefrehPeriodMS = 10 * 60 * 1000; -export const onlyOfficeForgottenFilesCheckPeriodMS = 10 * 60 * 1000; -export const onlyOfficeConnectivityCheckPeriodMS = 10 * 60 * 1000; +const secs = 1000, + mins = 60 * secs; -export const SERVER_TDRIVE_API_PREFIX = '/tdriveApi/1'; +export const twakeDriveTokenRefrehPeriodMS = 10 * mins; +export const onlyOfficeForgottenFilesCheckPeriodMS = 10 * mins; +export const onlyOfficeConnectivityCheckPeriodMS = 10 * mins; +export const onlyOfficeCallbackTimeoutMS = 10 * secs; diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts index c2779403e..e1bcf19f5 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts @@ -76,7 +76,7 @@ class BrowserEditorController { let editingSessionKey = null; if (!preview) { - editingSessionKey = await driveService.beginEditingSession(company_id, drive_file_id); + editingSessionKey = await driveService.beginEditingSession(company_id, drive_file_id, token); //TODO catch error and display to the user when we can't stopped editing //TODO Log error with format to be able to set up grafana alert fir such king of errors diff --git a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts index a509cf5cb..db046012e 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts @@ -91,7 +91,7 @@ class ApiService implements IApiService { }; public post = async (params: IApiServiceRequestParams): Promise => { - const { url, payload, headers } = params; + const { url, token, payload, headers } = params; const axiosWithToken = await this.requireAxios(); @@ -99,6 +99,7 @@ class ApiService implements IApiService { return await axiosWithToken.post(url, payload, { headers: { ...headers, + ...(token ? { Authorization: `Bearer ${token}` } : {}), }, }); } catch (error) { diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index 240463538..bf93d75ad 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -49,12 +49,12 @@ class DriveService implements IDriveService { } }; - public async beginEditingSession(company_id: string, drive_file_id: string) { + public async beginEditingSession(company_id: string, drive_file_id: string, user_token?: string) { try { const resource = await apiService.post<{}, { editingSessionKey: string }>({ url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}/editing_session`, payload: { - editorApplicationId: 'mock_application_id', + editorApplicationId: 'tdrive_onlyoffice', }, }); if (resource?.editingSessionKey) { From 5cf1fa581e724be8777643bec7e498984fcc954e Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 15 Sep 2024 01:25:59 +0200 Subject: [PATCH 21/52] =?UTF-8?q?=E2=9C=A8=20ooconnector:=20callback=20com?= =?UTF-8?q?mand=20retreival=20system=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/controllers/onlyoffice.controller.ts | 18 ++- .../src/lib/pending-request-matcher.ts | 106 ++++++++++++++++++ .../src/services/onlyoffice.service.ts | 89 ++++++++++++++- 3 files changed, 209 insertions(+), 4 deletions(-) create mode 100644 tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index fcd16c846..1605dcfc8 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -6,6 +6,7 @@ import logger from '@/lib/logger'; import * as OnlyOffice from '@/services/onlyoffice.service'; import { Request, Response, NextFunction } from 'express'; import jwt from 'jsonwebtoken'; +import * as Utils from '@/utils'; interface RequestQuery { company_id: string; @@ -63,11 +64,24 @@ class OnlyOfficeController { try { const { url, key } = req.body; const { token } = req.query; - logger.info('OO callback', req.body); - + logger.info( + `OO callback status: ${Utils.getKeyForValueSafe( + req.body.status, + OnlyOffice.Callback.Status, + 'OnlyOffice.Callback.Status', + )} - ${JSON.stringify(req.body.userdata)}`, + req.body, + ); const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; const { preview, company_id, file_id, /* user_id, */ drive_file_id, in_page_token, editing_session_key } = officeTokenPayload; + // Ignore errors generated by pending request + // try-catch not needed because it is async + // there may be later reasons to wait for callbacks + // to process and eventually respond accordingly to + // OO an error for certain statuses + void OnlyOffice.default.ooCallbackCalled(req.body); + // check token is an in_page_token and allow save if (!in_page_token) throw new Error('Invalid token, must be a in_page_token'); if (preview) throw new Error('Invalid token, must not be a preview token for save operation'); diff --git a/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts b/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts new file mode 100644 index 000000000..c739d47d5 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts @@ -0,0 +1,106 @@ +import logger from './logger'; + +export type PendingRequestCallback = (timeout: boolean, result?: TResult) => Promise; + +/** Represents a single pending query that can be resolved with success or failure */ +class PendingRequest { + private readonly startedAt = new Date(); + private callbacks?: PendingRequestCallback[] = []; + + constructor(public readonly key: string, public readonly userdata: string, callback: PendingRequestCallback) { + this.addCallback(callback); + } + + public addCallback(callback: PendingRequestCallback) { + if (!this.callbacks) throw new Error("Cannot add callback to PendingRequest after it's been resolved"); + this.callbacks.push(callback); + return this; + } + + public getAge() { + return new Date().getTime() - this.startedAt.getTime(); + } + + public matches(key: string, userdata: string) { + return this.key === key && this.userdata === userdata; + } + + public async resolve(...cbArgs: Parameters>) { + if (!this.callbacks) throw new Error("Cannot resolve PendingRequest after it's already been resolved"); + const callbacks = this.callbacks; + this.callbacks = null; + return Promise.all(callbacks.map(fn => fn(...cbArgs))); + } +} + +/** + * Tracks a list of pending requests that are started with a pair + * of `key` and `userdata` strings. (Note. `userdata` is not the + * typical use of user data as a `void *`, but instead used to + * identify the specific request asynchroneously). + * Both must match. + * + * There is no timing garantee as to when expired requests' callbacks + * are executed. + */ +export class PendingRequestQueue { + private queue: PendingRequest[] = []; + + constructor(private readonly timeoutMs: number, niquystSamplingRatio: 4) { + setInterval(() => { + this.flush(); + }, timeoutMs / niquystSamplingRatio); + } + + /** + * Add a callback to be called when `gotResult` or `cancelPending` are + * called with identical `key` and `userdata` + * @param key First half of identifying string + * @param userdata Second half of identifying string + * @param callback When the request is resolved, succesfully or in error + */ + public enqueue(key: string, userdata: string, callback: PendingRequestCallback) { + const existing = this.queue.find(pending => pending.key === key && pending.userdata === userdata); + if (existing) return void existing.addCallback(callback); + this.queue.push(new PendingRequest(key, userdata, callback)); + } + + private remove(predicate: (request: PendingRequest) => boolean): PendingRequest[] { + // Warning: This operation must be synchroneous + const foundRequests = []; + this.queue = this.queue.filter(pending => { + if (!predicate(pending)) return true; + foundRequests.push(pending); + return false; + }); + return foundRequests; + } + + private async resolve(predicate: (request: PendingRequest) => boolean, ...cbArgs: Parameters>) { + const foundRequests = this.remove(predicate); + if (!foundRequests.length) return null; + return Promise.all(foundRequests.map(request => request.resolve(...cbArgs))); + } + + protected async flush() { + const expiredRequests = this.remove(request => request.getAge() > this.timeoutMs); + return Promise.all(expiredRequests.map(request => request.resolve(true))); + } + + public async gotResult(key: string, userdata: string, result?: TResult) { + const resolutions = this.resolve(request => request.matches(key, userdata), false, result); + if (!resolutions) + logger.error(`Got resolution on pending request that was unknown`, { + key, + userdata, + result, + }); + await this.flush(); + return resolutions; + } + + /** Equivalent to `gotResult` with an `undefined` result. (Callbacks are called) */ + public async cancelPending(key: string, userdata: string) { + return this.gotResult(key, userdata, undefined); + } +} diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index 887b63632..9b67f2fda 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -1,6 +1,8 @@ +import { randomUUID } from 'crypto'; import axios from 'axios'; -import { ONLY_OFFICE_SERVER, onlyOfficeConnectivityCheckPeriodMS } from '@config'; +import { ONLY_OFFICE_SERVER, onlyOfficeConnectivityCheckPeriodMS, onlyOfficeCallbackTimeoutMS } from '@config'; import { PolledThingieValue } from '@/lib/polled-thingie-value'; +import { PendingRequestQueue, PendingRequestCallback } from '@/lib/pending-request-matcher'; import logger from '@/lib/logger'; import * as Utils from '@/utils'; @@ -14,6 +16,12 @@ export enum ErrorCode { COMMAND_NOT_CORRECT = 5, INVALID_TOKEN = 6, } + +/** Error generated by this connector. Should be string to share field with {@link ErrorCode} */ +export enum ConnectorErrorCode { + CALLBACK_TIMEOUT = 'callback_timeout', +} + /** Return the name of the error code in the `ErrorCode` enum if recognised, or a descript string */ export const ErrorCodeFromValue = (value: number) => Utils.getKeyForValueSafe(value, ErrorCode, 'OnlyOffice.ErrorCode'); @@ -59,6 +67,7 @@ export namespace Callback { url?: string; actions?: Action[]; users?: string[]; + userdata?: string; } } @@ -93,7 +102,7 @@ namespace CommandService { async postUnsafe(): Promise { logger.silly(`OnlyOffice command ${this.c} sent: ${JSON.stringify(this)}`); const result = await axios.post(Utils.joinURL([ONLY_OFFICE_SERVER, 'coauthoring/CommandService.ashx']), this); - logger.info(`OnlyOffice command ${this.c} response: ${result.status}: ${JSON.stringify(result.data)}`); + logger.info(`OnlyOffice command ${this.c} response ${result.status}: ${ErrorCodeFromValue(result.data.error)}: ${JSON.stringify(result.data)}`); return result.data as ErrorResponse | TSuccessResponse; } @@ -174,6 +183,32 @@ namespace CommandService { } } } + + export namespace Info { + export type Response = SuccessResponse; + export class Request extends BaseRequest { + constructor(public readonly key: string, public readonly userdata: string = '') { + super('info'); + } + } + } +} + +/** + * This object holds possible outcomes for commands like `info` who's result + * is sent to the callback instead of replied to the request + */ +class CallbackResponseFromCommand { + /** + * If the `info` command returned an error code, this is it, or it can + * be an internal error to this connector (eg.: timeout waiting for callback). + * (If the `info` command returned {@link ErrorCode.SUCCESS} then this field will be `undefined`) + */ + public readonly error: Exclude | ConnectorErrorCode | undefined; + + public constructor(error: ErrorCode | ConnectorErrorCode, public readonly result?: Callback.Parameters) { + this.error = error === ErrorCode.SUCCESS ? undefined : error; + } } /** @@ -182,6 +217,8 @@ namespace CommandService { */ class OnlyOfficeService { private readonly poller: PolledThingieValue; + // Technically the timeout field is from the PendingRequestQueue but avoid 2 classes + private readonly pendingRequests = new PendingRequestQueue(onlyOfficeCallbackTimeoutMS); constructor() { this.poller = new PolledThingieValue( @@ -227,6 +264,20 @@ class OnlyOfficeService { return deleted; } + /** Generates and returns a random UUID that has a matching pending task enqueued for */ + private enqueuePendingCallback(key: string, callback: PendingRequestCallback): string { + const userdata = randomUUID(); + this.pendingRequests.enqueue(key, userdata, callback); + return userdata; + } + + /** Called by the OnlyOffice controller when the OO document editing services uses our callback */ + async ooCallbackCalled(result: Callback.Parameters) { + if (!result.userdata) return; + logger.info('OO Callback pending request response received', result); + return this.pendingRequests.gotResult(result.key, result.userdata, new CallbackResponseFromCommand(ErrorCode.SUCCESS, result)); + } + // Note that `async` is important in the functions below. While they avoid the overhead // of `await`, the `async` is still required to catch the throw in `.post()` @@ -239,6 +290,40 @@ class OnlyOfficeService { //TODO: When typing the response more fully, don't return the response object itself as here return new CommandService.License.Request().post(); } + + /** + * Requests a document status and the list of the identifiers of the users who opened the document for editing. + * The response will be sent to the callback handler. + * This method just sends the command. The response from the callback will be ignored if + * this called by itself + * + * *Warning*: returns non succesful error codes instead of throwing errors like the other + * methods. This is because the immediate response is likely to be itself usefull + * for detecting errors. + */ + async getInfoUnsafe(key: string, userdata: string): Promise { + return await new CommandService.Info.Request(key, userdata).postUnsafe().then(({ error }) => error); + } + + async getInfoAndWaitForCallbackUnsafe(key: string): Promise { + // const userdata = randomUUID(); + return new Promise((resolve, reject) => { + const userdata = this.enqueuePendingCallback(key, async (timeout, result) => { + // The callback has called, unless timeout = true, or result is undefined (cancelled request) + return resolve(timeout ? new CallbackResponseFromCommand(ConnectorErrorCode.CALLBACK_TIMEOUT) : result); + }); + void this.getInfoUnsafe(key, userdata).then( + response => { + // The command service responded to the `info` command request + if (response !== ErrorCode.SUCCESS) return this.pendingRequests.gotResult(key, userdata, new CallbackResponseFromCommand(response)); + // If it succeded, just wait for timeout or resolution by the callback + }, + error => { + this.pendingRequests.cancelPending(key, userdata).then(() => reject(error)); + }, + ); + }); + } /** Force a save in the editing session key provided. `userdata` will be forwarded to the callback */ async forceSave(key: string, userdata = ''): Promise { return new CommandService.ForceSave.Request(key, userdata).post().then(response => response.key); From 87d0de92d8cffa99c9bcf2e3cf71bb861850b6a4 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 15 Sep 2024 01:41:48 +0200 Subject: [PATCH 22/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20ooconnector:=20routi?= =?UTF-8?q?ng=20refactor=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../onlyoffice-connector/src/app.ts | 43 ++++--------------- .../controllers/browser-editor.controller.ts | 2 +- .../controllers/health-status.controller.ts | 21 +++++++++ .../src/controllers/onlyoffice.controller.ts | 2 +- .../src/interfaces/drive.interface.ts | 2 +- ...interface.ts => office-token.interface.ts} | 7 --- .../onlyoffice-connector/src/routes/index.ts | 38 ++++++++-------- 7 files changed, 54 insertions(+), 61 deletions(-) create mode 100644 tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts rename tdrive/connectors/onlyoffice-connector/src/interfaces/{routes.interface.ts => office-token.interface.ts} (68%) diff --git a/tdrive/connectors/onlyoffice-connector/src/app.ts b/tdrive/connectors/onlyoffice-connector/src/app.ts index 95e2d37df..9b5f1bd26 100644 --- a/tdrive/connectors/onlyoffice-connector/src/app.ts +++ b/tdrive/connectors/onlyoffice-connector/src/app.ts @@ -1,16 +1,13 @@ +import path from 'path'; import express from 'express'; -import { Routes } from '@interfaces/routes.interface'; -import { NODE_ENV } from '@config'; import cors from 'cors'; import { renderFile } from 'eta'; -import path from 'path'; -import errorMiddleware from './middlewares/error.middleware'; -import { SERVER_PORT } from '@config'; -import logger from './lib/logger'; import cookieParser from 'cookie-parser'; -import apiService from './services/api.service'; -import onlyofficeService from './services/onlyoffice.service'; -import { makeURLTo, mountRoutes } from './routes'; + +import { NODE_ENV, SERVER_BIND, SERVER_PORT } from '@config'; +import logger from './lib/logger'; +import errorMiddleware from './middlewares/error.middleware'; +import { mountRoutes } from './routes'; class App { public app: express.Application; @@ -28,8 +25,9 @@ class App { } public listen = () => { - this.app.listen(parseInt(SERVER_PORT, 10), '0.0.0.0', () => { - logger.info(`🚀 App listening on port ${SERVER_PORT}`); + const binding_host = SERVER_BIND || '0.0.0.0'; + this.app.listen(parseInt(SERVER_PORT, 10), binding_host, () => { + logger.info(`🚀 App listening at http://${binding_host}:${SERVER_PORT}`); }); }; @@ -42,29 +40,6 @@ class App { }); mountRoutes(this.app); - - this.app.get('/health', (_req, res) => { - Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken(), onlyofficeService.getForgottenList()]).then( - ([onlyOfficeLicense, twakeDriveToken, forgottenKeys]) => - res.status(onlyOfficeLicense && twakeDriveToken ? 200 : 500).send({ - uptime: process.uptime(), - onlyOfficeLicense, - twakeDriveToken, - forgottenKeys, - }), - err => res.status(500).send(err), - ); - }); - - this.app.use( - makeURLTo.assets(), - (req, res, next) => { - res.header('Access-Control-Allow-Origin', '*'); - res.header('Access-Control-Allow-Headers', 'X-Requested-With'); - next(); - }, - express.static(path.join(__dirname, '../assets')), - ); }; private initMiddlewares = () => { diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts index e1bcf19f5..a32b98b43 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts @@ -5,7 +5,7 @@ import jwt from 'jsonwebtoken'; import driveService from '@/services/drive.service'; import { DriveFileType } from '@/interfaces/drive.interface'; import fileService from '@/services/file.service'; -import { OfficeToken } from '@/interfaces/routes.interface'; +import { OfficeToken } from '@/interfaces/office-token.interface'; import logger from '@/lib/logger'; import { makeURLTo } from '@/routes'; diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts new file mode 100644 index 000000000..88ccb136a --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts @@ -0,0 +1,21 @@ +import { NextFunction, Request, Response } from 'express'; +import onlyofficeService from '@/services/onlyoffice.service'; +import apiService from '@/services/api.service'; + +/** + * Health response for devops operational purposes. Should not be exposed. + */ +export const HealthStatusController = { + async get(req: Request<{}, {}, {}, {}>, res: Response, next: NextFunction): Promise { + Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken(), onlyofficeService.getForgottenList()]).then( + ([onlyOfficeLicense, twakeDriveToken, forgottenKeys]) => + res.status(onlyOfficeLicense && twakeDriveToken ? 200 : 500).send({ + uptimeS: Math.floor(process.uptime()), + onlyOfficeLicense, + twakeDriveToken, + forgottenCount: forgottenKeys?.length ?? 0, + }), + err => res.status(500).send(err), + ); + }, +}; diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 1605dcfc8..1293a9533 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -1,5 +1,5 @@ import { CREDENTIALS_SECRET } from '@/config'; -import { OfficeToken } from '@/interfaces/routes.interface'; +import { OfficeToken } from '@/interfaces/office-token.interface'; import driveService from '@/services/drive.service'; import fileService from '@/services/file.service'; import logger from '@/lib/logger'; diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts index c7b94f3ac..4249f5754 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts @@ -21,6 +21,6 @@ export interface IDriveService { get: (params: DriveRequestParams) => Promise; createVersion: (params: { company_id: string; drive_file_id: string; file_id: string }) => Promise; beginEditingSession: (company_id: string, drive_file_id: string) => Promise; - endEditing: (company_id: string, editing_session_key: string, file_source_url: string) => Promise; + endEditing: (company_id: string, editing_session_key: string, url: string) => Promise; getByEditingSessionKey: (params: { company_id: string; editing_session_key: string; user_token?: string }) => Promise; } diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/office-token.interface.ts similarity index 68% rename from tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts rename to tdrive/connectors/onlyoffice-connector/src/interfaces/office-token.interface.ts index bced3c8d9..a52fc3371 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/routes.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/office-token.interface.ts @@ -1,10 +1,3 @@ -import { Router } from 'express'; - -export interface Routes { - path?: string; - router: Router; -} - export interface OfficeToken { user_id: string; company_id: string; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/index.ts b/tdrive/connectors/onlyoffice-connector/src/routes/index.ts index 758ee1a7c..930c6e120 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/index.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/index.ts @@ -1,26 +1,41 @@ -import { Application, Router } from 'express'; +import * as path from 'path'; +import { static as expressStatic, Application, Router } from 'express'; import * as Utils from '@/utils'; +import { SERVER_ORIGIN, SERVER_PREFIX } from '@config'; +import { HealthStatusController } from '@/controllers/health-status.controller'; import { TwakeDriveBackendCallbackRoutes } from './backend-callbacks.route'; import { BrowserEditorRoutes } from './browser-editor.route'; import { OnlyOfficeRoutes } from './onlyoffice.route'; -import { SERVER_ORIGIN, SERVER_PREFIX, SERVER_TDRIVE_API_PREFIX } from '@config'; +function mountAssetsRoutes(router: Router) { + router.use( + '/assets', + (req, res, next) => { + res.header('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Headers', 'X-Requested-With'); + next(); + }, + expressStatic(path.join(__dirname, '../../assets')), + ); +} export function mountRoutes(app: Application) { + // Technical routes for ops. Should not be exposed. + app.get('/health', HealthStatusController.get); + // These routes are forwarded through the Twake Drive front, back and here const proxiedRouter = Router(); BrowserEditorRoutes.mount(proxiedRouter); OnlyOfficeRoutes.mount(proxiedRouter); - console.log('Mounting at ' + SERVER_PREFIX); - app.use(SERVER_PREFIX, proxiedRouter); + mountAssetsRoutes(proxiedRouter); + app.use(SERVER_PREFIX /* usually "plugins/onlyoffice" */, proxiedRouter); // These endpoints should only be accessible to the Twake Drive backend const apiRouter = Router(); - console.log('Mounting at ' + SERVER_TDRIVE_API_PREFIX); TwakeDriveBackendCallbackRoutes.mount(apiRouter); - app.use(SERVER_TDRIVE_API_PREFIX, apiRouter); + app.use('/tdriveApi/1', apiRouter); } export const makeURLTo = { @@ -30,14 +45,3 @@ export const makeURLTo = { return Utils.joinURL([SERVER_ORIGIN ?? '', SERVER_PREFIX, 'editor'], params); }, }; - -// export function makeURLToEditor2() { -// const initResponse = await editorService.init(company_id, file_name, file_id, user, preview, drive_file_id || file_id); - -// res.render('index', { -// ...initResponse, -// docId: preview ? file_id : editing_session_key, -// server: Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), -// token: inPageToken, -// }); -// } From ee63becdbb479b90df208247b2017c89a69dafc2 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Mon, 16 Sep 2024 04:21:50 +0200 Subject: [PATCH 23/52] =?UTF-8?q?=E2=9C=A8=20cli:=20improve=20editing=20se?= =?UTF-8?q?ssion=20listing=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/cli/cmds/editing_session_cmds/list.ts | 131 ++++++++++++++---- 1 file changed, 103 insertions(+), 28 deletions(-) diff --git a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts index 98c4bd484..d7386ea71 100644 --- a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts +++ b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts @@ -1,40 +1,100 @@ import yargs from "yargs"; import runWithPlatform from "../../lib/run-with-platform"; -import { TdrivePlatform } from "../../../core/platform/platform"; -import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; +import type { TdrivePlatform } from "../../../core/platform/platform"; +import type { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; import { DriveFile, EditingSessionKeyFormat, - TYPE, + TYPE as DriveFile_TYPE, } from "../../../services/documents/entities/drive-file"; +import { + FileVersion, + TYPE as FileVersion_TYPE, +} from "../../../services/documents/entities/file-version"; +import User, { TYPE as User_TYPE } from "../../../services/user/entities/user"; +import { FindOptions } from "../../../core/platform/services/database/services/orm/repository/repository"; + +async function makeUserCache(platform: TdrivePlatform) { + const usersRepo = await platform + .getProvider("database") + .getRepository(User_TYPE, User); + const cache: { [id: string]: User } = {}; + return async id => { + if (id in cache) return cache[id]; + const user = await usersRepo.findOne({ id }); + if (user) cache[id] = user; + return user; + }; +} + +interface ListArguments { + all: boolean; + name: string; +} -async function report(platform: TdrivePlatform) { +async function report(platform: TdrivePlatform, args: ListArguments) { + const users = await makeUserCache(platform); + async function formatUser(id) { + const user = await users(id); + return user?.email_canonical || id; + } const drivesRepo = await platform .getProvider("database") - .getRepository(TYPE, DriveFile); - const editedFiled = (await drivesRepo.find({ editing_session_key: { $ne: null } })).getEntities(); - console.error("DriveFiles with non null editing_session_key (url encoded):"); + .getRepository(DriveFile_TYPE, DriveFile); + const versionsRepo = await platform + .getProvider("database") + .getRepository(FileVersion_TYPE, FileVersion); + const filter = {}; + if (!args.all) filter["editing_session_key"] = { $ne: null }; + const opts: FindOptions = { sort: { name: "asc" } }; + if (args.name) filter["name"] = args.name; + const editedFiles = (await drivesRepo.find(filter, opts)).getEntities(); + console.error(`DriveFiles${args.all ? "" : " with non-null editing_session_key"}:`); console.error(""); - editedFiled.forEach(dfile => { + for (const dfile of editedFiles) { console.error(`- ${dfile.name} (${dfile.id}) has key:`); - const parsed = EditingSessionKeyFormat.parse(dfile.editing_session_key); - console.error(` - URL encoded: ${encodeURIComponent(dfile.editing_session_key)}`); - console.error(` - applicationId: ${parsed.applicationId}`); - console.error(` - companyId: ${parsed.companyId}`); - console.error(` - instanceId: ${parsed.instanceId}`); - console.error( - ` - userId: ${parsed.userId} (${ - parsed.userId === dfile.creator ? "same as creator ID" : "not the creator" - })`, - ); - console.error( - ` - timestamp: ${parsed.timestamp.toISOString()} (${Math.floor( - (new Date().getTime() - parsed.timestamp.getTime()) / 1000, - )}s ago)`, - ); - }); - if (!editedFiled.length) console.error(" (no DriveFile currently has an editing_session_key)"); + if (dfile.editing_session_key) { + const parsed = EditingSessionKeyFormat.parse(dfile.editing_session_key); + console.error(" - editing_session_key:"); + console.error(` - URL encoded: ${encodeURIComponent(dfile.editing_session_key)}`); + console.error(` - applicationId: ${parsed.applicationId}`); + console.error(` - companyId: ${parsed.companyId}`); + console.error(` - instanceId: ${parsed.instanceId}`); + console.error( + ` - userId: ${await formatUser(parsed.userId)} (${ + parsed.userId === dfile.creator ? "same as creator ID" : "not the creator" + })`, + ); + console.error( + ` - timestamp: ${parsed.timestamp.toISOString()} (${Math.floor( + (new Date().getTime() - parsed.timestamp.getTime()) / 1000, + )}s ago)`, + ); + } + + const versions = ( + await versionsRepo.find({ drive_item_id: dfile.id }, { sort: { date_added: "asc" } }) + ).getEntities(); + let previousSize = 0; + console.error(" - Versions:"); + for (const version of versions) { + console.error( + ` - ${new Date(version.date_added).toISOString()} by ${await formatUser( + version.creator_id, + )}`, + ); + console.error(` - id: ${version.id}`); + console.error( + ` - size: ${version.file_metadata.size} (${ + version.file_metadata.size > previousSize ? "+" : "" + }${version.file_metadata.size - previousSize})`, + ); + previousSize = version.file_metadata.size; + console.error(` - application: ${JSON.stringify(version.application_id)}`); + } + } + if (!editedFiles.length) console.error(" (no matching DriveFiles)"); } const command: yargs.CommandModule = { @@ -42,11 +102,26 @@ const command: yargs.CommandModule = { describe: ` List current DriveFile items that have an editing_session_key set `.trim(), - builder: {}, - handler: async _argv => { + + builder: { + all: { + type: "boolean", + alias: "a", + describe: "Include all DriveFiles (not just the ones with editing_session_keys)", + default: false, + }, + name: { + type: "string", + alias: "n", + describe: "Filter DriveFiles by name (must be exact)", + default: false, + }, + }, + handler: async argv => { + const args = argv as unknown as ListArguments; await runWithPlatform("editing_session list", async ({ spinner: _spinner, platform }) => { console.error("\n"); - await report(platform); + await report(platform, args); console.error("\n"); }); }, From b8814c0968687d65a8530de4770d333109d95061 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Mon, 16 Sep 2024 17:32:34 +0200 Subject: [PATCH 24/52] =?UTF-8?q?=F0=9F=A9=B9=20backend,oo:=20small=20fixe?= =?UTF-8?q?s,=20related=20to=20instance=20id=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/documents/services/index.ts | 3 +- .../src/services/documents/web/schemas.ts | 1 + .../onlyoffice-connector/src/config/index.ts | 1 + .../backend-callbacks.controller.ts | 61 +++++++++++++++++-- .../src/lib/pending-request-matcher.ts | 2 +- .../src/services/drive.service.ts | 4 +- 6 files changed, 63 insertions(+), 9 deletions(-) diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 89027fc2e..28179376d 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -960,7 +960,8 @@ export class DocumentsService { * @param id DriveFile ID of the document to begin editing * @param editorApplicationId Editor/Application/Plugin specific identifier * @param appInstanceId For that `editorApplicationId` a unique identifier - * when multiple instances are running. Unused today. + * when multiple instances are running. Unused today - would need a mapping + * from `appInstanceId` to server host. * @param context * @returns An object in the format `{}` with the unique identifier for the * editing session diff --git a/tdrive/backend/node/src/services/documents/web/schemas.ts b/tdrive/backend/node/src/services/documents/web/schemas.ts index 81a095d21..65384de78 100644 --- a/tdrive/backend/node/src/services/documents/web/schemas.ts +++ b/tdrive/backend/node/src/services/documents/web/schemas.ts @@ -115,6 +115,7 @@ export const beginEditingSchema = { type: "object", properties: { editorApplicationId: { type: "string" }, + appInstanceId: { type: "string" }, }, required: ["editorApplicationId"], }, diff --git a/tdrive/connectors/onlyoffice-connector/src/config/index.ts b/tdrive/connectors/onlyoffice-connector/src/config/index.ts index a312ed134..5fdc7997e 100644 --- a/tdrive/connectors/onlyoffice-connector/src/config/index.ts +++ b/tdrive/connectors/onlyoffice-connector/src/config/index.ts @@ -12,6 +12,7 @@ export const { CREDENTIALS_SECRET, SERVER_PREFIX, SERVER_ORIGIN, + INSTANCE_ID, } = process.env; const secs = 1000, diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts index 45a8801aa..7f2d584ff 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts @@ -1,6 +1,7 @@ -import { NextFunction, Request, Response } from 'express'; +import { Request, Response } from 'express'; import logger from '@/lib/logger'; -import onlyofficeService, { CommandError, ErrorCode } from '@/services/onlyoffice.service'; +import onlyofficeService, { Callback, CommandError, ErrorCode } from '@/services/onlyoffice.service'; +import driveService from '@/services/drive.service'; interface RequestQuery { editing_session_key: string; @@ -27,12 +28,60 @@ export default class TwakeDriveBackendCallbackController { * Get status of an `editing_session_key` from OO, and return a URL to get the latest version, * or an object with no `url` property, in which case the key is not known as forgotten by OO and should * be considered lost after an admin alert. + * + * @returns + * - `{ status: 'updated' }`: the key needed updating but is now invalid + * - `{ status: 'expired' }`: the key can't be used (it is verified unknown) + * - `{ status: 'live' }`: the key is valid and current and should be used again for the same file + * - `{ error: number }`: there was an error retreiving the status of the key, http status `!= 200` */ public async checkSessionStatus(req: Request, res: Response): Promise { - //TODO: Find a way to check if the key is live (`info` uses the callback url) - await ignoreMissingKeyErrorButNoneElse(res, async () => { - await res.send({ url: await onlyofficeService.getForgotten(req.params.editing_session_key) }); - }); + //TODO: check there is auth from backend before this is ran + + // have to get forgotten first, if it's there it's definitive, + // but if we paralelise it risks calling the callback + try { + const forgottenURL = await onlyofficeService.getForgotten(req.params.editing_session_key); + // Run upload before returning + return void res.send({ status: 'updated' }); + } catch (e) { + if (!(e instanceof CommandError && e.errorCode == ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND)) { + logger.error(`getForgotten failed`, e); + return void res.status(e instanceof CommandError ? 502 : 500).send({ error: -51 }); + } + } + const info = await onlyofficeService.getInfoAndWaitForCallbackUnsafe(req.params.editing_session_key); + if (info.error === ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND) { + // just cancel it + return void res.send({ status: 'expired' }); + } + if (info.error !== undefined) { + logger.error(`getInfo failed`, { error: info }); + return void res.status(502).send({ error: -52 }); + } + switch (info.result.status) { + case Callback.Status.BEING_EDITED: + case Callback.Status.BEING_EDITED_BUT_IS_SAVED: + // use it as is + return void res.send({ status: 'live' }); + + case Callback.Status.CLOSED_WITHOUT_CHANGES: + // just cancel it + return void res.send({ status: 'expired' }); + + case Callback.Status.ERROR_FORCE_SAVING: + case Callback.Status.ERROR_SAVING: + return void res.status(502).send({ error: info.result.status }); + + case Callback.Status.READY_FOR_SAVING: + // upload it, have to do it here for correct user stored in url in OO + //TODO: Need to fix so company_id is not needed by ooconnector but parsed from key server side + await driveService.endEditing(req.params.editing_session_key, info.result.url); + return void res.send({ status: 'updated' }); + + default: + throw new Error(`Unexpected callback status: ${JSON.stringify(info.result)}`); + } } /** diff --git a/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts b/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts index c739d47d5..26cd6387f 100644 --- a/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts +++ b/tdrive/connectors/onlyoffice-connector/src/lib/pending-request-matcher.ts @@ -46,7 +46,7 @@ class PendingRequest { export class PendingRequestQueue { private queue: PendingRequest[] = []; - constructor(private readonly timeoutMs: number, niquystSamplingRatio: 4) { + constructor(private readonly timeoutMs: number, niquystSamplingRatio = 4) { setInterval(() => { this.flush(); }, timeoutMs / niquystSamplingRatio); diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index bf93d75ad..7a91e8dd1 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -3,6 +3,7 @@ import apiService from './api.service'; import logger from '../lib/logger'; import { Stream } from 'stream'; import FormData from 'form-data'; +import { INSTANCE_ID } from '@/config'; /** * Client for Twake Drive's APIs dealing with `DriveItem`s, using {@see apiService} @@ -55,6 +56,7 @@ class DriveService implements IDriveService { url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}/editing_session`, payload: { editorApplicationId: 'tdrive_onlyoffice', + appInstanceId: INSTANCE_ID ?? '', }, }); if (resource?.editingSessionKey) { @@ -113,7 +115,7 @@ class DriveService implements IDriveService { headers: form.getHeaders(), }); } catch (error) { - logger.error('Failed to begin editing session: ', error.stack); + logger.error('Failed to end editing session: ', error.stack); throw error; //TODO make monitoring for such kind of errors } From 2eb6f73312a29ef8317375d43b2a65d06effa7d0 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Mon, 16 Sep 2024 20:12:42 +0200 Subject: [PATCH 25/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=F0=9F=A9=B9?= =?UTF-8?q?=F0=9F=9A=A8=20backend,oo:=20remove=20company=5Fid=20when=20key?= =?UTF-8?q?=20available=20and=20minor=20cleanup=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/documents/services/index.ts | 14 +++++++++++++ .../node/src/services/documents/web/routes.ts | 7 ++++--- .../backend/node/test/e2e/common/user-api.ts | 4 ++-- .../backend-callbacks.controller.ts | 11 ---------- .../src/controllers/onlyoffice.controller.ts | 3 +-- .../src/interfaces/drive.interface.ts | 2 +- .../src/middlewares/error.middleware.ts | 1 - .../src/routes/backend-callbacks.route.ts | 1 - .../src/services/drive.service.ts | 20 ++++++++----------- 9 files changed, 30 insertions(+), 33 deletions(-) diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 28179376d..0ba2e802d 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -1043,6 +1043,20 @@ export class DocumentsService { throw new CrudException("Invalid editing_session_key", 400); } + try { + const parsedKey = EditingSessionKeyFormat.parse(editing_session_key); + context = { + ...context, + company: { id: parsedKey.companyId }, + }; + } catch (e) { + this.logger.error( + "Invalid editing_session_key value: " + JSON.stringify(editing_session_key), + e, + ); + throw new CrudException("Invalid editing_session_key", 400); + } + const driveFile = await this.repository.findOne({ editing_session_key }, {}, context); if (!driveFile) { this.logger.error("Drive item not found by editing session key"); diff --git a/tdrive/backend/node/src/services/documents/web/routes.ts b/tdrive/backend/node/src/services/documents/web/routes.ts index 48a4caebc..9da696fda 100644 --- a/tdrive/backend/node/src/services/documents/web/routes.ts +++ b/tdrive/backend/node/src/services/documents/web/routes.ts @@ -5,6 +5,7 @@ import { createDocumentSchema, createVersionSchema, beginEditingSchema } from ". const baseUrl = "/companies/:company_id"; const serviceUrl = `${baseUrl}/item`; +const editingSessionBase = "/editing_session/:editing_session_key"; const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) => { const documentsController = new DocumentsController(); @@ -89,21 +90,21 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) fastify.route({ method: "GET", - url: `${serviceUrl}/editing_session/:editing_session_key`, + url: editingSessionBase, //TODO NONONO check authenticate*Optional* preValidation: [fastify.authenticateOptional], handler: documentsController.getByEditingSessionKey.bind(documentsController), }); fastify.route({ method: "POST", - url: `${serviceUrl}/editing_session/:editing_session_key`, + url: editingSessionBase, preValidation: [fastify.authenticateOptional], handler: documentsController.endEditing.bind(documentsController), }); fastify.route({ method: "DELETE", - url: `${serviceUrl}/editing_session/:editing_session_key`, + url: editingSessionBase, preValidation: [fastify.authenticateOptional], handler: documentsController.cancelEditing.bind(documentsController), }); diff --git a/tdrive/backend/node/test/e2e/common/user-api.ts b/tdrive/backend/node/test/e2e/common/user-api.ts index c0186901d..ec047e686 100644 --- a/tdrive/backend/node/test/e2e/common/user-api.ts +++ b/tdrive/backend/node/test/e2e/common/user-api.ts @@ -419,7 +419,7 @@ export default class UserApi { return await this.platform.app.inject({ method: "POST", - url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/editing_session/${editingSessionKey}`, + url: `${UserApi.DOC_URL}/editing_session/${editingSessionKey}`, headers: { authorization: `Bearer ${this.jwt}` }, @@ -432,7 +432,7 @@ export default class UserApi { ): Promise { return await this.platform.app.inject({ method: "DELETE", - url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/editing_session/${editingSessionKey}`, + url: `${UserApi.DOC_URL}/editing_session/${editingSessionKey}`, headers: { authorization: `Bearer ${this.jwt}` } diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts index 7f2d584ff..67083d314 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts @@ -83,15 +83,4 @@ export default class TwakeDriveBackendCallbackController { throw new Error(`Unexpected callback status: ${JSON.stringify(info.result)}`); } } - - /** - * Force deletion of the provided `editing_session_key` in the OO document server. - * If the key was succesfully deleted, the `done` property in the response body will be true. - */ - public async deleteSessionKey(req: Request, res: Response): Promise { - await ignoreMissingKeyErrorButNoneElse(res, async () => { - await onlyofficeService.deleteForgotten(req.params.editing_session_key); - await res.send({ done: true }); - }); - } } diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 1293a9533..4a74366f2 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -96,10 +96,9 @@ class OnlyOfficeController { break; case OnlyOffice.Callback.Status.READY_FOR_SAVING: - await driveService.endEditing(company_id, editing_session_key, url); - logger.info(`New version for session ${editing_session_key} created`); return respondToOO(); + await driveService.endEditing(editing_session_key, url); case OnlyOffice.Callback.Status.CLOSED_WITHOUT_CHANGES: // Save end of transaction diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts index 4249f5754..af8da61d5 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts @@ -21,6 +21,6 @@ export interface IDriveService { get: (params: DriveRequestParams) => Promise; createVersion: (params: { company_id: string; drive_file_id: string; file_id: string }) => Promise; beginEditingSession: (company_id: string, drive_file_id: string) => Promise; - endEditing: (company_id: string, editing_session_key: string, url: string) => Promise; + endEditing: (editing_session_key: string, url: string) => Promise; getByEditingSessionKey: (params: { company_id: string; editing_session_key: string; user_token?: string }) => Promise; } diff --git a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts index b452275a5..e03576635 100644 --- a/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts +++ b/tdrive/connectors/onlyoffice-connector/src/middlewares/error.middleware.ts @@ -1,6 +1,5 @@ import logger from '@/lib/logger'; import { NextFunction, Request, Response } from 'express'; -import { makeURLTo } from '@/routes'; export default (error: Error & { status?: number }, req: Request, res: Response, next: NextFunction): void => { try { diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts index f5b409279..2e54c12bc 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts @@ -11,6 +11,5 @@ export const TwakeDriveBackendCallbackRoutes = { const controller = new TwakeDriveBackendCallbackController(); // Why post ? to garantee it is never cached and always ran router.post('/session/:editing_session_key/check', authMiddleware, controller.checkSessionStatus); - router.delete('/session/:editing_session_key', authMiddleware, controller.deleteSessionKey); }, }; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index 7a91e8dd1..cf4644063 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -70,10 +70,10 @@ class DriveService implements IDriveService { } } - public async cancelEditing(company_id: string, editing_session_key) { + public async cancelEditing(editing_session_key: string) { try { await apiService.delete<{}>({ - url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${encodeURIComponent(editing_session_key)}`, + url: `/internal/services/documents/v1/editing_session/${encodeURIComponent(editing_session_key)}`, }); } catch (error) { logger.error('Failed to begin editing session: ', error.stack); @@ -82,13 +82,13 @@ class DriveService implements IDriveService { } } - public async endEditing(company_id: string, editing_session_key: string, url: string) { + public async endEditing(editing_session_key: string, url: string) { try { if (!url) { throw Error('no url found'); } - const originalFile = await this.getByEditingSessionKey({ company_id, editing_session_key }); + const originalFile = await this.getByEditingSessionKey({ editing_session_key }); if (!originalFile) { throw Error('original file not found'); @@ -110,7 +110,7 @@ class DriveService implements IDriveService { logger.info('Saving file version to Twake Drive: ', filename); await apiService.post({ - url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${encodeURIComponent(editing_session_key)}`, + url: `/internal/services/documents/v1/editing_session/${encodeURIComponent(editing_session_key)}`, payload: form, headers: form.getHeaders(), }); @@ -126,15 +126,11 @@ class DriveService implements IDriveService { * /item/editing_session/${editing_session_key} * @param params */ - public getByEditingSessionKey = async (params: { - company_id: string; - editing_session_key: string; - user_token?: string; - }): Promise => { + public getByEditingSessionKey = async (params: { editing_session_key: string; user_token?: string }): Promise => { try { - const { company_id, editing_session_key } = params; + const { editing_session_key } = params; return await apiService.get({ - url: `/internal/services/documents/v1/companies/${company_id}/item/editing_session/${encodeURIComponent(editing_session_key)}`, + url: `/internal/services/documents/v1/editing_session/${encodeURIComponent(editing_session_key)}`, token: params.user_token, }); } catch (error) { From c9db0bf3d764437b30c42aae7e1ccfa7e8df7532 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Tue, 17 Sep 2024 17:03:36 +0200 Subject: [PATCH 26/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20cli:=20better=20info?= =?UTF-8?q?rmation=20about=20file=20history=20and=20key=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../node/src/cli/cmds/editing_session_cmds/list.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts index d7386ea71..e796a7f5f 100644 --- a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts +++ b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts @@ -20,11 +20,11 @@ async function makeUserCache(platform: TdrivePlatform) { .getProvider("database") .getRepository(User_TYPE, User); const cache: { [id: string]: User } = {}; - return async id => { - if (id in cache) return cache[id]; + return async (id): Promise<[boolean, User]> => { + if (id in cache) return [true, cache[id]]; const user = await usersRepo.findOne({ id }); if (user) cache[id] = user; - return user; + return [false, user]; }; } @@ -36,8 +36,10 @@ interface ListArguments { async function report(platform: TdrivePlatform, args: ListArguments) { const users = await makeUserCache(platform); async function formatUser(id) { - const user = await users(id); - return user?.email_canonical || id; + const [wasKnown, user] = await users(id); + return user?.email_canonical + ? user.email_canonical + (wasKnown ? "" : ` (${id})`) + : `${JSON.stringify(id)} (user id not found)`; } const drivesRepo = await platform .getProvider("database") @@ -60,7 +62,7 @@ async function report(platform: TdrivePlatform, args: ListArguments) { console.error(` - URL encoded: ${encodeURIComponent(dfile.editing_session_key)}`); console.error(` - applicationId: ${parsed.applicationId}`); console.error(` - companyId: ${parsed.companyId}`); - console.error(` - instanceId: ${parsed.instanceId}`); + console.error(` - instanceId: ${JSON.stringify(parsed.instanceId)}`); console.error( ` - userId: ${await formatUser(parsed.userId)} (${ parsed.userId === dfile.creator ? "same as creator ID" : "not the creator" From 816fb58e891a9b3b918c959c8b536e598c1e1ef9 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Tue, 17 Sep 2024 17:37:20 +0200 Subject: [PATCH 27/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20backend,oo:=20split?= =?UTF-8?q?=20endEditing=20into=20add=20version=20and=20actual=20end,=20vi?= =?UTF-8?q?a=20"updateEditing"=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/documents/services/index.ts | 48 ++++++++++++------- .../documents/web/controllers/documents.ts | 16 ++++--- .../node/src/services/documents/web/routes.ts | 2 +- .../src/interfaces/drive.interface.ts | 1 + .../src/services/drive.service.ts | 16 ++++++- 5 files changed, 56 insertions(+), 27 deletions(-) diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 0ba2e802d..5e82cb85f 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -977,6 +977,8 @@ export class DocumentsService { return null; } + //TODO: This needs to try in a loop depending on oo-connector response + // when there is already a key let newKey: string; try { newKey = EditingSessionKeyFormat.generate( @@ -986,6 +988,7 @@ export class DocumentsService { context.user.id, ); } catch (e) { + logger.error(`Error generating new editing_session_key: ${e}`, { error: e }); CrudException.throwMe(e, new CrudException("Error generating new editing_session_key", 500)); } @@ -1026,12 +1029,15 @@ export class DocumentsService { * @param editing_session_key Editing key of the DriveFile * @param file Multipart files from the incoming http request * @param options Optional upload information from the request + * @param keepEditing If `true`, the file will be saved as a new version, + * and the DriveFile will keep its editing_session_key. If `true`, a file is required. * @param context */ - endEditing = async ( + updateEditing = async ( editing_session_key: string, file: MultipartFile, options: UploadOptions, + keepEditing: boolean, context: CompanyExecutionContext, ) => { if (!context) { @@ -1043,6 +1049,7 @@ export class DocumentsService { throw new CrudException("Invalid editing_session_key", 400); } + //TODO If the app is the "user" calling, set user to that from the parsed key try { const parsedKey = EditingSessionKeyFormat.parse(editing_session_key); context = { @@ -1087,26 +1094,31 @@ export class DocumentsService { }, context, ); + } else if (keepEditing) { + this.logger.error("Inconsistent endEditing call"); + throw new CrudException("Inconsistent endEditing call", 500); } - try { - const result = await this.repository.atomicCompareAndSet( - driveFile, - "editing_session_key", - editing_session_key, - null, - ); - if (!result.didSet) - throw new Error( - `Couldn't set editing_session_key ${JSON.stringify( - editing_session_key, - )} on DriveFile ${JSON.stringify(driveFile.id)} because it is ${JSON.stringify( - result.currentValue, - )}`, + if (!keepEditing) { + try { + const result = await this.repository.atomicCompareAndSet( + driveFile, + "editing_session_key", + editing_session_key, + null, ); - } catch (error) { - logger.error({ error: `${error}` }, "Failed to cancel editing Drive item"); - CrudException.throwMe(error, new CrudException("Failed to cancel editing Drive item", 500)); + if (!result.didSet) + throw new Error( + `Couldn't set editing_session_key ${JSON.stringify( + editing_session_key, + )} on DriveFile ${JSON.stringify(driveFile.id)} because it is ${JSON.stringify( + result.currentValue, + )}`, + ); + } catch (error) { + logger.error({ error: `${error}` }, "Failed to cancel editing Drive item"); + CrudException.throwMe(error, new CrudException("Failed to cancel editing Drive item", 500)); + } } }; diff --git a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts index c654a21cc..a094f604a 100644 --- a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts +++ b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts @@ -381,10 +381,11 @@ export class DocumentsController { if (!editing_session_key) throw new CrudException("Missing editing_session_key", 400); - return await globalResolver.services.documents.documents.endEditing( + return await globalResolver.services.documents.documents.updateEditing( editing_session_key, null, null, + false, context, ); } catch (error) { @@ -394,12 +395,13 @@ export class DocumentsController { }; //TODO: will need a save under session key, but without ending the edit (for force saves) /** - * Finish an editing session for a given `editing_session_key` by uploading the new version of the File + * Finish an editing session for a given `editing_session_key` by uploading the new version of the File. + * Unless the `keepEditing` query param is `true`, then just save and stay in editing mode. */ - endEditing = async ( + updateEditing = async ( request: FastifyRequest<{ Params: ItemRequestByEditingSessionKeyParams; - Querystring: Record; + Querystring: { keepEditing?: string }; Body: { item: Partial; version: Partial; @@ -423,17 +425,19 @@ export class DocumentsController { waitForThumbnail: !!q.thumbnail_sync, ignoreThumbnails: false, }; - return await globalResolver.services.documents.documents.endEditing( + return await globalResolver.services.documents.documents.updateEditing( editing_session_key, file, options, + request.query.keepEditing == "true", context, ); } else { - return await globalResolver.services.documents.documents.endEditing( + return await globalResolver.services.documents.documents.updateEditing( editing_session_key, null, null, + true, context, ); } diff --git a/tdrive/backend/node/src/services/documents/web/routes.ts b/tdrive/backend/node/src/services/documents/web/routes.ts index 9da696fda..25fde912a 100644 --- a/tdrive/backend/node/src/services/documents/web/routes.ts +++ b/tdrive/backend/node/src/services/documents/web/routes.ts @@ -99,7 +99,7 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) method: "POST", url: editingSessionBase, preValidation: [fastify.authenticateOptional], - handler: documentsController.endEditing.bind(documentsController), + handler: documentsController.updateEditing.bind(documentsController), }); fastify.route({ diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts index af8da61d5..d3195931b 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts @@ -21,6 +21,7 @@ export interface IDriveService { get: (params: DriveRequestParams) => Promise; createVersion: (params: { company_id: string; drive_file_id: string; file_id: string }) => Promise; beginEditingSession: (company_id: string, drive_file_id: string) => Promise; + addEditingSessionVersion: (editing_session_key: string, url: string) => Promise; endEditing: (editing_session_key: string, url: string) => Promise; getByEditingSessionKey: (params: { company_id: string; editing_session_key: string; user_token?: string }) => Promise; } diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index cf4644063..6ba08b766 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -54,6 +54,7 @@ class DriveService implements IDriveService { try { const resource = await apiService.post<{}, { editingSessionKey: string }>({ url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}/editing_session`, + token: user_token, payload: { editorApplicationId: 'tdrive_onlyoffice', appInstanceId: INSTANCE_ID ?? '', @@ -82,7 +83,15 @@ class DriveService implements IDriveService { } } - public async endEditing(editing_session_key: string, url: string) { + public async addEditingSessionVersion(editing_session_key: string, url: string, user_token?: string) { + return this.updateEditing(editing_session_key, url, true, user_token); + } + + public async endEditing(editing_session_key: string, url: string, user_token?: string) { + return this.updateEditing(editing_session_key, url, false, user_token); + } + + private async updateEditing(editing_session_key: string, url: string, keepEditing: boolean, user_token?: string) { try { if (!url) { throw Error('no url found'); @@ -109,9 +118,12 @@ class DriveService implements IDriveService { logger.info('Saving file version to Twake Drive: ', filename); + const queryString = keepEditing ? '?keepEditing=true' : ''; + await apiService.post({ - url: `/internal/services/documents/v1/editing_session/${encodeURIComponent(editing_session_key)}`, + url: `/internal/services/documents/v1/editing_session/${encodeURIComponent(editing_session_key)}` + queryString, payload: form, + token: user_token, headers: form.getHeaders(), }); } catch (error) { From 7b78c32a65a45e38e803195ef1e465f62ca629e0 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Tue, 17 Sep 2024 17:41:11 +0200 Subject: [PATCH 28/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20oo-connector:=20wip?= =?UTF-8?q?=20on=20callbacks=20and=20key=20reactions,=20and=20minor=20clea?= =?UTF-8?q?nup=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend-callbacks.controller.ts | 43 +++++++------- .../src/controllers/onlyoffice.controller.ts | 57 +++++++++++-------- .../src/services/onlyoffice.service.ts | 4 ++ 3 files changed, 56 insertions(+), 48 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts index 67083d314..6a31afefd 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts @@ -7,18 +7,6 @@ interface RequestQuery { editing_session_key: string; } -async function ignoreMissingKeyErrorButNoneElse(res: Response, call: () => Promise): Promise { - try { - await call(); - } catch (e) { - if (e instanceof CommandError && e.errorCode == ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND) { - return void (await res.send({ info: 'Unknown editing_session_key' })); - } - logger.error('Running OO command for TDrive backend', e); - return void (await res.sendStatus(500)); - } -} - /** * These routes are called by Twake Drive backend, for ex. before editing or retreiving a file, * if it has an editing_session_key still, get the status of that and force a resolution. @@ -30,34 +18,42 @@ export default class TwakeDriveBackendCallbackController { * be considered lost after an admin alert. * * @returns + * - `{ status: 'unknown' }`: the key isn't known and maybe used for a new session * - `{ status: 'updated' }`: the key needed updating but is now invalid - * - `{ status: 'expired' }`: the key can't be used (it is verified unknown) + * - `{ status: 'expired' }`: the key was already used in a finished session and can't be used again * - `{ status: 'live' }`: the key is valid and current and should be used again for the same file * - `{ error: number }`: there was an error retreiving the status of the key, http status `!= 200` */ public async checkSessionStatus(req: Request, res: Response): Promise { - //TODO: check there is auth from backend before this is ran - - // have to get forgotten first, if it's there it's definitive, - // but if we paralelise it risks calling the callback try { const forgottenURL = await onlyofficeService.getForgotten(req.params.editing_session_key); - // Run upload before returning + try { + await driveService.endEditing(req.params.editing_session_key, forgottenURL); + } catch (error) { + logger.error(`endEditing failed`, { error }); + return void res.status(502).send({ error: -57649 }); + } + try { + await onlyofficeService.deleteForgotten(req.params.editing_session_key); + } catch (error) { + logger.error(`deleteForgotten failed`, { error }); + return void res.status(502).send({ error: -57650 }); + } return void res.send({ status: 'updated' }); } catch (e) { if (!(e instanceof CommandError && e.errorCode == ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND)) { - logger.error(`getForgotten failed`, e); - return void res.status(e instanceof CommandError ? 502 : 500).send({ error: -51 }); + logger.error(`getForgotten failed`, { error: e }); + return void res.status(e instanceof CommandError ? 502 : 500).send({ error: -57651 }); } } const info = await onlyofficeService.getInfoAndWaitForCallbackUnsafe(req.params.editing_session_key); if (info.error === ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND) { - // just cancel it - return void res.send({ status: 'expired' }); + // just start using it + return void res.send({ status: 'unknown' }); } if (info.error !== undefined) { logger.error(`getInfo failed`, { error: info }); - return void res.status(502).send({ error: -52 }); + return void res.status(502).send({ error: -57652 }); } switch (info.result.status) { case Callback.Status.BEING_EDITED: @@ -75,7 +71,6 @@ export default class TwakeDriveBackendCallbackController { case Callback.Status.READY_FOR_SAVING: // upload it, have to do it here for correct user stored in url in OO - //TODO: Need to fix so company_id is not needed by ooconnector but parsed from key server side await driveService.endEditing(req.params.editing_session_key, info.result.url); return void res.send({ status: 'updated' }); diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 4a74366f2..112e25be1 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -65,61 +65,70 @@ class OnlyOfficeController { const { url, key } = req.body; const { token } = req.query; logger.info( - `OO callback status: ${Utils.getKeyForValueSafe( - req.body.status, - OnlyOffice.Callback.Status, - 'OnlyOffice.Callback.Status', - )} - ${JSON.stringify(req.body.userdata)}`, + `OO callback status: ${Utils.getKeyForValueSafe(req.body.status, OnlyOffice.Callback.Status, 'Callback.Status')} - ${JSON.stringify( + req.body.userdata, + )}`, req.body, ); const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; - const { preview, company_id, file_id, /* user_id, */ drive_file_id, in_page_token, editing_session_key } = officeTokenPayload; + const { preview, /* company_id, file_id, user_id, drive_file_id, */ in_page_token /* editing_session_key */ } = officeTokenPayload; // Ignore errors generated by pending request // try-catch not needed because it is async // there may be later reasons to wait for callbacks // to process and eventually respond accordingly to // OO an error for certain statuses - void OnlyOffice.default.ooCallbackCalled(req.body); + void OnlyOffice.default.ooCallbackCalled(req.body); // has to be single thread per key // check token is an in_page_token and allow save - if (!in_page_token) throw new Error('Invalid token, must be a in_page_token'); - if (preview) throw new Error('Invalid token, must not be a preview token for save operation'); + if (!in_page_token) throw new Error('OO Callback invalid token, must be a in_page_token'); + if (preview) throw new Error('OO Callback invalid token, must not be a preview token for save operation'); switch (req.body.status) { case OnlyOffice.Callback.Status.BEING_EDITED: - // TODO this call back we recieve almost all the time, and here we save - // the user identifiers who start file editing and even control the amount of onlin users - // to have license constraint warning before OnlyOffice error about this - case OnlyOffice.Callback.Status.BEING_EDITED_BUT_IS_SAVED: + // TODO this call back we recieve almost all the time, and here we save + // the user identifiers who start file editing and even control the amount of onlin users + // to have license constraint warning before OnlyOffice error about this // No-op break; + case OnlyOffice.Callback.Status.BEING_EDITED_BUT_IS_SAVED: + logger.info(`OO Callback force save for session ${key} for reason: ${OnlyOffice.Callback.ForceSaveTypeToString(req.body.forcesavetype)}`); + await driveService.addEditingSessionVersion(key, url); //, token); //TODO Fix user token (getting 401) + break; + case OnlyOffice.Callback.Status.READY_FOR_SAVING: - logger.info(`New version for session ${editing_session_key} created`); - return respondToOO(); - await driveService.endEditing(editing_session_key, url); + logger.info(`OO Callback new version for session ${key} created`); + await driveService.endEditing(key, url); //, token); //TODO Fix user token (getting 401) + break; case OnlyOffice.Callback.Status.CLOSED_WITHOUT_CHANGES: - // Save end of transaction + await driveService.cancelEditing(key); break; case OnlyOffice.Callback.Status.ERROR_SAVING: - // Save end of transaction - logger.info(`Error saving file ${req.body.url} (key: ${req.body.key})`); + // TODO: Save end of transaction ? + logger.error(`OO Callback with Status.ERROR_SAVING: ${req.body.url} (key: ${req.body.key})`); break; case OnlyOffice.Callback.Status.ERROR_FORCE_SAVING: // TODO: notify user ? - logger.info(`Error force saving (reason: ${req.body.forcesavetype}) file ${req.body.url} (key: ${req.body.key})`); - return void res.send({ error: 0 }); + logger.error( + `OO Callback with Status.ERROR_FORCE_SAVING (reason: ${OnlyOffice.Callback.ForceSaveTypeToString(req.body.forcesavetype)}) file ${ + req.body.url + } (key: ${req.body.key})`, + ); + break; default: - throw new Error(`Unexpected OO Callback status field: ${req.body.status}`); + throw new Error( + `OO Callback unexpected status field: ${OnlyOffice.Callback.StatusToString(req.body.status)} in ${JSON.stringify(req.body)}`, + ); } - return respondToOO(); + return respondToOO(0); } catch (error) { - next(error); + logger.error(`OO Callback root error`, { error }); + next(error || 'error'); } }; } diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index 9b67f2fda..7fd81c28e 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -32,6 +32,7 @@ export namespace Callback { USER_CONNECTED = 1, USER_INITIATED_FORCE_SAVE = 2, } + export const ActionTypeToString = (value: number) => Utils.getKeyForValueSafe(value, ActionType, 'OnlyOffice.Callback.ActionType'); interface Action { type: ActionType; @@ -44,6 +45,7 @@ export namespace Callback { SERVER_TIMER = 2, FORM_SUBMITTED = 3, } + export const ForceSaveTypeToString = (value: number) => Utils.getKeyForValueSafe(value, ForceSaveType, 'OnlyOffice.Callback.ForceSaveType'); export enum Status { BEING_EDITED = 1, @@ -57,6 +59,7 @@ export namespace Callback { /** `url` and `forcesavetype` fields present with this status */ ERROR_FORCE_SAVING = 7, } + export const StatusToString = (value: number) => Utils.getKeyForValueSafe(value, Status, 'OnlyOffice.Callback.Status'); /** Parameters given to the callback by the editing service */ export interface Parameters { @@ -305,6 +308,7 @@ class OnlyOfficeService { return await new CommandService.Info.Request(key, userdata).postUnsafe().then(({ error }) => error); } + /** Send the info command, wait for the callback if warranted, and return the error or callback body */ async getInfoAndWaitForCallbackUnsafe(key: string): Promise { // const userdata = randomUUID(); return new Promise((resolve, reject) => { From 18f9208a43e6b4cb10fcd8ab85d25210e40bd0d0 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Wed, 18 Sep 2024 01:55:18 +0200 Subject: [PATCH 29/52] =?UTF-8?q?=E2=9C=A8=20oo-connector:=20OO=20forgotte?= =?UTF-8?q?n=20files=20batch=20processor,=20and=20testing,=20but=20WIP=20(?= =?UTF-8?q?#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../onlyoffice-connector/src/app.ts | 4 ++ .../controllers/health-status.controller.ts | 16 ++++-- .../src/services/api.service.ts | 26 +-------- .../src/services/drive.service.ts | 16 +++++- .../services/forgotten-processor.service.ts | 56 +++++++++++++++++++ .../src/services/onlyoffice.service.ts | 7 ++- 6 files changed, 94 insertions(+), 31 deletions(-) create mode 100644 tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts diff --git a/tdrive/connectors/onlyoffice-connector/src/app.ts b/tdrive/connectors/onlyoffice-connector/src/app.ts index 9b5f1bd26..d85424a94 100644 --- a/tdrive/connectors/onlyoffice-connector/src/app.ts +++ b/tdrive/connectors/onlyoffice-connector/src/app.ts @@ -9,6 +9,8 @@ import logger from './lib/logger'; import errorMiddleware from './middlewares/error.middleware'; import { mountRoutes } from './routes'; +import forgottenProcessorService from './services/forgotten-processor.service'; + class App { public app: express.Application; public env: string; @@ -22,6 +24,8 @@ class App { this.initMiddlewares(); this.initRoutes(); this.initErrorHandling(); + + forgottenProcessorService.makeSureItsLoaded(); } public listen = () => { diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts index 88ccb136a..a5eee4daa 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts @@ -1,19 +1,27 @@ -import { NextFunction, Request, Response } from 'express'; +import { Request, Response } from 'express'; import onlyofficeService from '@/services/onlyoffice.service'; import apiService from '@/services/api.service'; +import forgottenProcessorService from '@/services/forgotten-processor.service'; + /** * Health response for devops operational purposes. Should not be exposed. */ export const HealthStatusController = { - async get(req: Request<{}, {}, {}, {}>, res: Response, next: NextFunction): Promise { - Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken(), onlyofficeService.getForgottenList()]).then( - ([onlyOfficeLicense, twakeDriveToken, forgottenKeys]) => + async get(req: Request<{}, {}, {}, {}>, res: Response): Promise { + Promise.all([ + onlyofficeService.getLatestLicence(), + apiService.hasToken(), + onlyofficeService.getForgottenList(), + forgottenProcessorService.getLastStartTimeAgoS(), + ]).then( + ([onlyOfficeLicense, twakeDriveToken, forgottenKeys, forgottenLastProcessAgoS]) => res.status(onlyOfficeLicense && twakeDriveToken ? 200 : 500).send({ uptimeS: Math.floor(process.uptime()), onlyOfficeLicense, twakeDriveToken, forgottenCount: forgottenKeys?.length ?? 0, + forgottenLastProcessAgoS, }), err => res.status(500).send(err), ); diff --git a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts index db046012e..e563a3732 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts @@ -5,17 +5,10 @@ import { IApiServiceApplicationTokenResponse, } from '@/interfaces/api.interface'; import axios, { Axios, AxiosRequestConfig, AxiosResponse } from 'axios'; -import { - CREDENTIALS_ENDPOINT, - CREDENTIALS_ID, - CREDENTIALS_SECRET, - twakeDriveTokenRefrehPeriodMS, - onlyOfficeForgottenFilesCheckPeriodMS, -} from '@config'; +import { CREDENTIALS_ENDPOINT, CREDENTIALS_ID, CREDENTIALS_SECRET, twakeDriveTokenRefrehPeriodMS } from '@config'; import logger from '../lib/logger'; import * as Utils from '@/utils'; import { PolledThingieValue } from '@/lib/polled-thingie-value'; -import onlyofficeService from './onlyoffice.service'; /** * Client for the Twake Drive backend API on behalf of the plugin (or provided token in parameters). @@ -23,15 +16,9 @@ import onlyofficeService from './onlyoffice.service'; */ class ApiService implements IApiService { private readonly tokenPoller: PolledThingieValue; - private readonly forgottenFilesPoller: PolledThingieValue; constructor() { this.tokenPoller = new PolledThingieValue('Refresh Twake Drive token', async () => await this.refreshToken(), twakeDriveTokenRefrehPeriodMS); - this.forgottenFilesPoller = new PolledThingieValue( - 'Process forgotten files in OO', - async () => await this.processForgottenFiles(), - onlyOfficeForgottenFilesCheckPeriodMS, - ); } public async hasToken() { @@ -42,14 +29,6 @@ class ApiService implements IApiService { return this.tokenPoller.requireLatestValueWithTry('No Twake Drive app token.'); } - private async processForgottenFiles() { - if (!this.tokenPoller.hasValue()) return -1; - return await onlyofficeService.processForgotten(async (/* key, url */) => { - //TODO: when endpoint decided, call here. See if accept HTTP 202 for ex. to avoid deleting. - return false; - }); - } - public get = async (params: IApiServiceRequestParams): Promise => { const { url, token, responseType, headers } = params; @@ -95,6 +74,7 @@ class ApiService implements IApiService { const axiosWithToken = await this.requireAxios(); + logger.info(`POST to Twake Drive ${url} - payload: ${payload}`); try { return await axiosWithToken.post(url, payload, { headers: { @@ -103,7 +83,7 @@ class ApiService implements IApiService { }, }); } catch (error) { - logger.error('Failed to post to Twake drive: ', error.stack); + logger.error('Failed to post to Twake Drive: ', { error }); this.refreshToken(); } }; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index 6ba08b766..92b3dbe14 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -96,11 +96,20 @@ class DriveService implements IDriveService { if (!url) { throw Error('no url found'); } - + //TODO: It would be better to avoid two requests for this operation const originalFile = await this.getByEditingSessionKey({ editing_session_key }); if (!originalFile) { - throw Error('original file not found'); + // TODO: Make a single request and don't require the filename at all + // then in POST /editing_session/... if the key is not found, just + // put it in a users or company "lost and found" folder. + // and accept without error. Because really, if backend doesn't know + // the key anymore, there's not much we can do, and we should get OO + // to clean up and stop trying to upload it. + // but for today: + logger.fatal("Losing OO document because Twake Drive doesn't know that Key.", { editing_session_key, url }); + throw new Error(`Unknown key ${JSON.stringify(editing_session_key)}`); + // TODO: Distinguish this case from a long disconnected browser tab waking up } const newFile = await apiService.get({ @@ -137,6 +146,7 @@ class DriveService implements IDriveService { * Get the document information by the editing session key. Just simple call to the drive API * /item/editing_session/${editing_session_key} * @param params + * @returns null if the key was not found, or the api response body */ public getByEditingSessionKey = async (params: { editing_session_key: string; user_token?: string }): Promise => { try { @@ -146,7 +156,7 @@ class DriveService implements IDriveService { token: params.user_token, }); } catch (error) { - logger.error('Failed to fetch file metadata by editing session key: ', error.stack); + if (error?.response?.status === 404) return null; throw error; } }; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts new file mode 100644 index 000000000..3be7713d3 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts @@ -0,0 +1,56 @@ +import { onlyOfficeForgottenFilesCheckPeriodMS } from '@/config'; +import { PolledThingieValue } from '@/lib/polled-thingie-value'; + +import apiService from './api.service'; +import onlyofficeService from './onlyoffice.service'; +import driveService from './drive.service'; +import logger from '@/lib/logger'; + +/** + * Periodically poll the Only Office document server for forgotten + * files and try to upload to Twake Drive or get rid if unknown. + */ +class ForgottenProcessor { + private readonly forgottenFilesPoller: PolledThingieValue; + private lastStart = 0; + + constructor() { + let skippedFirst = false; + this.forgottenFilesPoller = new PolledThingieValue( + 'Process forgotten files in OO', + async () => (skippedFirst ? await this.processForgottenFiles() : ((skippedFirst = true), -1)), + onlyOfficeForgottenFilesCheckPeriodMS, + ); + } + + /** Get the number of seconds since the last time this process started */ + public getLastStartTimeAgoS() { + return ~~((new Date().getTime() - this.lastStart) / 1000); + } + + public makeSureItsLoaded() { + // The point of this is to ensure this file is imported, + // which is needed for side-effect of starting this timer + return true; + } + + private async processForgottenFiles() { + this.lastStart = new Date().getTime(); + if (!(await apiService.hasToken())) return -1; + return await onlyofficeService.processForgotten(async (key, url) => { + try { + await driveService.endEditing(key, url); + return true; + } catch (error) { + logger.error(`Error processing forgotten file by key ${JSON.stringify(key)}: ${error}`, { key, url, error }); + // Can't do much about it here, hope it goes in retry, but don't + // throw to keep processing + //TODO: Maybe make a date string, compare to key, if old enough, delete... + // this logic should probably in Twake Drive backend though + } + return false; + }); + } +} + +export default new ForgottenProcessor(); diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index 7fd81c28e..cca8c733a 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -249,8 +249,13 @@ class OnlyOfficeService { * @returns The number of files processed and deleted */ public async processForgotten(processor: (key: string, url: string) => Promise): Promise { + logger.info(`Begin to process forgotten files in OnlyOffice`); + //TODO: filter by instance id of the key const forgottenFiles = await this.getForgottenList(); - if (forgottenFiles.length === 0) return 0; + if (forgottenFiles.length === 0) { + logger.info(`No forgotten files in OnlyOffice`); + return 0; + } Utils.fisherYattesShuffleInPlace(forgottenFiles); logger.info(`Forgotten files found: ${forgottenFiles.length}`); let deleted = 0; From 6685b7b98d34a7e12465938bbcf0f5ce4258aea5 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Wed, 18 Sep 2024 03:34:18 +0200 Subject: [PATCH 30/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20oo-connector:=20cent?= =?UTF-8?q?ralising=20forgotten=20file=20batch=20management=20with=20sessi?= =?UTF-8?q?on=20key=20check,=20bit=20of=20health=20refactoring=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend-callbacks.controller.ts | 11 +- .../controllers/health-status.controller.ts | 12 +- .../src/lib/single-processor-lock.ts | 161 ++++++++++++++++++ .../src/services/drive.service.ts | 7 +- .../services/forgotten-processor.service.ts | 63 +++++-- 5 files changed, 220 insertions(+), 34 deletions(-) create mode 100644 tdrive/connectors/onlyoffice-connector/src/lib/single-processor-lock.ts diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts index 6a31afefd..82fca7eeb 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts @@ -2,6 +2,7 @@ import { Request, Response } from 'express'; import logger from '@/lib/logger'; import onlyofficeService, { Callback, CommandError, ErrorCode } from '@/services/onlyoffice.service'; import driveService from '@/services/drive.service'; +import forgottenProcessorService from '@/services/forgotten-processor.service'; interface RequestQuery { editing_session_key: string; @@ -28,15 +29,9 @@ export default class TwakeDriveBackendCallbackController { try { const forgottenURL = await onlyofficeService.getForgotten(req.params.editing_session_key); try { - await driveService.endEditing(req.params.editing_session_key, forgottenURL); + await forgottenProcessorService.processForgottenFile(req.params.editing_session_key, forgottenURL); } catch (error) { - logger.error(`endEditing failed`, { error }); - return void res.status(502).send({ error: -57649 }); - } - try { - await onlyofficeService.deleteForgotten(req.params.editing_session_key); - } catch (error) { - logger.error(`deleteForgotten failed`, { error }); + logger.error(`processForgottenFile failed`, { error }); return void res.status(502).send({ error: -57650 }); } return void res.send({ status: 'updated' }); diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts index a5eee4daa..97fa28262 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts @@ -9,19 +9,13 @@ import forgottenProcessorService from '@/services/forgotten-processor.service'; */ export const HealthStatusController = { async get(req: Request<{}, {}, {}, {}>, res: Response): Promise { - Promise.all([ - onlyofficeService.getLatestLicence(), - apiService.hasToken(), - onlyofficeService.getForgottenList(), - forgottenProcessorService.getLastStartTimeAgoS(), - ]).then( - ([onlyOfficeLicense, twakeDriveToken, forgottenKeys, forgottenLastProcessAgoS]) => + Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken(), forgottenProcessorService.getHealthData()]).then( + ([onlyOfficeLicense, twakeDriveToken, forgottenProcessorHealth]) => res.status(onlyOfficeLicense && twakeDriveToken ? 200 : 500).send({ uptimeS: Math.floor(process.uptime()), onlyOfficeLicense, twakeDriveToken, - forgottenCount: forgottenKeys?.length ?? 0, - forgottenLastProcessAgoS, + ...forgottenProcessorHealth, }), err => res.status(500).send(err), ); diff --git a/tdrive/connectors/onlyoffice-connector/src/lib/single-processor-lock.ts b/tdrive/connectors/onlyoffice-connector/src/lib/single-processor-lock.ts new file mode 100644 index 000000000..108f3d6a7 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/lib/single-processor-lock.ts @@ -0,0 +1,161 @@ +import logger from './logger'; + +export class CannotSettleAlreadyReleasedLockError extends Error {} + +const debugLocks = false; + +/** Simplifies use of the Promise constructor function out of its scope */ +class ExplodedPromise { + public readonly startAtMs = new Date().getTime(); + public readonly promise: Promise; + private _waiters = 0; + private _resolve: (value: T) => void; + private _reject: (reason?: any) => void; + + constructor() { + this.promise = new Promise((resolve, reject) => { + // Works because this is garanteed to be called synchroneously + this._resolve = resolve; + this._reject = reject; + }); + } + private check() { + if (!this._resolve) throw new CannotSettleAlreadyReleasedLockError(`Promise is already settled`); + } + private finish() { + this._resolve = this._reject = null; + } + /** Increase waiter count and return new count */ + public addWaiter() { + return ++this._waiters; + } + get waiting() { + return this._waiters; + } + public resolve(...args: Parameters['_resolve']>): void { + this.check(); + this._resolve(...args); + this.finish(); + } + public reject(...args: Parameters['_reject']>): void { + this.check(); + this._reject(...args); + this.finish(); + } +} + +/** + * Use this instance to release the lock acquired by {@link SingleProcessorLock.acquire}. + * Only call either `resolve` or `reject` once or they will + * throw a {@link CannotSettleAlreadyReleasedLockError}. + */ +export interface LockReleaser { + /** To distinguish return from {@link LockWaiter} */ + didAcquire: true; + /** This is the same Promise that `resolve` and `reject` will settle, and waiters get */ + promise: LockWaiter['promise']; + /** Release the lock and resolve the promise of all the waiters */ + resolve: ExplodedPromise['resolve']; + /** Release the lock and reject the promise of all the waiters */ + reject: ExplodedPromise['reject']; +} + +/** Use this instance to wait if {@link SingleProcessorLock.acquire} was already processing that key */ +export interface LockWaiter { + /** To distinguish return from {@link LockReleaser} */ + didAcquire: false; + /** Number of waiters in the queue at the start of this one, including this one (ie. first to wait is `1`) */ + numberInQueue: number; + /** Promise that must be waited for after the acquire failed. Will settle as per called by the {@see LockReleaser} */ + promise: Promise; +} + +/** + * Create an synchronisation primitive that permits a single caller + * to process for a given `key`. Other callers with the same `key` + * will receive an `Promise` that should be waited for. When the + * processor caller releases the lock, the promise is settled accordingly. + */ +export function createSingleProcessorLock() { + const promisesByKey = new Map>(); + + /** + * The first caller for the given `key` is the processor that must settle the lock. The + * return value will be an instance of {@link LockReleaser} that can be tested because + * {@link LockReleaser.didAcquire} will be `true`. This caller has the responsibility to + * ensure a call to either {@link LockReleaser.resolve} or {@link LockReleaser.reject}. + * There is no other cleanup mecanism. + * + * All callers following them for the same `key` will get an instance of {@link LockWaiter}, + * that can be tested because {@link LockWaiter.didAcquire} will be `false`. They must + * `await` {@link LockWaiter.promise}. It will be settled by the processor when they call + * the methods of {@link LockReleaser}. must be ensured, there is no cleanup mecanism. + * + * @param key Unique key to lock on + */ + const acquire = (key: string): LockReleaser | LockWaiter => { + const existing = promisesByKey.get(key); + if (existing) { + const numberInQueue = existing.addWaiter(); + debugLocks && logger.debug(`LOCK blocked ${numberInQueue}: ${key}`); + return { + didAcquire: false, + numberInQueue, + promise: existing.promise, + }; + } + const processorPromise = new ExplodedPromise(); + promisesByKey.set(key, processorPromise); + debugLocks && logger.debug(`LOCK started: ${key}`); + return { + didAcquire: true, + promise: processorPromise.promise, + resolve(...args: Parameters['resolve']>) { + debugLocks && logger.debug(`LOCK resolved: ${key}`); + promisesByKey.delete(key); + processorPromise.resolve(...args); + }, + reject(...args: Parameters['reject']>) { + debugLocks && logger.debug(`LOCK rejected: ${key}`); + promisesByKey.delete(key); + processorPromise.reject(...args); + }, + }; + }; + return { + acquire, + + /** + * Attempt to acquire a lock. If succesful, settle the promise with the + * return value of `processor`. In both cases return the Promise. + */ + async runWithLock(key: string, processor: () => Promise): Promise { + const lock = acquire(key); + if (lock.didAcquire) + try { + lock.resolve(await processor()); + } catch (e) { + lock.reject(e); + } + return lock.promise; + }, + + /** Get statistics about pending locks */ + getWorstStats() { + const all = [...promisesByKey.values()]; + const now = new Date().getTime(); + return all.reduce( + ({ oldestMs, waiting, total }, cur) => ({ + oldestMs: Math.max(oldestMs, now - cur.startAtMs), + waiting: Math.max(waiting, cur.waiting), + total: total + 1, + }), + { + oldestMs: 0, + waiting: 0, + total: 0, + }, + ); + }, + }; +} diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index 92b3dbe14..2f8ed3ed6 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -5,6 +5,9 @@ import { Stream } from 'stream'; import FormData from 'form-data'; import { INSTANCE_ID } from '@/config'; +/** Thrown when Twake Drive returns a 404 for a key */ +export class UnknownKeyInDriveError extends Error {} + /** * Client for Twake Drive's APIs dealing with `DriveItem`s, using {@see apiService} * to handle authorization @@ -107,8 +110,8 @@ class DriveService implements IDriveService { // the key anymore, there's not much we can do, and we should get OO // to clean up and stop trying to upload it. // but for today: - logger.fatal("Losing OO document because Twake Drive doesn't know that Key.", { editing_session_key, url }); - throw new Error(`Unknown key ${JSON.stringify(editing_session_key)}`); + logger.error("Forgotten OO document that Twake Drive doesn't know the key of.", { editing_session_key, url }); + throw new UnknownKeyInDriveError(`Unknown key ${JSON.stringify(editing_session_key)}`); // TODO: Distinguish this case from a long disconnected browser tab waking up } diff --git a/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts index 3be7713d3..c295af3ac 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts @@ -1,10 +1,11 @@ +import logger from '@/lib/logger'; import { onlyOfficeForgottenFilesCheckPeriodMS } from '@/config'; import { PolledThingieValue } from '@/lib/polled-thingie-value'; +import { createSingleProcessorLock } from '@/lib/single-processor-lock'; import apiService from './api.service'; import onlyofficeService from './onlyoffice.service'; -import driveService from './drive.service'; -import logger from '@/lib/logger'; +import driveService, { UnknownKeyInDriveError } from './drive.service'; /** * Periodically poll the Only Office document server for forgotten @@ -12,6 +13,7 @@ import logger from '@/lib/logger'; */ class ForgottenProcessor { private readonly forgottenFilesPoller: PolledThingieValue; + public readonly forgottenSynchroniser = createSingleProcessorLock(); private lastStart = 0; constructor() { @@ -23,34 +25,65 @@ class ForgottenProcessor { ); } - /** Get the number of seconds since the last time this process started */ - public getLastStartTimeAgoS() { - return ~~((new Date().getTime() - this.lastStart) / 1000); + public async getHealthData() { + const keys = await onlyofficeService.getForgottenList(); + return { + forgotten: { + timeSinceLastStartS: ~~((new Date().getTime() - this.lastStart) / 1000), + count: keys?.length ?? 0, + locks: this.forgottenSynchroniser.getWorstStats(), + }, + }; } + /** + * The point of this is to ensure this file is imported, + * which is needed for side-effect of starting this timer. + * The only other use being the health stuff it could easily + * be refactored out as unused. + */ public makeSureItsLoaded() { - // The point of this is to ensure this file is imported, - // which is needed for side-effect of starting this timer - return true; + return 'yup this module is loaded !'; } - private async processForgottenFiles() { - this.lastStart = new Date().getTime(); - if (!(await apiService.hasToken())) return -1; - return await onlyofficeService.processForgotten(async (key, url) => { + /** + * Try to upload the forgotten file, optionally delete it from OO, will return if it was + * (or should be) deleted. Does not throw unless the OO deletion itself threw. + */ + private async safeEndEditing(key: string, url: string, deleteForgotten: boolean) { + return this.forgottenSynchroniser.runWithLock(key, async () => { + let succeded = false; try { await driveService.endEditing(key, url); - return true; + succeded = true; } catch (error) { - logger.error(`Error processing forgotten file by key ${JSON.stringify(key)}: ${error}`, { key, url, error }); + if (!(error instanceof UnknownKeyInDriveError)) + logger.error(`Error processing forgotten file by key ${JSON.stringify(key)}: ${error}`, { key, url, error }); // Can't do much about it here, hope it goes in retry, but don't // throw to keep processing //TODO: Maybe make a date string, compare to key, if old enough, delete... // this logic should probably in Twake Drive backend though } - return false; + if (succeded && deleteForgotten) await onlyofficeService.deleteForgotten(key); + return succeded; }); } + + private async processForgottenFiles() { + this.lastStart = new Date().getTime(); + if (!(await apiService.hasToken())) return -1; + return await onlyofficeService.processForgotten((key, url) => + this.safeEndEditing(key, url, false /* `onlyofficeService.processForgotten` will do it */), + ); + } + + /** + * Attempt to upload the forgotten OO file, only throws if deleting it failed, + * returns wether it was deleted. + */ + public async processForgottenFile(key: string, url: string) { + return this.safeEndEditing(key, url, true); + } } export default new ForgottenProcessor(); From 488d5a59367e8959778514888448270142914e04 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Wed, 18 Sep 2024 17:47:55 +0200 Subject: [PATCH 31/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20oo-connector:=20sing?= =?UTF-8?q?le=20request=20endediting,=20refactor=20URLs,=20bit=20of=20logg?= =?UTF-8?q?ing=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/interfaces/drive.interface.ts | 1 - .../src/services/api.service.ts | 3 +- .../src/services/drive.service.ts | 75 +++++++------------ .../onlyoffice-connector/src/utils.ts | 2 +- 4 files changed, 28 insertions(+), 53 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts index d3195931b..9755461c9 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts @@ -23,5 +23,4 @@ export interface IDriveService { beginEditingSession: (company_id: string, drive_file_id: string) => Promise; addEditingSessionVersion: (editing_session_key: string, url: string) => Promise; endEditing: (editing_session_key: string, url: string) => Promise; - getByEditingSessionKey: (params: { company_id: string; editing_session_key: string; user_token?: string }) => Promise; } diff --git a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts index e563a3732..4d26860e1 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts @@ -83,8 +83,9 @@ class ApiService implements IApiService { }, }); } catch (error) { - logger.error('Failed to post to Twake Drive: ', { error }); + logger.error('Failed to post to Twake Drive: ', { error: error.stack }); this.refreshToken(); + throw error; } }; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index 2f8ed3ed6..c006991ba 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -1,6 +1,7 @@ import { DriveFileType, IDriveService } from '@/interfaces/drive.interface'; import apiService from './api.service'; import logger from '../lib/logger'; +import * as Utils from '../utils'; import { Stream } from 'stream'; import FormData from 'form-data'; import { INSTANCE_ID } from '@/config'; @@ -8,6 +9,12 @@ import { INSTANCE_ID } from '@/config'; /** Thrown when Twake Drive returns a 404 for a key */ export class UnknownKeyInDriveError extends Error {} +const makeTwakeDriveApiUrl = (paths: string[], params?: Utils.QueryParams) => Utils.joinURL(['/internal/services/documents/v1', ...paths], params); +const makeNonEditingSessionItemUrl = (company_id: string, drive_file_id: string, paths: string[] = [], params?: Utils.QueryParams) => + makeTwakeDriveApiUrl(['companies', company_id, 'item', drive_file_id, ...paths], params); +const makeEditingSessionItemUrl = (editing_session: string, params?: Utils.QueryParams) => + makeTwakeDriveApiUrl(['editing_session', editing_session], params); + /** * Client for Twake Drive's APIs dealing with `DriveItem`s, using {@see apiService} * to handle authorization @@ -17,7 +24,7 @@ class DriveService implements IDriveService { try { const { company_id, drive_file_id } = params; const resource = await apiService.get({ - url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}`, + url: makeNonEditingSessionItemUrl(company_id, drive_file_id), token: params.user_token, }); @@ -37,7 +44,7 @@ class DriveService implements IDriveService { try { const { company_id, drive_file_id, file_id } = params; return await apiService.post<{}, DriveFileType['item']['last_version_cache']>({ - url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}/version`, + url: makeNonEditingSessionItemUrl(company_id, drive_file_id, ['version']), payload: { drive_item_id: drive_file_id, provider: 'internal', @@ -56,7 +63,7 @@ class DriveService implements IDriveService { public async beginEditingSession(company_id: string, drive_file_id: string, user_token?: string) { try { const resource = await apiService.post<{}, { editingSessionKey: string }>({ - url: `/internal/services/documents/v1/companies/${company_id}/item/${drive_file_id}/editing_session`, + url: makeNonEditingSessionItemUrl(company_id, drive_file_id, ['editing_session']), token: user_token, payload: { editorApplicationId: 'tdrive_onlyoffice', @@ -77,7 +84,7 @@ class DriveService implements IDriveService { public async cancelEditing(editing_session_key: string) { try { await apiService.delete<{}>({ - url: `/internal/services/documents/v1/editing_session/${encodeURIComponent(editing_session_key)}`, + url: makeEditingSessionItemUrl(editing_session_key), }); } catch (error) { logger.error('Failed to begin editing session: ', error.stack); @@ -99,21 +106,6 @@ class DriveService implements IDriveService { if (!url) { throw Error('no url found'); } - //TODO: It would be better to avoid two requests for this operation - const originalFile = await this.getByEditingSessionKey({ editing_session_key }); - - if (!originalFile) { - // TODO: Make a single request and don't require the filename at all - // then in POST /editing_session/... if the key is not found, just - // put it in a users or company "lost and found" folder. - // and accept without error. Because really, if backend doesn't know - // the key anymore, there's not much we can do, and we should get OO - // to clean up and stop trying to upload it. - // but for today: - logger.error("Forgotten OO document that Twake Drive doesn't know the key of.", { editing_session_key, url }); - throw new UnknownKeyInDriveError(`Unknown key ${JSON.stringify(editing_session_key)}`); - // TODO: Distinguish this case from a long disconnected browser tab waking up - } const newFile = await apiService.get({ url, @@ -122,47 +114,30 @@ class DriveService implements IDriveService { const form = new FormData(); - const filename = encodeURIComponent(originalFile.last_version_cache.file_metadata.name); - - form.append('file', newFile, { - filename, - }); - - logger.info('Saving file version to Twake Drive: ', filename); + form.append('file', newFile); - const queryString = keepEditing ? '?keepEditing=true' : ''; + logger.info(`Saving file version to Twake Drive`, { editing_session_key, url }); await apiService.post({ - url: `/internal/services/documents/v1/editing_session/${encodeURIComponent(editing_session_key)}` + queryString, + url: makeEditingSessionItemUrl(editing_session_key, { + keepEditing: keepEditing ? 'true' : null, + }), payload: form, token: user_token, headers: form.getHeaders(), }); } catch (error) { - logger.error('Failed to end editing session: ', error.stack); - throw error; - //TODO make monitoring for such kind of errors + if (error.response?.status === 404) { + logger.error('Forgotten OO document that Twake Drive doesnt know the key of.', { editing_session_key, url }); + throw new UnknownKeyInDriveError(`Unknown key ${JSON.stringify(editing_session_key)}`); + //TODO: Distinguish this case from a long disconnected browser tab waking up + //TODO make monitoring for such kind of errors + } else { + logger.error('Failed to end editing session: ', error.stack); + throw error; + } } } - - /** - * Get the document information by the editing session key. Just simple call to the drive API - * /item/editing_session/${editing_session_key} - * @param params - * @returns null if the key was not found, or the api response body - */ - public getByEditingSessionKey = async (params: { editing_session_key: string; user_token?: string }): Promise => { - try { - const { editing_session_key } = params; - return await apiService.get({ - url: `/internal/services/documents/v1/editing_session/${encodeURIComponent(editing_session_key)}`, - token: params.user_token, - }); - } catch (error) { - if (error?.response?.status === 404) return null; - throw error; - } - }; } export default new DriveService(); diff --git a/tdrive/connectors/onlyoffice-connector/src/utils.ts b/tdrive/connectors/onlyoffice-connector/src/utils.ts index 1fc1d55b3..36b731400 100644 --- a/tdrive/connectors/onlyoffice-connector/src/utils.ts +++ b/tdrive/connectors/onlyoffice-connector/src/utils.ts @@ -16,7 +16,7 @@ export type QueryParams = { [key: string]: string | number }; export function joinURL(path: string[], params?: QueryParams) { let joinedPath = path.map(x => x.replace(/(?:^\/+)+|(?:\/+$)/g, '')).join('/'); if (path[path.length - 1].endsWith('/')) joinedPath += '/'; - const paramEntries = Object.entries(params || {}); + const paramEntries = Object.entries(params || {}).filter(([, value]) => value !== undefined && value !== null); if (paramEntries.length === 0) return joinedPath; const query = paramEntries.map(p => p.map(encodeURIComponent).join('=')).join('&'); return joinedPath + (joinedPath.indexOf('?') > -1 ? '&' : '?') + query; From be103934358d0de67fb621ff87c5678362284e73 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Wed, 18 Sep 2024 19:04:04 +0200 Subject: [PATCH 32/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20oo-connector:=20refa?= =?UTF-8?q?ctored=20health=20=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../onlyoffice-connector/src/config/index.ts | 1 + .../controllers/health-status.controller.ts | 23 +++++++------------ .../src/services/api.service.ts | 8 ++++++- .../services/forgotten-processor.service.ts | 6 +++-- .../src/services/health-providers.service.ts | 13 +++++++++++ .../src/services/onlyoffice.service.ts | 8 ++++++- 6 files changed, 40 insertions(+), 19 deletions(-) create mode 100644 tdrive/connectors/onlyoffice-connector/src/services/health-providers.service.ts diff --git a/tdrive/connectors/onlyoffice-connector/src/config/index.ts b/tdrive/connectors/onlyoffice-connector/src/config/index.ts index 5fdc7997e..9dc314361 100644 --- a/tdrive/connectors/onlyoffice-connector/src/config/index.ts +++ b/tdrive/connectors/onlyoffice-connector/src/config/index.ts @@ -13,6 +13,7 @@ export const { SERVER_PREFIX, SERVER_ORIGIN, INSTANCE_ID, + OOCONNECTOR_HEALTH_SECRET, } = process.env; const secs = 1000, diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts index 97fa28262..61ca91be3 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/health-status.controller.ts @@ -1,23 +1,16 @@ import { Request, Response } from 'express'; -import onlyofficeService from '@/services/onlyoffice.service'; -import apiService from '@/services/api.service'; - -import forgottenProcessorService from '@/services/forgotten-processor.service'; +import { getProviders } from '@/services/health-providers.service'; +import { OOCONNECTOR_HEALTH_SECRET } from '@/config'; /** * Health response for devops operational purposes. Should not be exposed. */ export const HealthStatusController = { - async get(req: Request<{}, {}, {}, {}>, res: Response): Promise { - Promise.all([onlyofficeService.getLatestLicence(), apiService.hasToken(), forgottenProcessorService.getHealthData()]).then( - ([onlyOfficeLicense, twakeDriveToken, forgottenProcessorHealth]) => - res.status(onlyOfficeLicense && twakeDriveToken ? 200 : 500).send({ - uptimeS: Math.floor(process.uptime()), - onlyOfficeLicense, - twakeDriveToken, - ...forgottenProcessorHealth, - }), - err => res.status(500).send(err), - ); + async get(req: Request<{}, {}, {}, { secret: string }>, res: Response): Promise { + if (req.query.secret !== OOCONNECTOR_HEALTH_SECRET) return void res.status(404).send(`Cannot GET ${req.path}`); + const entries = await Promise.all(getProviders().map(p => p.getHealthData())); + let result = {}; + entries.forEach(entry => (result = { ...result, ...entry })); + res.send(result); }, }; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts index 4d26860e1..92a75c3d9 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/api.service.ts @@ -9,22 +9,28 @@ import { CREDENTIALS_ENDPOINT, CREDENTIALS_ID, CREDENTIALS_SECRET, twakeDriveTok import logger from '../lib/logger'; import * as Utils from '@/utils'; import { PolledThingieValue } from '@/lib/polled-thingie-value'; +import { IHealthProvider, registerHealthProvider } from './health-providers.service'; /** * Client for the Twake Drive backend API on behalf of the plugin (or provided token in parameters). * Periodically updates authorization and adds to requests. */ -class ApiService implements IApiService { +class ApiService implements IApiService, IHealthProvider { private readonly tokenPoller: PolledThingieValue; constructor() { this.tokenPoller = new PolledThingieValue('Refresh Twake Drive token', async () => await this.refreshToken(), twakeDriveTokenRefrehPeriodMS); + registerHealthProvider(this); } public async hasToken() { return (await this.tokenPoller.latestValueWithTry()) !== undefined; } + async getHealthData() { + return { TwakeDriveApi: { tokenAgeS: this.tokenPoller.latest()?.ageS ?? -1 } }; + } + private requireAxios() { return this.tokenPoller.requireLatestValueWithTry('No Twake Drive app token.'); } diff --git a/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts index c295af3ac..de47e632c 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/forgotten-processor.service.ts @@ -6,12 +6,13 @@ import { createSingleProcessorLock } from '@/lib/single-processor-lock'; import apiService from './api.service'; import onlyofficeService from './onlyoffice.service'; import driveService, { UnknownKeyInDriveError } from './drive.service'; +import { IHealthProvider, registerHealthProvider } from './health-providers.service'; /** * Periodically poll the Only Office document server for forgotten * files and try to upload to Twake Drive or get rid if unknown. */ -class ForgottenProcessor { +class ForgottenProcessor implements IHealthProvider { private readonly forgottenFilesPoller: PolledThingieValue; public readonly forgottenSynchroniser = createSingleProcessorLock(); private lastStart = 0; @@ -23,13 +24,14 @@ class ForgottenProcessor { async () => (skippedFirst ? await this.processForgottenFiles() : ((skippedFirst = true), -1)), onlyOfficeForgottenFilesCheckPeriodMS, ); + registerHealthProvider(this); } public async getHealthData() { const keys = await onlyofficeService.getForgottenList(); return { forgotten: { - timeSinceLastStartS: ~~((new Date().getTime() - this.lastStart) / 1000), + timeSinceLastStartS: this.lastStart ? ~~((new Date().getTime() - this.lastStart) / 1000) : -1, count: keys?.length ?? 0, locks: this.forgottenSynchroniser.getWorstStats(), }, diff --git a/tdrive/connectors/onlyoffice-connector/src/services/health-providers.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/health-providers.service.ts new file mode 100644 index 000000000..da85ff623 --- /dev/null +++ b/tdrive/connectors/onlyoffice-connector/src/services/health-providers.service.ts @@ -0,0 +1,13 @@ +const providers: IHealthProvider[] = []; + +export interface IHealthProvider { + getHealthData(): Promise<{ [key: string]: unknown }>; +} + +export function registerHealthProvider(provider: IHealthProvider) { + providers.push(provider); +} + +export function getProviders() { + return providers; +} diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index cca8c733a..985729fbc 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -5,6 +5,7 @@ import { PolledThingieValue } from '@/lib/polled-thingie-value'; import { PendingRequestQueue, PendingRequestCallback } from '@/lib/pending-request-matcher'; import logger from '@/lib/logger'; import * as Utils from '@/utils'; +import { IHealthProvider, registerHealthProvider } from './health-providers.service'; /** @see https://api.onlyoffice.com/editors/basic */ export enum ErrorCode { @@ -218,7 +219,7 @@ class CallbackResponseFromCommand { * Exposed OnlyOffice command service * @see https://api.onlyoffice.com/editors/command/ */ -class OnlyOfficeService { +class OnlyOfficeService implements IHealthProvider { private readonly poller: PolledThingieValue; // Technically the timeout field is from the PendingRequestQueue but avoid 2 classes private readonly pendingRequests = new PendingRequestQueue(onlyOfficeCallbackTimeoutMS); @@ -232,6 +233,7 @@ class OnlyOfficeService { }, onlyOfficeConnectivityCheckPeriodMS, ); + registerHealthProvider(this); } /** Get the latest Only Office licence status from polling. If the return is `undefined` * it probably means there is a connection issue contacting the OnlyOffice server @@ -299,6 +301,10 @@ class OnlyOfficeService { return new CommandService.License.Request().post(); } + async getHealthData() { + return { OO: { license: this.poller.latest() } }; + } + /** * Requests a document status and the list of the identifiers of the users who opened the document for editing. * The response will be sent to the callback handler. From 9db0fddc98cc5f1003abff34a3c77f673e6237fc Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Wed, 18 Sep 2024 21:25:30 +0200 Subject: [PATCH 33/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20backend:=20fix=20e2e?= =?UTF-8?q?=20editing=20session=20test,=20make=20ApplicationApiService=20a?= =?UTF-8?q?=20singleton=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/applications-api/index.ts | 24 +++++++++++++++---- .../src/services/documents/services/index.ts | 14 ++++++++--- .../backend/node/test/e2e/common/user-api.ts | 2 +- .../e2e/documents/editing-session.spec.ts | 10 +++++++- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/tdrive/backend/node/src/services/applications-api/index.ts b/tdrive/backend/node/src/services/applications-api/index.ts index 312566cb6..d817e62ad 100644 --- a/tdrive/backend/node/src/services/applications-api/index.ts +++ b/tdrive/backend/node/src/services/applications-api/index.ts @@ -14,6 +14,11 @@ export default class ApplicationsApiService extends TdriveService { version = "1"; name = "applicationsapi"; + private static default: ApplicationsApiService; + public static getDefault() { + return this.default; + } + public async doInit(): Promise { const fastify = this.context.getProvider("webserver").getServer(); fastify.register((instance, _opts, next) => { @@ -70,10 +75,23 @@ export default class ApplicationsApiService extends TdriveService { } } } - + ApplicationsApiService.default = this; return this; } + /** Get the configuration of a given `appId` or `undefined` if unknown */ + public getApplicationConfig(appId: string) { + const apps = config.get("applications.plugins") || []; + return apps.find(app => app.id === appId); + } + + /** Get the configuration of a given `appId` or throw an error if unknown */ + public requireApplicationConfig(appId: string) { + const app = this.getApplicationConfig(appId); + if (!app) throw new Error(`Unknown application.id ${JSON.stringify(appId)}`); + return app; + } + /** Send a request to the plugin by its application id * @param url Full URL that doesn't start with a `/` */ @@ -82,9 +100,7 @@ export default class ApplicationsApiService extends TdriveService { url: string, appId: string, ) { - const apps = config.get("applications.plugins") || []; - const app = apps.find(app => app.id === appId); - if (!app) throw new Error(`Unknown application.id ${JSON.stringify(appId)}`); + const app = this.requireApplicationConfig(appId); if (!app.internal_domain) throw new Error(`application.id ${JSON.stringify(appId)} missing an internal_domain`); const signature = jwt.sign( diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 5e82cb85f..8c70a5ebb 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -61,6 +61,7 @@ import config from "config"; import { MultipartFile } from "@fastify/multipart"; import { UploadOptions } from "src/services/files/types"; import { SortType } from "src/core/platform/services/search/api"; +import ApplicationsApiService from "../../applications-api"; export class DocumentsService { version: "1"; @@ -976,9 +977,16 @@ export class DocumentsService { this.logger.error("invalid execution context"); return null; } - - //TODO: This needs to try in a loop depending on oo-connector response - // when there is already a key + if ( + !editorApplicationId || + !ApplicationsApiService.getDefault().getApplicationConfig(editorApplicationId) + ) { + logger.error(`Missing or invalid application ID: ${JSON.stringify(editorApplicationId)}`); + CrudException.throwMe( + new Error("Unknown appId"), + new CrudException("Missing or invalid application ID", 400), + ); + } let newKey: string; try { newKey = EditingSessionKeyFormat.generate( diff --git a/tdrive/backend/node/test/e2e/common/user-api.ts b/tdrive/backend/node/test/e2e/common/user-api.ts index ec047e686..133a3efff 100644 --- a/tdrive/backend/node/test/e2e/common/user-api.ts +++ b/tdrive/backend/node/test/e2e/common/user-api.ts @@ -516,7 +516,7 @@ export default class UserApi { async getDocumentByEditingKey(editing_session_key: string) { return await this.platform.app.inject({ method: "GET", - url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/editing_session/${encodeURIComponent(editing_session_key)}`, + url: `${UserApi.DOC_URL}/editing_session/${encodeURIComponent(editing_session_key)}`, headers: { authorization: `Bearer ${this.jwt}` } diff --git a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts index 845ec9c94..fd5d02317 100644 --- a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts +++ b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts @@ -1,10 +1,13 @@ -import { describe, beforeAll, beforeEach, it, expect, afterAll } from "@jest/globals"; +import { describe, beforeAll, beforeEach, it, expect, afterAll, jest } from "@jest/globals"; import { init, TestPlatform } from "../setup"; import UserApi from "../common/user-api"; import { DriveFile, TYPE as DriveFileType } from "../../../src/services/documents/entities/drive-file"; import exp = require("node:constants"); +import ApplicationsApiService from "../../../src/services/applications-api"; +import { afterEach } from "node:test"; +import Application from "../../../src/services/applications/entities/application"; describe("the Drive's documents' editing session kind-of-lock", () => { let platform: TestPlatform | null; @@ -49,6 +52,11 @@ describe("the Drive's documents' editing session kind-of-lock", () => { parent_id: currentUserRoot, scope: "personal", }); + jest.spyOn(ApplicationsApiService.getDefault(), 'getApplicationConfig').mockImplementation((id) => id === "e2e_testing" ? {} as Application : undefined); + }); + + afterEach(() => { + jest.restoreAllMocks(); }); it("atomicCompareAndSet allows a single value at a time", async () => { From c23984355dd60739cb37313c5eb556c0a572b65b Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 00:55:55 +0200 Subject: [PATCH 34/52] =?UTF-8?q?=F0=9F=A9=B9=20backend,oo:=20fixed=20begi?= =?UTF-8?q?n=20editing=20session=20process=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/orm/repository/repository.ts | 10 ++- .../src/services/applications-api/index.ts | 24 ++++- .../src/services/documents/services/index.ts | 75 +++++++++++----- .../backend-callbacks.controller.ts | 90 +++++++++++-------- .../src/controllers/onlyoffice.controller.ts | 15 ++-- 5 files changed, 140 insertions(+), 74 deletions(-) diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts index 250385834..e79df829c 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/repository/repository.ts @@ -43,6 +43,11 @@ export type FindOptions = { sort?: SortOption; }; +export type AtomicCompareAndSetResult = { + didSet: boolean; + currentValue: FieldValueType | null; +}; + /** * Repository to work with entities. Each entity type has its own repository instance. */ @@ -129,10 +134,7 @@ export default class Repository { fieldName: keyof EntityType, previousValue: FieldValueType | null, newValue: FieldValueType | null, - ): Promise<{ - didSet: boolean; - currentValue: FieldValueType | null; - }> { + ): Promise> { if (previousValue === newValue) throw new Error(`Previous and new values are identical: ${JSON.stringify(previousValue)}`); return this.connector.atomicCompareAndSet(entity, fieldName, previousValue, newValue); diff --git a/tdrive/backend/node/src/services/applications-api/index.ts b/tdrive/backend/node/src/services/applications-api/index.ts index d817e62ad..c0aee25df 100644 --- a/tdrive/backend/node/src/services/applications-api/index.ts +++ b/tdrive/backend/node/src/services/applications-api/index.ts @@ -9,6 +9,17 @@ import { logger } from "../../core/platform/framework/logger"; import { EditingSessionKeyFormat } from "../documents/entities/drive-file"; import jwt from "jsonwebtoken"; +export enum ApplicationEditingKeyStatus { + /** the key isn't known and maybe used for a new session */ + unknown = "unknown", + /** the key needed updating but is now invalid */ + updated = "updated", + /** the key was already used in a finished session and can't be used again */ + expired = "expired", + /** the key is valid and current and should be used again for the same file */ + live = "live", +} + @Prefix("/api") export default class ApplicationsApiService extends TdriveService { version = "1"; @@ -128,17 +139,22 @@ export default class ApplicationsApiService extends TdriveService { /** * Check status of `editing_session_key` in the corresponding application. * @param editingSessionKey {@see DriveFile.editing_session_key} to check - * @returns a URL string if there is a pending version to add, `null` - * if the key is unknown. + * @returns status of the provided key as far as the application knows */ - async checkPendingEditingStatus(editingSessionKey: string): Promise { + async checkPendingEditingStatus(editingSessionKey: string): Promise { const parsedKey = EditingSessionKeyFormat.parse(editingSessionKey); const response = await this.requestFromApplication( "POST", "tdriveApi/1/session/" + encodeURIComponent(editingSessionKey) + "/check", parsedKey.applicationId, ); - return (response.data.url as string) || null; + if (response.status != 200 || response.data.error) + throw new Error( + `Application check key ${editingSessionKey} failed with HTTP ${ + response.status + }: ${JSON.stringify(response.data)}`, + ); + return (response.data.status as ApplicationEditingKeyStatus) || null; } /** diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 8c70a5ebb..4b9e431c6 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -6,6 +6,7 @@ import { Pagination, } from "../../../core/platform/framework/api/crud-service"; import Repository, { + AtomicCompareAndSetResult, comparisonType, inType, } from "../../../core/platform/services/database/services/orm/repository/repository"; @@ -61,7 +62,7 @@ import config from "config"; import { MultipartFile } from "@fastify/multipart"; import { UploadOptions } from "src/services/files/types"; import { SortType } from "src/core/platform/services/search/api"; -import ApplicationsApiService from "../../applications-api"; +import ApplicationsApiService, { ApplicationEditingKeyStatus } from "../../applications-api"; export class DocumentsService { version: "1"; @@ -987,18 +988,45 @@ export class DocumentsService { new CrudException("Missing or invalid application ID", 400), ); } - let newKey: string; - try { - newKey = EditingSessionKeyFormat.generate( - editorApplicationId, - appInstanceId, - context.company.id, - context.user.id, - ); - } catch (e) { - logger.error(`Error generating new editing_session_key: ${e}`, { error: e }); - CrudException.throwMe(e, new CrudException("Error generating new editing_session_key", 500)); - } + const spinLoopUntilEditable = async ( + provider: { + generateKey: () => string; + atomicSet: ( + key: string | null, + previous: string | null, + ) => Promise>; + getPluginKeyStatus: (key: string) => Promise; + }, + attemptCount = 8, + tarpitS = 1, + tarpitWorsenCoeff = 1.2, + ) => { + while (attemptCount-- > 0) { + const newKey = provider.generateKey(); + const swapResult = await provider.atomicSet(newKey, null); + logger.debug(`Begin edit try ${newKey}, got: ${JSON.stringify(swapResult)}`); + if (swapResult.didSet) return newKey; + if (!swapResult.currentValue) continue; // glitch in the matrix but ok because atomicCompareAndSet is not actually completely atomic + const existingStatus = await provider.getPluginKeyStatus(swapResult.currentValue); + logger.debug(`Begin edit get status of ${newKey}: ${JSON.stringify(existingStatus)}`); + switch (existingStatus) { + case ApplicationEditingKeyStatus.unknown: + case ApplicationEditingKeyStatus.live: + return swapResult.currentValue; + case ApplicationEditingKeyStatus.updated: + case ApplicationEditingKeyStatus.expired: + logger.debug(`Begin edit emptying previous ${swapResult.currentValue}`); + await provider.atomicSet(null, swapResult.currentValue); + break; + default: + throw new Error( + `Unexpected ApplicationEditingKeyStatus: ${JSON.stringify(existingStatus)}`, + ); + } + await new Promise(resolve => setTimeout(resolve, tarpitS * 1000)); + tarpitS *= tarpitWorsenCoeff; + } + }; const hasAccess = await checkAccess(id, null, "write", this.repository, context); if (!hasAccess) { @@ -1018,13 +1046,20 @@ export class DocumentsService { {}, context, ); - const result = await this.repository.atomicCompareAndSet( - driveFile, - "editing_session_key", - null, - newKey, - ); - return { editingSessionKey: result.currentValue }; + const editingSessionKey = await spinLoopUntilEditable({ + atomicSet: (key, previous) => + this.repository.atomicCompareAndSet(driveFile, "editing_session_key", previous, key), + generateKey: () => + EditingSessionKeyFormat.generate( + editorApplicationId, + appInstanceId, + context.company.id, + context.user.id, + ), + getPluginKeyStatus: key => + ApplicationsApiService.getDefault().checkPendingEditingStatus(key), + }); + return { editingSessionKey }; } catch (error) { logger.error({ error: `${error}` }, "Failed to begin editing Drive item"); CrudException.throwMe(error, new CrudException("Failed to begin editing Drive item", 500)); diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts index 82fca7eeb..5475a80b4 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts @@ -1,12 +1,21 @@ import { Request, Response } from 'express'; import logger from '@/lib/logger'; +import { createSingleProcessorLock } from '@/lib/single-processor-lock'; import onlyofficeService, { Callback, CommandError, ErrorCode } from '@/services/onlyoffice.service'; import driveService from '@/services/drive.service'; import forgottenProcessorService from '@/services/forgotten-processor.service'; +import { IHealthProvider, registerHealthProvider } from '@/services/health-providers.service'; interface RequestQuery { editing_session_key: string; } +const keyCheckLock = createSingleProcessorLock<[status: number, body: unknown]>(); + +registerHealthProvider({ + async getHealthData() { + return { checks: { locks: keyCheckLock.getWorstStats() } }; + }, +}); /** * These routes are called by Twake Drive backend, for ex. before editing or retreiving a file, @@ -26,51 +35,54 @@ export default class TwakeDriveBackendCallbackController { * - `{ error: number }`: there was an error retreiving the status of the key, http status `!= 200` */ public async checkSessionStatus(req: Request, res: Response): Promise { - try { - const forgottenURL = await onlyofficeService.getForgotten(req.params.editing_session_key); + const [status, body] = await keyCheckLock.runWithLock(req.params.editing_session_key, async () => { try { - await forgottenProcessorService.processForgottenFile(req.params.editing_session_key, forgottenURL); - } catch (error) { - logger.error(`processForgottenFile failed`, { error }); - return void res.status(502).send({ error: -57650 }); + const forgottenURL = await onlyofficeService.getForgotten(req.params.editing_session_key); + try { + await forgottenProcessorService.processForgottenFile(req.params.editing_session_key, forgottenURL); + } catch (error) { + logger.error(`processForgottenFile failed`, { error }); + return [502, { error: -57650 }]; + } + return [200, { status: 'updated' }]; + } catch (e) { + if (!(e instanceof CommandError && e.errorCode == ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND)) { + logger.error(`getForgotten failed`, { error: e }); + return [e instanceof CommandError ? 502 : 500, { error: -57651 }]; + } + } + const info = await onlyofficeService.getInfoAndWaitForCallbackUnsafe(req.params.editing_session_key); + if (info.error === ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND) { + // just start using it + return [200, { status: 'unknown' }]; } - return void res.send({ status: 'updated' }); - } catch (e) { - if (!(e instanceof CommandError && e.errorCode == ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND)) { - logger.error(`getForgotten failed`, { error: e }); - return void res.status(e instanceof CommandError ? 502 : 500).send({ error: -57651 }); + if (info.error !== undefined) { + logger.error(`getInfo failed`, { error: info }); + return [502, { error: -57652 }]; } - } - const info = await onlyofficeService.getInfoAndWaitForCallbackUnsafe(req.params.editing_session_key); - if (info.error === ErrorCode.KEY_MISSING_OR_DOC_NOT_FOUND) { - // just start using it - return void res.send({ status: 'unknown' }); - } - if (info.error !== undefined) { - logger.error(`getInfo failed`, { error: info }); - return void res.status(502).send({ error: -57652 }); - } - switch (info.result.status) { - case Callback.Status.BEING_EDITED: - case Callback.Status.BEING_EDITED_BUT_IS_SAVED: - // use it as is - return void res.send({ status: 'live' }); + switch (info.result.status) { + case Callback.Status.BEING_EDITED: + case Callback.Status.BEING_EDITED_BUT_IS_SAVED: + // use it as is + return [200, { status: 'live' }]; - case Callback.Status.CLOSED_WITHOUT_CHANGES: - // just cancel it - return void res.send({ status: 'expired' }); + case Callback.Status.CLOSED_WITHOUT_CHANGES: + // just cancel it + return [200, { status: 'expired' }]; - case Callback.Status.ERROR_FORCE_SAVING: - case Callback.Status.ERROR_SAVING: - return void res.status(502).send({ error: info.result.status }); + case Callback.Status.ERROR_FORCE_SAVING: + case Callback.Status.ERROR_SAVING: + return [502, { error: info.result.status }]; - case Callback.Status.READY_FOR_SAVING: - // upload it, have to do it here for correct user stored in url in OO - await driveService.endEditing(req.params.editing_session_key, info.result.url); - return void res.send({ status: 'updated' }); + case Callback.Status.READY_FOR_SAVING: + // upload it, have to do it here for correct user stored in url in OO + await driveService.endEditing(req.params.editing_session_key, info.result.url); + return [200, { status: 'updated' }]; - default: - throw new Error(`Unexpected callback status: ${JSON.stringify(info.result)}`); - } + default: + throw new Error(`Unexpected callback status: ${JSON.stringify(info.result)}`); + } + }); + await res.status(status).send(body); } } diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 112e25be1..f472d34a0 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -73,13 +73,6 @@ class OnlyOfficeController { const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; const { preview, /* company_id, file_id, user_id, drive_file_id, */ in_page_token /* editing_session_key */ } = officeTokenPayload; - // Ignore errors generated by pending request - // try-catch not needed because it is async - // there may be later reasons to wait for callbacks - // to process and eventually respond accordingly to - // OO an error for certain statuses - void OnlyOffice.default.ooCallbackCalled(req.body); // has to be single thread per key - // check token is an in_page_token and allow save if (!in_page_token) throw new Error('OO Callback invalid token, must be a in_page_token'); if (preview) throw new Error('OO Callback invalid token, must not be a preview token for save operation'); @@ -125,6 +118,14 @@ class OnlyOfficeController { `OO Callback unexpected status field: ${OnlyOffice.Callback.StatusToString(req.body.status)} in ${JSON.stringify(req.body)}`, ); } + + // Ignore errors generated by pending request + // try-catch not needed because it is async + // there may be later reasons to wait for callbacks + // to process and eventually respond accordingly to + // OO an error for certain statuses + void OnlyOffice.default.ooCallbackCalled(req.body); // has to be single thread per key + return respondToOO(0); } catch (error) { logger.error(`OO Callback root error`, { error }); From 1ee4f57e050e54fd6f0a58db8008e3c835f8203c Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 01:28:29 +0200 Subject: [PATCH 35/52] =?UTF-8?q?=E2=9C=85=20backend:=20mock=20oo-connecto?= =?UTF-8?q?r=20response=20to=20fix=20an=20e2e=20test=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdrive/backend/node/test/e2e/documents/editing-session.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts index fd5d02317..fae3f7475 100644 --- a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts +++ b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts @@ -5,7 +5,7 @@ import UserApi from "../common/user-api"; import { DriveFile, TYPE as DriveFileType } from "../../../src/services/documents/entities/drive-file"; import exp = require("node:constants"); -import ApplicationsApiService from "../../../src/services/applications-api"; +import ApplicationsApiService, { ApplicationEditingKeyStatus } from "../../../src/services/applications-api"; import { afterEach } from "node:test"; import Application from "../../../src/services/applications/entities/application"; @@ -53,6 +53,7 @@ describe("the Drive's documents' editing session kind-of-lock", () => { scope: "personal", }); jest.spyOn(ApplicationsApiService.getDefault(), 'getApplicationConfig').mockImplementation((id) => id === "e2e_testing" ? {} as Application : undefined); + jest.spyOn(ApplicationsApiService.getDefault(), 'checkPendingEditingStatus').mockImplementation(async () => ApplicationEditingKeyStatus.unknown); }); afterEach(() => { From 021d82ced06f1dac119d977519fd5a8885813f87 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 01:41:55 +0200 Subject: [PATCH 36/52] =?UTF-8?q?=F0=9F=93=9D=20doc:=20bit=20of=20document?= =?UTF-8?q?ation=20about=20the=20editing=20session=20key=20system=20for=20?= =?UTF-8?q?plugins=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Documentation/docs/plugins.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Documentation/docs/plugins.md b/Documentation/docs/plugins.md index 53fbf43b4..581acaabe 100644 --- a/Documentation/docs/plugins.md +++ b/Documentation/docs/plugins.md @@ -133,6 +133,36 @@ When a user requests a preview and then possibly to edit the file, an IFrame is In the case of the OnlyOffice application, these URLs are pointing to the connector plugin, which then proxies back and forth with the OnlyOffice document server. +#### Editing session key + +When editing a file, at first, a new session key is created on the file by the +backend and atomically swapped. + +This session key uniquely identifies the user, company, and plugin and it is not encrypted. + +An editing session key prevents others from starting, can be updated with new versions, and +ended with and without a new version. + +See operations `beginEditing` and `updateEditing` in `tdrive/backend/node/src/services/documents/services/index.ts` for operations available on that key. + +#### API to expose by the application + +Authentication from the Twake Drive backend is a JWT with the property `type` being +`"tdriveToApplication"` and signed with the shared secret. The application must accept +multiple parallel requests for the same key gracefully. + +- `POST /tdriveApi/1/session/${editing_session_key}/check` + + Sent by Twake Drive backend when beginning a new editing session or investigating + stored keys. The application is expected to process the key if possible before responding, + and provide a response in JSON. The `error` key of that body should be truthy if the response + is not known. Otherwise it should respond: + + - `{ status: 'unknown' }`: the key isn't known and maybe used for a new session + - `{ status: 'updated' }`: the key needed updating but is now invalid + - `{ status: 'expired' }`: the key was already used in a finished session and can't be used again + - `{ status: 'live' }`: the key is valid and current and should be used again for the same file + ### Example: OnlyOffice plugin The [OnlyOffice connector plugin](https://github.com/linagora/twake-drive/tree/main/tdrive/connectors/onlyoffice-connector) is an example of plugin. It's readme includes an example configuration for the backend. From d353ec80887aa0dce3c359d5ceba111413341efe Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 02:06:21 +0200 Subject: [PATCH 37/52] =?UTF-8?q?=F0=9F=A9=B9=20cli:=20fix=20details=20of?= =?UTF-8?q?=20editing=5Fsession=20list=20command=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/cli/cmds/editing_session_cmds/list.ts | 47 ++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts index e796a7f5f..97e2d68ed 100644 --- a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts +++ b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/list.ts @@ -47,15 +47,23 @@ async function report(platform: TdrivePlatform, args: ListArguments) { const versionsRepo = await platform .getProvider("database") .getRepository(FileVersion_TYPE, FileVersion); - const filter = {}; + const filter = { + is_in_trash: false, + }; if (!args.all) filter["editing_session_key"] = { $ne: null }; const opts: FindOptions = { sort: { name: "asc" } }; if (args.name) filter["name"] = args.name; const editedFiles = (await drivesRepo.find(filter, opts)).getEntities(); + const formatDate = (date: Date) => date.toISOString(); + const formatTS = (ts: number) => formatDate(new Date(ts)); console.error(`DriveFiles${args.all ? "" : " with non-null editing_session_key"}:`); console.error(""); for (const dfile of editedFiles) { - console.error(`- ${dfile.name} (${dfile.id}) has key:`); + console.error(`- ${dfile.name} (${dfile.id}) of ${await formatUser(dfile.creator)}`); + if (dfile.scope !== "personal") console.error(` - scope: ${dfile.scope}`); + if (dfile.is_directory) console.error(" - directory !"); + if (dfile.is_in_trash) console.error(" - in trash !"); + console.error(` - modified: ${formatTS(dfile.last_modified)}`); if (dfile.editing_session_key) { const parsed = EditingSessionKeyFormat.parse(dfile.editing_session_key); console.error(" - editing_session_key:"); @@ -69,7 +77,7 @@ async function report(platform: TdrivePlatform, args: ListArguments) { })`, ); console.error( - ` - timestamp: ${parsed.timestamp.toISOString()} (${Math.floor( + ` - timestamp: ${formatDate(parsed.timestamp)} (${Math.floor( (new Date().getTime() - parsed.timestamp.getTime()) / 1000, )}s ago)`, ); @@ -79,12 +87,11 @@ async function report(platform: TdrivePlatform, args: ListArguments) { await versionsRepo.find({ drive_item_id: dfile.id }, { sort: { date_added: "asc" } }) ).getEntities(); let previousSize = 0; + let lastVersion: FileVersion; console.error(" - Versions:"); for (const version of versions) { console.error( - ` - ${new Date(version.date_added).toISOString()} by ${await formatUser( - version.creator_id, - )}`, + ` - ${formatTS(version.date_added)} by ${await formatUser(version.creator_id)}`, ); console.error(` - id: ${version.id}`); console.error( @@ -93,8 +100,36 @@ async function report(platform: TdrivePlatform, args: ListArguments) { }${version.file_metadata.size - previousSize})`, ); previousSize = version.file_metadata.size; + lastVersion = version; console.error(` - application: ${JSON.stringify(version.application_id)}`); } + if (previousSize != dfile.size) + console.error( + ` - mismatched sizes: DriveFile.size is ${dfile.size} but last Version.file_metadata is ${previousSize}`, + ); + if (lastVersion) { + const lastTimestamp = lastVersion.date_added; + if (lastTimestamp != dfile.last_modified) + console.error( + ` - mismatched FileVersion.date_added (${formatTS( + lastTimestamp, + )}) != DriveFile.last_modified (${formatTS(dfile.last_modified)}) - delta: ${ + (lastTimestamp - dfile.last_modified) / 1000 + }s`, + ); + if (lastTimestamp != dfile.last_version_cache.date_added) + console.error( + ` - mismatched FileVersion.date_added (${formatTS( + lastTimestamp, + )}) != DriveFile.last_version_cache.date_added (${formatTS( + dfile.last_version_cache.date_added, + )}) - delta: ${(lastTimestamp - dfile.last_version_cache.date_added) / 1000}s`, + ); + if (lastVersion.file_size != dfile.size) + console.error( + ` - mismatched FileVersion.file_size (${lastVersion.file_size}) != DriveFile.dfile.size (${dfile.size})`, + ); + } } if (!editedFiles.length) console.error(" (no matching DriveFiles)"); } From 5ae17824906651127acf513c30892893a866a88d Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 02:45:27 +0200 Subject: [PATCH 38/52] =?UTF-8?q?=F0=9F=90=9B=20backend:=20when=20adding?= =?UTF-8?q?=20a=20FileVersion,=20update=20DriveFile.last=5Fmodified=20(#52?= =?UTF-8?q?5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdrive/backend/node/src/services/documents/services/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 4b9e431c6..c8f08854e 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -918,6 +918,7 @@ export class DocumentsService { item.last_version_cache = driveItemVersion; item.size = driveItemVersion.file_size; + item.last_modified = driveItemVersion.date_added; await this.repository.save(item); From c855c296642b99990e94f669b1f77d0e483bd9fd Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 02:50:36 +0200 Subject: [PATCH 39/52] =?UTF-8?q?=F0=9F=A9=B9=20backend,oo:=20adding=20use?= =?UTF-8?q?rId=20override=20for=20application=20updatingEditingSession=20(?= =?UTF-8?q?#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/documents/services/index.ts | 18 ++++++++++++++++-- .../documents/web/controllers/documents.ts | 5 ++++- .../backend/node/test/e2e/common/user-api.ts | 12 ++++++++---- .../test/e2e/documents/editing-session.spec.ts | 3 +-- .../src/services/drive.service.ts | 12 ++++++------ 5 files changed, 35 insertions(+), 15 deletions(-) diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index c8f08854e..744b73f32 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -1075,6 +1075,8 @@ export class DocumentsService { * @param options Optional upload information from the request * @param keepEditing If `true`, the file will be saved as a new version, * and the DriveFile will keep its editing_session_key. If `true`, a file is required. + * @param userId When authentified by the root token of an application, this user + * will override the creator of this version * @param context */ updateEditing = async ( @@ -1082,8 +1084,12 @@ export class DocumentsService { file: MultipartFile, options: UploadOptions, keepEditing: boolean, + userId: string | null, context: CompanyExecutionContext, ) => { + //TODO rethink the locking stuff shouldn't be just forgotten + //TODO Make this accept even if missing and act ok about it, + // store to dump folder or such if (!context) { this.logger.error("invalid execution context"); return null; @@ -1092,14 +1098,22 @@ export class DocumentsService { this.logger.error("Invalid editing_session_key: " + JSON.stringify(editing_session_key)); throw new CrudException("Invalid editing_session_key", 400); } - - //TODO If the app is the "user" calling, set user to that from the parsed key try { const parsedKey = EditingSessionKeyFormat.parse(editing_session_key); context = { ...context, company: { id: parsedKey.companyId }, }; + + if (context.user.id === context.user.application_id && context.user.application_id) { + context = { + ...context, + user: { + ...context.user, + id: userId || parsedKey.userId, + }, + }; + } } catch (e) { this.logger.error( "Invalid editing_session_key value: " + JSON.stringify(editing_session_key), diff --git a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts index a094f604a..655e9a41d 100644 --- a/tdrive/backend/node/src/services/documents/web/controllers/documents.ts +++ b/tdrive/backend/node/src/services/documents/web/controllers/documents.ts @@ -386,6 +386,7 @@ export class DocumentsController { null, null, false, + null, context, ); } catch (error) { @@ -401,7 +402,7 @@ export class DocumentsController { updateEditing = async ( request: FastifyRequest<{ Params: ItemRequestByEditingSessionKeyParams; - Querystring: { keepEditing?: string }; + Querystring: { keepEditing?: string; userId?: string }; Body: { item: Partial; version: Partial; @@ -430,6 +431,7 @@ export class DocumentsController { file, options, request.query.keepEditing == "true", + request.query.userId, context, ); } else { @@ -438,6 +440,7 @@ export class DocumentsController { null, null, true, + request.query.userId, context, ); } diff --git a/tdrive/backend/node/test/e2e/common/user-api.ts b/tdrive/backend/node/test/e2e/common/user-api.ts index 133a3efff..0ce15cde9 100644 --- a/tdrive/backend/node/test/e2e/common/user-api.ts +++ b/tdrive/backend/node/test/e2e/common/user-api.ts @@ -409,17 +409,21 @@ export default class UserApi { }); } - async endEditingDocument( - editingSessionKey: string + async updateEditingDocument( + editingSessionKey: string, + keepEditing: boolean = false, + userId: string | null = null, ): Promise { const fullPath = `${__dirname}/assets/${UserApi.ALL_FILES[0]}`; const readable= Readable.from(fs.createReadStream(fullPath)); const form = formAutoContent({ file: readable }); form.headers["authorization"] = `Bearer ${this.jwt}`; - + let queryString = keepEditing ? "keepEditing=true" : ""; + if (userId) + queryString += `${queryString.length ? "&" : ""}userId=${encodeURIComponent(userId)}`; return await this.platform.app.inject({ method: "POST", - url: `${UserApi.DOC_URL}/editing_session/${editingSessionKey}`, + url: `${UserApi.DOC_URL}/editing_session/${encodeURIComponent(editingSessionKey)}${queryString ? "?" : ""}${queryString}`, headers: { authorization: `Bearer ${this.jwt}` }, diff --git a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts index fae3f7475..3f9660499 100644 --- a/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts +++ b/tdrive/backend/node/test/e2e/documents/editing-session.spec.ts @@ -4,7 +4,6 @@ import { init, TestPlatform } from "../setup"; import UserApi from "../common/user-api"; import { DriveFile, TYPE as DriveFileType } from "../../../src/services/documents/entities/drive-file"; -import exp = require("node:constants"); import ApplicationsApiService, { ApplicationEditingKeyStatus } from "../../../src/services/applications-api"; import { afterEach } from "node:test"; import Application from "../../../src/services/applications/entities/application"; @@ -135,7 +134,7 @@ describe("the Drive's documents' editing session kind-of-lock", () => { //given const editingSessionKey = await currentUser.beginEditingDocumentExpectOk(temporaryDocument.id, 'e2e_testing'); //when - const response = await currentUser.endEditingDocument(editingSessionKey); + const response = await currentUser.updateEditingDocument(editingSessionKey); //then expect(response.statusCode).toBe(200); diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index c006991ba..2199bb661 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -93,15 +93,15 @@ class DriveService implements IDriveService { } } - public async addEditingSessionVersion(editing_session_key: string, url: string, user_token?: string) { - return this.updateEditing(editing_session_key, url, true, user_token); + public async addEditingSessionVersion(editing_session_key: string, url: string, userId?: string) { + return this.updateEditing(editing_session_key, url, true, userId); } - public async endEditing(editing_session_key: string, url: string, user_token?: string) { - return this.updateEditing(editing_session_key, url, false, user_token); + public async endEditing(editing_session_key: string, url: string, userId?: string) { + return this.updateEditing(editing_session_key, url, false, userId); } - private async updateEditing(editing_session_key: string, url: string, keepEditing: boolean, user_token?: string) { + private async updateEditing(editing_session_key: string, url: string, keepEditing: boolean, userId?: string) { try { if (!url) { throw Error('no url found'); @@ -121,9 +121,9 @@ class DriveService implements IDriveService { await apiService.post({ url: makeEditingSessionItemUrl(editing_session_key, { keepEditing: keepEditing ? 'true' : null, + userId, }), payload: form, - token: user_token, headers: form.getHeaders(), }); } catch (error) { From b6d66d62b5736cb8171f342cfe9fe1dec09178d9 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 02:51:58 +0200 Subject: [PATCH 40/52] =?UTF-8?q?=F0=9F=9A=A8=20oo-connector:=20minor=20li?= =?UTF-8?q?nter=20cleanup=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/controllers/backend-callbacks.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts index 5475a80b4..10f313ff2 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts @@ -4,7 +4,7 @@ import { createSingleProcessorLock } from '@/lib/single-processor-lock'; import onlyofficeService, { Callback, CommandError, ErrorCode } from '@/services/onlyoffice.service'; import driveService from '@/services/drive.service'; import forgottenProcessorService from '@/services/forgotten-processor.service'; -import { IHealthProvider, registerHealthProvider } from '@/services/health-providers.service'; +import { registerHealthProvider } from '@/services/health-providers.service'; interface RequestQuery { editing_session_key: string; From c0182cff5113204d2ab60e7784f84ca907c01cad Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 03:14:49 +0200 Subject: [PATCH 41/52] =?UTF-8?q?=F0=9F=A9=B9=20oo-connector:=20respect=20?= =?UTF-8?q?user=20picked=20by=20OO=20for=20callback=20url=20in=20updateSes?= =?UTF-8?q?sion=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../orm/connectors/postgres/postgres-query-builder.ts | 2 +- .../src/controllers/onlyoffice.controller.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts index 441ff0b08..a223035ee 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres-query-builder.ts @@ -47,7 +47,7 @@ export class PostgresQueryBuilder { values.push(...inClause); } } else { - const isANotEqualFilter = filter && Object.keys(filter).join("") === "$ne"; + const isANotEqualFilter = filter && Object.keys(filter).join("!") === "$ne"; if (filter === null || (isANotEqualFilter && filter["$ne"] === null)) { whereClause += `${key} IS${filter === null ? "" : " NOT"} NULL`; } else { diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index f472d34a0..a5186f888 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -71,7 +71,7 @@ class OnlyOfficeController { req.body, ); const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; - const { preview, /* company_id, file_id, user_id, drive_file_id, */ in_page_token /* editing_session_key */ } = officeTokenPayload; + const { preview, user_id, /* company_id, file_id, drive_file_id, */ in_page_token /* editing_session_key */ } = officeTokenPayload; // check token is an in_page_token and allow save if (!in_page_token) throw new Error('OO Callback invalid token, must be a in_page_token'); @@ -87,12 +87,12 @@ class OnlyOfficeController { case OnlyOffice.Callback.Status.BEING_EDITED_BUT_IS_SAVED: logger.info(`OO Callback force save for session ${key} for reason: ${OnlyOffice.Callback.ForceSaveTypeToString(req.body.forcesavetype)}`); - await driveService.addEditingSessionVersion(key, url); //, token); //TODO Fix user token (getting 401) + await driveService.addEditingSessionVersion(key, url, user_id); break; case OnlyOffice.Callback.Status.READY_FOR_SAVING: logger.info(`OO Callback new version for session ${key} created`); - await driveService.endEditing(key, url); //, token); //TODO Fix user token (getting 401) + await driveService.endEditing(key, url, user_id); break; case OnlyOffice.Callback.Status.CLOSED_WITHOUT_CHANGES: From ea18aafde9a4a1602e4bfde0a093e36f5b13f829 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 03:25:13 +0200 Subject: [PATCH 42/52] =?UTF-8?q?=E2=9A=B0=EF=B8=8F=20backend:=20deleting?= =?UTF-8?q?=20rather=20quite=20boring=20file=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/e2e/common/user-authorization.ts | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 tdrive/backend/node/test/e2e/common/user-authorization.ts diff --git a/tdrive/backend/node/test/e2e/common/user-authorization.ts b/tdrive/backend/node/test/e2e/common/user-authorization.ts deleted file mode 100644 index 8a36ac19c..000000000 --- a/tdrive/backend/node/test/e2e/common/user-authorization.ts +++ /dev/null @@ -1,52 +0,0 @@ -// import { OidcJwtVerifier } from "../../../src/services/console/clients/remote-jwks-verifier"; -// -// export class UserAuthorization { -// /** -// * Just send the login requests without any validation and login response assertion -// */ -// public async login(session?: string) { -// if (session !== undefined) { -// this.session = session; -// } else { -// this.session = uuidv1(); -// } -// const payload = { -// claims: { -// sub: this.user.id, -// first_name: this.user.first_name, -// sid: this.session, -// }, -// }; -// const verifierMock = jest.spyOn(OidcJwtVerifier.prototype, "verifyIdToken"); -// verifierMock.mockImplementation(() => { -// return Promise.resolve(payload); // Return the predefined payload -// }); -// return await this.api.post("/internal/services/console/v1/login", { -// oidc_id_token: "sample_oidc_token", -// }); -// } -// -// public async logout() { -// const payload = { -// claims: { -// iss: "tdrive_lemonldap", -// sub: this.user.id, -// sid: this.session, -// aud: "your-audience", -// iat: Math.floor(Date.now() / 1000), -// jti: "jwt-id", -// events: { -// "http://schemas.openid.net/event/backchannel-logout": {}, -// }, -// } -// }; -// const verifierMock = jest.spyOn(OidcJwtVerifier.prototype, "verifyLogoutToken"); -// verifierMock.mockImplementation(() => { -// return Promise.resolve(payload); // Return the predefined payload -// }); -// -// return await this.api.post("/internal/services/console/v1/backchannel_logout", { -// logout_token: "logout_token_rsa256", -// }); -// } -// } \ No newline at end of file From 5285de4abf45292216818322a91e3ad629ade07a Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 15:25:37 +0200 Subject: [PATCH 43/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=F0=9F=A9=B9=20backend,?= =?UTF-8?q?oo:=20fixing=20instances=20of=20controllers,=20and=20wip=20rena?= =?UTF-8?q?me=20from=20drive=20->=20oo=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/applications-api/index.ts | 15 +++++++++------ .../backend-callbacks.controller.ts | 12 ++++++++++++ .../src/routes/backend-callbacks.route.ts | 3 ++- .../src/routes/browser-editor.route.ts | 4 ++-- .../src/routes/onlyoffice.route.ts | 4 ++-- .../src/services/onlyoffice.service.ts | 19 +++++++++++++++++++ 6 files changed, 46 insertions(+), 11 deletions(-) diff --git a/tdrive/backend/node/src/services/applications-api/index.ts b/tdrive/backend/node/src/services/applications-api/index.ts index c0aee25df..5853e7c8e 100644 --- a/tdrive/backend/node/src/services/applications-api/index.ts +++ b/tdrive/backend/node/src/services/applications-api/index.ts @@ -110,6 +110,7 @@ export default class ApplicationsApiService extends TdriveService { method: "GET" | "POST" | "DELETE", url: string, appId: string, + data?: unknown, ) { const app = this.requireApplicationConfig(appId); if (!app.internal_domain) @@ -129,6 +130,7 @@ export default class ApplicationsApiService extends TdriveService { return axios.request({ url: finalURL, method: method, + data, headers: { Authorization: signature, }, @@ -158,16 +160,17 @@ export default class ApplicationsApiService extends TdriveService { } /** - * Remove any reference to the `editing_session_key` in the plugin - * @param editingSessionKey {@see DriveFile.editing_session_key} to delete - * @returns `true` if the key was deleted + * Change the filename in the external editing session + * @param editingSessionKey {@see DriveFile.editing_session_key} to change + * @param filename The new filename */ - async deleteEditingKey(editingSessionKey: string): Promise { + async renameEditingKeyFilename(editingSessionKey: string, filename: string): Promise { const parsedKey = EditingSessionKeyFormat.parse(editingSessionKey); const response = await this.requestFromApplication( - "DELETE", - "tdriveApi/1/session/" + encodeURIComponent(editingSessionKey), + "POST", + `tdriveApi/1/session/${encodeURIComponent(editingSessionKey)}/title`, parsedKey.applicationId, + { title: filename }, ); return !!response.data.done as boolean; } diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts index 10f313ff2..7ccb76ee5 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/backend-callbacks.controller.ts @@ -9,6 +9,9 @@ import { registerHealthProvider } from '@/services/health-providers.service'; interface RequestQuery { editing_session_key: string; } +interface RenameRequestBody { + title: string; +} const keyCheckLock = createSingleProcessorLock<[status: number, body: unknown]>(); registerHealthProvider({ @@ -85,4 +88,13 @@ export default class TwakeDriveBackendCallbackController { }); await res.status(status).send(body); } + + public async updateSessionFilename(req: Request, res: Response): Promise { + try { + await onlyofficeService.meta(req.params.editing_session_key, req.body.title); + res.send({ ok: 1 }); + } catch (err) { + res.status(500).send({ error: -58650 }); + } + } } diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts index 2e54c12bc..f2f85a395 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/backend-callbacks.route.ts @@ -10,6 +10,7 @@ export const TwakeDriveBackendCallbackRoutes = { mount(router: Router) { const controller = new TwakeDriveBackendCallbackController(); // Why post ? to garantee it is never cached and always ran - router.post('/session/:editing_session_key/check', authMiddleware, controller.checkSessionStatus); + router.post('/session/:editing_session_key/check', authMiddleware, controller.checkSessionStatus.bind(controller)); + router.post('/session/:editing_session_key/title', authMiddleware, controller.updateSessionFilename.bind(controller)); }, }; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts index 4f02de215..f1e1a158c 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/browser-editor.route.ts @@ -10,7 +10,7 @@ import type { Router } from 'express'; export const BrowserEditorRoutes = { mount(router: Router) { const controller = new BrowserEditorController(); - router.get('/', requirementsMiddleware, authMiddleware, controller.index); - router.get('/editor', requirementsMiddleware, authMiddleware, controller.editor); + router.get('/', requirementsMiddleware, authMiddleware, controller.index.bind(controller)); + router.get('/editor', requirementsMiddleware, authMiddleware, controller.editor.bind(controller)); }, }; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts index 0392cc3ce..5faa09c75 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts @@ -9,7 +9,7 @@ import type { Router } from 'express'; export const OnlyOfficeRoutes = { mount(router: Router) { const controller = new OnlyOfficeController(); - router.get(`/:mode/read`, requirementsMiddleware, controller.read); - router.post(`/:mode/callback`, requirementsMiddleware, controller.ooCallback); + router.get(`/:mode/read`, requirementsMiddleware, controller.read.bind(controller)); + router.post(`/:mode/callback`, requirementsMiddleware, controller.ooCallback.bind(controller)); }, }; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts index 985729fbc..425cf6932 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/onlyoffice.service.ts @@ -196,6 +196,17 @@ namespace CommandService { } } } + + export namespace Meta { + export interface Response extends SuccessResponse { + key: string; + } + export class Request extends BaseRequest { + constructor(public readonly key: string, public readonly meta: { title: string }) { + super('meta'); + } + } + } } /** @@ -355,6 +366,14 @@ class OnlyOfficeService implements IHealthProvider { async deleteForgotten(key: string): Promise { return new CommandService.DeleteForgotten.Request(key).post().then(response => response.key); } + /** + * Updates the meta information of the document for all collaborative editors. + * + * That's the official description. It send file renames to OO. + */ + async meta(key: string, title: string): Promise { + return new CommandService.Meta.Request(key, { title }).post().then(response => response.key); + } } export default new OnlyOfficeService(); From a40cb7eae67e329c20df46d827c745c2e239d3d8 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 19 Sep 2024 21:55:35 +0200 Subject: [PATCH 44/52] =?UTF-8?q?=F0=9F=A9=B9=20backend,oo:=20minor=20fixe?= =?UTF-8?q?s=20for=20file=20renames=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/services/documents/services/index.ts | 16 +++++++++++++--- .../src/controllers/browser-editor.controller.ts | 2 +- .../src/interfaces/drive.interface.ts | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tdrive/backend/node/src/services/documents/services/index.ts b/tdrive/backend/node/src/services/documents/services/index.ts index 744b73f32..278816285 100644 --- a/tdrive/backend/node/src/services/documents/services/index.ts +++ b/tdrive/backend/node/src/services/documents/services/index.ts @@ -549,7 +549,7 @@ export class DocumentsService { } const updatable = ["access_info", "name", "tags", "parent_id", "description", "is_in_trash"]; - + let renamedTo: string | undefined; for (const key of updatable) { if ((content as any)[key]) { if ( @@ -590,7 +590,7 @@ export class DocumentsService { } }); } else if (key === "name") { - item.name = await getItemName( + renamedTo = item.name = await getItemName( content.parent_id || item.parent_id, item.id, content.name, @@ -621,7 +621,17 @@ export class DocumentsService { await updateItemSize(oldParent, this.repository, context); } - + if (renamedTo && item.editing_session_key) + ApplicationsApiService.getDefault() + .renameEditingKeyFilename(item.editing_session_key, renamedTo) + .catch(err => { + logger.error("Error rename editing session to new name", { + err, + editing_session_key: item.editing_session_key, + renamedTo, + }); + /* Ignore errors, just throw it out there... */ + }); if (item.parent_id === this.TRASH) { //When moving to trash we recompute the access level to make them flat item.access_info = await makeStandaloneAccessLevel( diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts index a32b98b43..7b9126123 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts @@ -89,7 +89,7 @@ class BrowserEditorController { drive_file_id, editing_session_key: editingSessionKey, file_id: file.id, - file_name: file.filename || file?.metadata?.name || '', + file_name: driveFile?.item?.name || file.filename || file.metadata?.name || '', preview: !!preview, } as OfficeToken, CREDENTIALS_SECRET, diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts index 9755461c9..4384d561e 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/drive.interface.ts @@ -1,6 +1,7 @@ export type DriveFileType = { access: 'manage' | 'write' | 'read' | 'none'; item: { + name: string; last_version_cache: { id: string; date_added: number; From 5616ff111254496f772611f642098ec212686bed Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Sun, 22 Sep 2024 23:34:15 +0200 Subject: [PATCH 45/52] =?UTF-8?q?=F0=9F=8C=90=20frontend:=20fixed=20typo?= =?UTF-8?q?=20in=20rename=20modal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdrive/frontend/public/locales/en.json | 8 ++++---- tdrive/frontend/public/locales/fr.json | 8 ++++---- tdrive/frontend/public/locales/ru.json | 8 ++++---- tdrive/frontend/public/locales/vn.json | 8 ++++---- .../views/client/body/drive/modals/properties/index.tsx | 8 ++++---- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tdrive/frontend/public/locales/en.json b/tdrive/frontend/public/locales/en.json index 550a76f6c..ed30b1707 100644 --- a/tdrive/frontend/public/locales/en.json +++ b/tdrive/frontend/public/locales/en.json @@ -242,10 +242,10 @@ "components.SelectorModalContent_no_items":"No item selected", "components.SelectorModalContent_select":"Selected", "components.SelectorModalContent_files":"files(s)", - "compenents.ProprietiesModalContent_rename":"Rename", - "compenents.ProprietiesModalContent_name":"Name", - "compenents.ProprietiesModalContent_place_holder":"Document or folder name", - "compenents.ProprietiesModalContent_update_button":"Update name", + "components.PropertiesModalContent_rename":"Rename", + "components.PropertiesModalContent_name":"Name", + "components.PropertiesModalContent_place_holder":"Document or folder name", + "components.PropertiesModalContent_update_button":"Update name", "compenents.ConfirmTrashModalContent_move":"Move", "compenents.ConfirmTrashModalContent_to_trash":"to trash", "compenents.ConfirmTrashModalContent_items_to_trash":"items to trash", diff --git a/tdrive/frontend/public/locales/fr.json b/tdrive/frontend/public/locales/fr.json index 57f6bc0a3..ce1d214c2 100644 --- a/tdrive/frontend/public/locales/fr.json +++ b/tdrive/frontend/public/locales/fr.json @@ -218,10 +218,10 @@ "components.SelectorModalContent_no_items": "Pas de fichier sélectionné", "components.SelectorModalContent_select": "sélectionné(s)", "components.SelectorModalContent_files": "fichier(s)", - "compenents.ProprietiesModalContent_rename": "Renommer", - "compenents.ProprietiesModalContent_name": "Nom", - "compenents.ProprietiesModalContent_place_holder": "Nom du fichier ou du document", - "compenents.ProprietiesModalContent_update_button": "Renommer", + "components.PropertiesModalContent_rename": "Renommer", + "components.PropertiesModalContent_name": "Nom", + "components.PropertiesModalContent_place_holder": "Nom du fichier ou du document", + "components.PropertiesModalContent_update_button": "Renommer", "compenents.ConfirmTrashModalContent_move": "Déplacer", "compenents.ConfirmTrashModalContent_to_trash": "vers la corbeille", "compenents.ConfirmTrashModalContent_items_to_trash": "éléments vers la corbeille", diff --git a/tdrive/frontend/public/locales/ru.json b/tdrive/frontend/public/locales/ru.json index a8a256da0..2b4392ca4 100644 --- a/tdrive/frontend/public/locales/ru.json +++ b/tdrive/frontend/public/locales/ru.json @@ -218,10 +218,10 @@ "components.SelectorModalContent_no_items":"No item selected", "components.SelectorModalContent_select":"Selected", "components.SelectorModalContent_files":"files(s)", - "compenents.ProprietiesModalContent_rename":"Rename", - "compenents.ProprietiesModalContent_name":"Name", - "compenents.ProprietiesModalContent_place_holder":"Document or folder name", - "compenents.ProprietiesModalContent_update_button":"Update name", + "components.PropertiesModalContent_rename":"Rename", + "components.PropertiesModalContent_name":"Name", + "components.PropertiesModalContent_place_holder":"Document or folder name", + "components.PropertiesModalContent_update_button":"Update name", "compenents.ConfirmTrashModalContent_move":"Move", "compenents.ConfirmTrashModalContent_to_trash":"to trash", "compenents.ConfirmTrashModalContent_items_to_trash":"items to trash", diff --git a/tdrive/frontend/public/locales/vn.json b/tdrive/frontend/public/locales/vn.json index b40aa0720..39d92f221 100644 --- a/tdrive/frontend/public/locales/vn.json +++ b/tdrive/frontend/public/locales/vn.json @@ -233,10 +233,10 @@ "components.SelectorModalContent_no_items": "Không có mục nào được chọn", "components.SelectorModalContent_select": "Đã chọn", "components.SelectorModalContent_files": "tệp", - "compenents.ProprietiesModalContent_rename": "Đổi tên", - "compenents.ProprietiesModalContent_name": "Tên", - "compenents.ProprietiesModalContent_place_holder": "Tên tài liệu hoặc thư mục", - "compenents.ProprietiesModalContent_update_button": "Cập nhật tên", + "components.PropertiesModalContent_rename": "Đổi tên", + "components.PropertiesModalContent_name": "Tên", + "components.PropertiesModalContent_place_holder": "Tên tài liệu hoặc thư mục", + "components.PropertiesModalContent_update_button": "Cập nhật tên", "compenents.ConfirmTrashModalContent_move": "Di chuyển", "compenents.ConfirmTrashModalContent_to_trash": "vào thùng rác", "compenents.ConfirmTrashModalContent_items_to_trash": "mục vào thùng rác", diff --git a/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx b/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx index e3c75e8b6..ea61ac7ed 100644 --- a/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx +++ b/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx @@ -49,16 +49,16 @@ const PropertiesModalContent = ({ id, onClose }: { id: string; onClose: () => vo return ( setName(e.target.value)} - placeholder={Languages.t('compenents.ProprietiesModalContent_place_holder')} + placeholder={Languages.t('components.PropertiesModalContent_place_holder')} /> } /> @@ -85,7 +85,7 @@ const PropertiesModalContent = ({ id, onClose }: { id: string; onClose: () => vo setLoading(false); }} > - {Languages.t('compenents.ProprietiesModalContent_update_button')} + {Languages.t('components.PropertiesModalContent_update_button')} ); From 77d58d067d11ce3a726895616b038d31db9ff14a Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Mon, 23 Sep 2024 00:04:44 +0200 Subject: [PATCH 46/52] =?UTF-8?q?=F0=9F=92=84=20front:=20fix=20rename=20di?= =?UTF-8?q?alog=20for=20keyboard=20use?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../body/drive/modals/properties/index.tsx | 59 +++++++++++++------ 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx b/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx index ea61ac7ed..3272ef9f6 100644 --- a/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx +++ b/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx @@ -4,7 +4,7 @@ import { Input } from '@atoms/input/input-text'; import { Modal, ModalContent } from '@atoms/modal'; import { useDriveActions } from '@features/drive/hooks/use-drive-actions'; import { useDriveItem } from '@features/drive/hooks/use-drive-item'; -import { useEffect, useState } from 'react'; +import { useEffect, useRef, useState } from 'react'; import { atom, useRecoilState } from 'recoil'; import Languages from '@features/global/services/languages-service'; @@ -38,15 +38,46 @@ const PropertiesModalContent = ({ id, onClose }: { id: string; onClose: () => vo const { update } = useDriveActions(); const [loading, setLoading] = useState(false); const [name, setName] = useState(''); + const inputRef = useRef(null); useEffect(() => { refresh(id); }, []); + useEffect(() => { + setTimeout(() => { + if (inputRef.current) { + inputRef.current.focus(); + const lastDot = inputRef.current.value.lastIndexOf('.'); + const endRange = lastDot >= 0 ? lastDot : inputRef.current.value.length; + inputRef.current.setSelectionRange(0, endRange); + } + }, 100); + }, []); + useEffect(() => { if (!name) setName(item?.name || ''); }, [item?.name]); + const doSave = async () => { + setLoading(true); + if (item) { + let finalName = name; + //TODO: Confirm rename if extension changed ? + if (!item?.is_directory) { + //TODO: Why do we trim extensions on folders ? + const lastDotIndex = finalName.lastIndexOf('.'); + if (lastDotIndex !== -1) { + const fileExtension = name.slice(lastDotIndex); + finalName = finalName.slice(0, lastDotIndex) + fileExtension; + } + } + await update({ name: finalName }, id, item.parent_id); + } + onClose(); + setLoading(false); + } + return ( vo input={ setName(e.target.value)} + onKeyUp={({ key }) => { + if (!loading) { + if (key === 'Enter') + doSave(); + else if (key === "Escape") + onClose(); + } + }} placeholder={Languages.t('components.PropertiesModalContent_place_holder')} /> } @@ -68,22 +108,7 @@ const PropertiesModalContent = ({ id, onClose }: { id: string; onClose: () => vo className="float-right mt-4" theme="primary" loading={loading} - onClick={async () => { - setLoading(true); - if (item) { - let finalName = name; - if (!item?.is_directory) { - const lastDotIndex = finalName.lastIndexOf('.'); - if (lastDotIndex !== -1) { - const fileExtension = name.slice(lastDotIndex); - finalName = finalName.slice(0, lastDotIndex) + fileExtension; - } - } - await update({ name: finalName }, id, item.parent_id); - } - onClose(); - setLoading(false); - }} + onClick={doSave} > {Languages.t('components.PropertiesModalContent_update_button')} From ebe8a78c7f244bc9edcad543b4e5673eb1237a5b Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Mon, 23 Sep 2024 02:56:15 +0200 Subject: [PATCH 47/52] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20oo-connector:=20use?= =?UTF-8?q?=20drive=5Ffile=5Fid=20as=20much=20as=20possible=20in=20OO=20fl?= =?UTF-8?q?ow=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/browser-editor.controller.ts | 4 +++- .../src/controllers/onlyoffice.controller.ts | 17 +++++++++++++++-- .../src/interfaces/editor.interface.ts | 3 ++- .../onlyoffice-connector/src/routes/index.ts | 10 +++++++++- .../src/services/editor.service.ts | 4 ++-- .../onlyoffice-connector/src/views/index.eta | 9 +++++---- 6 files changed, 36 insertions(+), 11 deletions(-) diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts index 7b9126123..d2c82065d 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/browser-editor.controller.ts @@ -22,6 +22,7 @@ interface RequestEditorQuery { office_token: string; company_id: string; file_id: string; + drive_file_id: string; } /** @@ -102,6 +103,7 @@ class BrowserEditorController { makeURLTo.editorAbsolute({ token, file_id, + drive_file_id, editing_session_key: editingSessionKey, company_id, preview, @@ -131,7 +133,7 @@ class BrowserEditorController { throw new Error('Cant start editing without "editing session key"'); } - const initResponse = await editorService.init(company_id, file_name, file_id, user, preview, drive_file_id || file_id); + const initResponse = await editorService.init(company_id, file_name, file_id, user, preview, drive_file_id); const inPageToken = jwt.sign( { diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index a5186f888..1f94d87f4 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -11,6 +11,7 @@ import * as Utils from '@/utils'; interface RequestQuery { company_id: string; file_id: string; + drive_file_id: string; token: string; } @@ -29,15 +30,27 @@ class OnlyOfficeController { const { token } = req.query; const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; - const { company_id, file_id, in_page_token } = officeTokenPayload; + const { company_id, file_id, drive_file_id, in_page_token } = officeTokenPayload; // check token is an in_page_token if (!in_page_token) throw new Error('Invalid token, must be a in_page_token'); + let fileId = file_id; + if (drive_file_id) { + //Get the drive file + const driveFile = await driveService.get({ + company_id, + drive_file_id, + }); + if (driveFile) { + fileId = driveFile?.item?.last_version_cache?.file_metadata?.external_id; + } + } + if (!file_id) throw new Error(`File id is missing in the last version cache for ${JSON.stringify(file_id)}`); const file = await fileService.download({ company_id, - file_id: file_id, + file_id: fileId, }); file.pipe(res); diff --git a/tdrive/connectors/onlyoffice-connector/src/interfaces/editor.interface.ts b/tdrive/connectors/onlyoffice-connector/src/interfaces/editor.interface.ts index 7aef5de3a..c23aedb4b 100644 --- a/tdrive/connectors/onlyoffice-connector/src/interfaces/editor.interface.ts +++ b/tdrive/connectors/onlyoffice-connector/src/interfaces/editor.interface.ts @@ -17,7 +17,7 @@ export type EditConfigInitResult = { onlyoffice_server: string; color: string; company_id: string; - file_id: string; + drive_file_id: string; file_version_id: string; filename: string; file_type: string; @@ -33,6 +33,7 @@ export interface IEditorService { user: UserType, preview: boolean, file_id: string, + drive_file_id: string, ) => Promise; } diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/index.ts b/tdrive/connectors/onlyoffice-connector/src/routes/index.ts index 930c6e120..c930d71aa 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/index.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/index.ts @@ -41,7 +41,15 @@ export function mountRoutes(app: Application) { export const makeURLTo = { rootAbsolute: () => Utils.joinURL([SERVER_ORIGIN, SERVER_PREFIX]), assets: () => Utils.joinURL([SERVER_PREFIX, 'assets']), - editorAbsolute(params: { token: string; file_id: string; editing_session_key: string; company_id: string; preview: string; office_token: string }) { + editorAbsolute(params: { + token: string; + drive_file_id: string; + file_id: string; + editing_session_key: string; + company_id: string; + preview: string; + office_token: string; + }) { return Utils.joinURL([SERVER_ORIGIN ?? '', SERVER_PREFIX, 'editor'], params); }, }; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/editor.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/editor.service.ts index 5cd82f719..12794a0a1 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/editor.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/editor.service.ts @@ -10,7 +10,7 @@ class EditorService implements IEditorService { file_version_id: string, user: UserType, preview: boolean, - file_id: string, + drive_file_id: string, ): Promise => { const { color, mode: fileMode } = this.getFileMode(file_name); let [, extension] = Utils.splitFilename(file_name); @@ -18,7 +18,7 @@ class EditorService implements IEditorService { extension = extension.toLocaleLowerCase(); return { color, - file_id, + drive_file_id, file_version_id, file_type: extension, filename: file_name, diff --git a/tdrive/connectors/onlyoffice-connector/src/views/index.eta b/tdrive/connectors/onlyoffice-connector/src/views/index.eta index 6d9b9ec0d..8cb20ad10 100644 --- a/tdrive/connectors/onlyoffice-connector/src/views/index.eta +++ b/tdrive/connectors/onlyoffice-connector/src/views/index.eta @@ -22,12 +22,13 @@ $('#onlyoffice_container').html("
"); + const callbackQueryString = '?drive_file_id=<%= it.drive_file_id %>&company_id=<%= it.company_id %>&token=<%= it.token %>'; let doc = { title: "<%= it.filename %>", - url: `${window.baseURL}read?file_id=<%= it.file_id %>&company_id=<%= it.company_id %>&token=<%= it.token %>`, + url: `${window.baseURL}read${callbackQueryString}`, fileType: "<%= it.file_type %>", key: "<%= it.docId %>", - token: "<%= it.file_id %>", + token: "<%= it.drive_file_id %>", permissions: { download: true, edit: <%= it.editable %>, @@ -41,10 +42,10 @@ height: '100%', documentType: window.mode, document: doc, - token: "<%= it.file_id %>", + token: "<%= it.drive_file_id %>", type: screen.width < 600 ? 'mobile' : 'desktop', editorConfig: { - callbackUrl: `${window.baseURL}callback?file_id=<%= it.file_id %>&company_id=<%= it.company_id %>&token=<%= it.token %>`, + callbackUrl: `${window.baseURL}callback${callbackQueryString}`, lang: window.user.language, user: { id: window.user.id, From 528d8830fbeffaf0f078e9979b0c70b17be1cd60 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Mon, 23 Sep 2024 02:59:34 +0200 Subject: [PATCH 48/52] =?UTF-8?q?=E2=9C=A8=20oo-connector:=20rename=20from?= =?UTF-8?q?=20OO=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/controllers/onlyoffice.controller.ts | 23 ++++++++++++++++++ .../src/routes/onlyoffice.route.ts | 1 + .../src/services/drive.service.ts | 22 +++++++++++++++++ .../onlyoffice-connector/src/views/index.eta | 24 +++++++++++++++++++ 4 files changed, 70 insertions(+) diff --git a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts index 1f94d87f4..3a7ad59c2 100644 --- a/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts +++ b/tdrive/connectors/onlyoffice-connector/src/controllers/onlyoffice.controller.ts @@ -15,6 +15,10 @@ interface RequestQuery { token: string; } +interface RenameRequestBody { + name: string; +} + /** These expose a OnlyOffice document storage service methods, called by the OnlyOffice document editing service * to load and save files */ @@ -145,6 +149,25 @@ class OnlyOfficeController { next(error || 'error'); } }; + + /** This route is called directly by the inline JS in the editor page, called by the client-side OO editor component */ + public rename = async (req: Request<{}, {}, RenameRequestBody, RequestQuery>, res: Response, next: NextFunction): Promise => { + try { + const { token } = req.query; + const officeTokenPayload = jwt.verify(token, CREDENTIALS_SECRET) as OfficeToken; + const { company_id, drive_file_id } = officeTokenPayload; + const { name } = req.body; + + if (!drive_file_id) throw new Error('OO Rename request missing drive_file_id'); + if (!name) throw new Error('OO Rename request missing name'); + + const result = await driveService.update({ company_id, drive_file_id, changes: { name } }); + res.send(result); + } catch (error) { + logger.error(`OO Rename request root error`, { error }); + next(error || 'error'); + } + }; } export default OnlyOfficeController; diff --git a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts index 5faa09c75..921157cd0 100644 --- a/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts +++ b/tdrive/connectors/onlyoffice-connector/src/routes/onlyoffice.route.ts @@ -11,5 +11,6 @@ export const OnlyOfficeRoutes = { const controller = new OnlyOfficeController(); router.get(`/:mode/read`, requirementsMiddleware, controller.read.bind(controller)); router.post(`/:mode/callback`, requirementsMiddleware, controller.ooCallback.bind(controller)); + router.post(`/:mode/rename`, requirementsMiddleware, controller.rename.bind(controller)); }, }; diff --git a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts index 2199bb661..ebb2bf658 100644 --- a/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts +++ b/tdrive/connectors/onlyoffice-connector/src/services/drive.service.ts @@ -36,6 +36,28 @@ class DriveService implements IDriveService { } }; + public update = async (params: { + company_id: string; + drive_file_id: string; + changes: Partial; + completeResult?: boolean; + }): Promise<(typeof params)['changes']> => { + try { + const { company_id, drive_file_id } = params; + const resource = await apiService.post<(typeof params)['changes'], ReturnType>({ + url: makeNonEditingSessionItemUrl(company_id, drive_file_id), + payload: params.changes, + }); + if (params.completeResult) return resource; + const result = {}; + Object.keys(params.changes).forEach(k => (result[k] = resource[k])); + return result; + } catch (error) { + logger.error('Failed to update file metadata: ', error.stack); + return Promise.reject(); + } + }; + public createVersion = async (params: { company_id: string; drive_file_id: string; diff --git a/tdrive/connectors/onlyoffice-connector/src/views/index.eta b/tdrive/connectors/onlyoffice-connector/src/views/index.eta index 8cb20ad10..fa1c3fd43 100644 --- a/tdrive/connectors/onlyoffice-connector/src/views/index.eta +++ b/tdrive/connectors/onlyoffice-connector/src/views/index.eta @@ -5,6 +5,7 @@ + Twake Drive @@ -35,6 +36,11 @@ preview: <%= it.preview %>, } } + function updateTitle(title) { + const el = document.querySelector('head title'); + if (el) el.innerText = title + ' — Twake Drive'; + } + updateTitle(doc.title); window.docEditor = new DocsAPI.DocEditor('onlyoffice_container_instance', { scrollSensitivity: window.mode === 'text' ? 100 : 40, @@ -44,6 +50,24 @@ document: doc, token: "<%= it.drive_file_id %>", type: screen.width < 600 ? 'mobile' : 'desktop', + events: { + onRequestRename: function (event) { + let name = event.data; + const prevExtension = /\.[^.]+$/.exec(doc.title); + if (prevExtension) name += prevExtension[0]; + $. + post(`${window.baseURL}rename${callbackQueryString}`, { name }). + done((changed) => { + if (changed.name != name) + window.docEditor.showMessage(`✅ Renamed to: ${changed.name}`); + }). + fail(() => window.docEditor.showMessage('🚨 Error renaming the file')); + }, + onMetaChange: function (event) { + const { title } = event.data; + if (title) updateTitle(title); + }, + }, editorConfig: { callbackUrl: `${window.baseURL}callback${callbackQueryString}`, lang: window.user.language, From 1a43a75479af0a23655755c97971ede7aa401a40 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Mon, 23 Sep 2024 03:02:56 +0200 Subject: [PATCH 49/52] =?UTF-8?q?=F0=9F=92=84=20frontend:=20remove=20exit?= =?UTF-8?q?=20from=20trash=20context=20menu=20entry=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit didn't work, and no real point --- tdrive/frontend/public/locales/en.json | 1 - tdrive/frontend/public/locales/fr.json | 1 - tdrive/frontend/public/locales/ru.json | 1 - tdrive/frontend/public/locales/vn.json | 1 - .../src/app/views/client/body/drive/context-menu.tsx | 6 ------ 5 files changed, 10 deletions(-) diff --git a/tdrive/frontend/public/locales/en.json b/tdrive/frontend/public/locales/en.json index ed30b1707..640ea389f 100644 --- a/tdrive/frontend/public/locales/en.json +++ b/tdrive/frontend/public/locales/en.json @@ -338,7 +338,6 @@ "components.item_context_menu.clear_selection": "Clear selection", "components.item_context_menu.delete_multiple": "Delete", "components.item_context_menu.to_trash_multiple": "Move selected items to trash", - "components.item_context_menu.trash.exit": "Exit trash", "components.item_context_menu.trash.empty": "Empty trash", "components.item_context_menu.add_documents": "Add document or folder", "components.item_context_menu.download_folder": "Download folder", diff --git a/tdrive/frontend/public/locales/fr.json b/tdrive/frontend/public/locales/fr.json index ce1d214c2..9eff15cc6 100644 --- a/tdrive/frontend/public/locales/fr.json +++ b/tdrive/frontend/public/locales/fr.json @@ -329,7 +329,6 @@ "components.item_context_menu.clear_selection": "Annuler la sélection", "components.item_context_menu.delete_multiple": "Supprimer", "components.item_context_menu.to_trash_multiple": "Supprimer", - "components.item_context_menu.trash.exit": "Quitter la corbeille", "components.item_context_menu.trash.empty": "Vider la corbeille", "components.item_context_menu.add_documents": "Ajouter un document ou un dossier", "components.item_context_menu.download_folder": "Télécharger le dossier", diff --git a/tdrive/frontend/public/locales/ru.json b/tdrive/frontend/public/locales/ru.json index 2b4392ca4..881afb883 100644 --- a/tdrive/frontend/public/locales/ru.json +++ b/tdrive/frontend/public/locales/ru.json @@ -328,7 +328,6 @@ "components.item_context_menu.clear_selection": "Снять выделение", "components.item_context_menu.delete_multiple": "Удалить", "components.item_context_menu.to_trash_multiple": "Удалить", - "components.item_context_menu.trash.exit": "Выйти из корзины", "components.item_context_menu.trash.empty": "Очистить корзину", "components.item_context_menu.add_documents": "Добавить документ в папку", "components.item_context_menu.download_folder": "Скачать папку", diff --git a/tdrive/frontend/public/locales/vn.json b/tdrive/frontend/public/locales/vn.json index 39d92f221..71528743e 100644 --- a/tdrive/frontend/public/locales/vn.json +++ b/tdrive/frontend/public/locales/vn.json @@ -326,7 +326,6 @@ "components.item_context_menu.clear_selection": "Xóa lựa chọn", "components.item_context_menu.delete_multiple": "Xóa", "components.item_context_menu.to_trash_multiple": "Di chuyển các mục đã chọn vào thùng rác", - "components.item_context_menu.trash.exit": "Thoát thùng rác", "components.item_context_menu.trash.empty": "Làm trống thùng rác", "components.item_context_menu.add_documents": "Thêm tài liệu hoặc thư mục", "components.item_context_menu.download_folder": "Tải xuống thư mục", diff --git a/tdrive/frontend/src/app/views/client/body/drive/context-menu.tsx b/tdrive/frontend/src/app/views/client/body/drive/context-menu.tsx index 5b265e3bf..c7c6fc942 100644 --- a/tdrive/frontend/src/app/views/client/body/drive/context-menu.tsx +++ b/tdrive/frontend/src/app/views/client/body/drive/context-menu.tsx @@ -268,12 +268,6 @@ export const useOnBuildContextMenu = (children: DriveItem[], initialParentId?: s //Add parent related menus const newMenuActions: any[] = inTrash ? [ - { - type: 'menu', - text: Languages.t('components.item_context_menu.trash.exit'), - onClick: () => setParentId('root'), - }, - { type: 'separator' }, { type: 'menu', text: Languages.t('components.item_context_menu.trash.empty'), From d86165d6807158bf13e5a5b166ce93cbce1a59a6 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Wed, 25 Sep 2024 00:08:57 +0200 Subject: [PATCH 50/52] =?UTF-8?q?=E2=9C=A8=20cli:=20add=20`editing=5Fsessi?= =?UTF-8?q?on=20parse`=20to=20output=20key=20information=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cli/cmds/editing_session_cmds/parse.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tdrive/backend/node/src/cli/cmds/editing_session_cmds/parse.ts diff --git a/tdrive/backend/node/src/cli/cmds/editing_session_cmds/parse.ts b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/parse.ts new file mode 100644 index 000000000..5329a13a3 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/editing_session_cmds/parse.ts @@ -0,0 +1,34 @@ +import yargs from "yargs"; + +import { NonPlatformCommandYargsBuilder } from "../../utils/non-plaform-command-yargs-builder"; +import { EditingSessionKeyFormat } from "../../../services/documents/entities/drive-file"; + +interface ParseArguments { + editing_session_key: string; +} + +const command: yargs.CommandModule = { + command: "parse ", + describe: ` + Parse the provided editing_session_key and output json data (to stderr) + `.trim(), + + builder: { + ...NonPlatformCommandYargsBuilder, + }, + handler: async argv => { + const args = argv as unknown as ParseArguments; + const parsed = EditingSessionKeyFormat.parse(decodeURIComponent("" + args.editing_session_key)); + console.error( + JSON.stringify( + { + ageH: (new Date().getTime() - parsed.timestamp.getTime()) / (60 * 60 * 1000), + ...parsed, + }, + null, + 2, + ), + ); + }, +}; +export default command; From e7e4db6dc936d9ed21dbc33f2968c7cf38301eb5 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Thu, 26 Sep 2024 12:41:19 +0200 Subject: [PATCH 51/52] =?UTF-8?q?=F0=9F=93=9D=20documentation:=20describin?= =?UTF-8?q?g=20editing=5Fsession=5Fkey=20in=20more=20detail=20(#525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Documentation/docs/plugins.md | 85 ++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/Documentation/docs/plugins.md b/Documentation/docs/plugins.md index 581acaabe..78754b12a 100644 --- a/Documentation/docs/plugins.md +++ b/Documentation/docs/plugins.md @@ -145,6 +145,15 @@ ended with and without a new version. See operations `beginEditing` and `updateEditing` in `tdrive/backend/node/src/services/documents/services/index.ts` for operations available on that key. +Basic flow is: + - Call `beginEditing` + - If a key previously was set for that file, uses the `/check` endpoint of the plugin to update the key status first + - Twake Drive generates an editing session key and returns it (or returns the existing one). + - Optionally call `updateEditing?keepEditing=true` with a file stream to generate intermediary versions + - End the session by calling `updateEditing` (without `keepEditing=true`). If a file is provided, a new (and final) FileVersion + is created with that content. Otherwise, the editing session is cleared without creating a new version. + + #### API to expose by the application Authentication from the Twake Drive backend is a JWT with the property `type` being @@ -161,7 +170,81 @@ multiple parallel requests for the same key gracefully. - `{ status: 'unknown' }`: the key isn't known and maybe used for a new session - `{ status: 'updated' }`: the key needed updating but is now invalid - `{ status: 'expired' }`: the key was already used in a finished session and can't be used again - - `{ status: 'live' }`: the key is valid and current and should be used again for the same file + - `{ status: 'live' }`: the key is valid and current and should be used again for the same file (if multiple writers are allowed) + +#### Example flow of editing session + +```mermaid +sequenceDiagram + autonumber + actor User as User/Browser + participant Front as Frontend + box Server side + participant Back as Backend + participant Conn as onlyoffice
connector + participant OO as Only Office
Edition Server + end + + User->>Front: Open in
editor + Front->>Back: Get plugin config + Front->>User: Open plugin config
editor.edition_url + + User->>Conn: Open editor_url
(proxied by backend) + Conn->>Back: beginEditing + alt existing key + Back->>Conn: checkSessionStatus + Conn->>OO: getForgotten + note over Conn, OO: recover forgotten
process, and new key + Conn->>OO: info command + note right of Conn: decide status of key
live or stale + note over Conn, OO: detect ended but
not changed keys + note over Conn, OO: normal callback processing with
update if required + OO->>Conn: callback with key status + Conn->>Back: key status
(live/expired/updated/etc) + end + activate Back + Back->>Conn: editing_session_key + Conn->>User: HTML host for Editor with
special callback URL + User->>OO: Load JS Editor directly from OO server + activate User + loop User editing + User->>User: Furious Document
Editing + User-->>OO: Periodic saving + end + deactivate User + note left of User: Closes editor + note over User,OO: 10 seconds after last user closes their editor + OO->>Conn: callback to save the new version
or close without changes + Conn->>Back: updateEditing?keepEditing=false
with URL to new version from OO + deactivate Back +``` + +#### Batch processing of unknown keys + +Periodically, the plugin, and twake drive, should run batch cleanup operations on editing session keys +to ensure they are live, or removed, as they may block other operations until then. + +Here is an example initiated by the plugin: + +```mermaid +sequenceDiagram + autonumber + actor User as User/Browser + participant Front as Frontend + box Server side + participant Back as Backend + participant Conn as onlyoffice
connector + participant OO as Only Office
Edition Server + end + + alt Periodic scheduled task + Conn->>OO: getForgottenList + loop Each forgotten file + Conn->>Back: Save new version
end editing session + Conn->>OO: Delete forgotten file + end + end +``` ### Example: OnlyOffice plugin From 630d2667d30cf86f9496f47722c474a465be40a4 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Fri, 27 Sep 2024 12:18:27 +0200 Subject: [PATCH 52/52] =?UTF-8?q?=F0=9F=92=84=20front:=20trim=20filename?= =?UTF-8?q?=20before=20rename=20operation=20(fixes=20#660)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/views/client/body/drive/modals/properties/index.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx b/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx index 3272ef9f6..189d609aa 100644 --- a/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx +++ b/tdrive/frontend/src/app/views/client/body/drive/modals/properties/index.tsx @@ -62,7 +62,7 @@ const PropertiesModalContent = ({ id, onClose }: { id: string; onClose: () => vo const doSave = async () => { setLoading(true); if (item) { - let finalName = name; + let finalName = (name || '').trim(); //TODO: Confirm rename if extension changed ? if (!item?.is_directory) { //TODO: Why do we trim extensions on folders ? @@ -104,7 +104,7 @@ const PropertiesModalContent = ({ id, onClose }: { id: string; onClose: () => vo />