diff --git a/gulpfile.js b/gulpfile.js index 7eb4b6f67..5212b156d 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -65,9 +65,9 @@ var injectInclude = [ 'translate/rdf/identity.js', 'translate/rdf/rdfparser.js', 'translate/translation/translate.js', + 'translate/translation/translate_item.js', 'translate/translator.js', 'translate/utilities_translate.js', - 'translate_item.js', 'inject/http.js', 'inject/sandboxManager.js', 'integration/connectorIntegration.js', @@ -101,7 +101,7 @@ var injectIncludeBrowserExt = ['browser-polyfill.js'].concat( var injectIncludeManifestV3 = ['browser-polyfill.js'].concat( injectInclude, ['api.js'], - ['translateSandbox/translateSandboxFunctionOverrides.js', 'translateSandbox/translateSandboxManager.js'], + ['inject/virtualOffscreenTranslate.js'], injectIncludeLast); var backgroundInclude = [ @@ -149,7 +149,8 @@ var backgroundIncludeBrowserExt = ['browser-polyfill.js'].concat(backgroundInclu 'webRequestIntercept.js', 'contentTypeHandler.js', 'saveWithoutProgressWindow.js', - 'translateSandbox/translateBlocklistManager.js' + 'messagingGeneric.js', + 'offscreen/offscreenFunctionOverrides.js', 'background/offscreenManager.js', ]); function reloadChromeExtensionsTab(cb) { diff --git a/src/browserExt/background.js b/src/browserExt/background.js index 40d8b38c4..7df38fdb2 100644 --- a/src/browserExt/background.js +++ b/src/browserExt/background.js @@ -52,12 +52,13 @@ Zotero.Connector_Browser = new function() { this.init = async function() { if (Zotero.isManifestV3) { - if (!Zotero.isFirefox) { + if (Zotero.isChromium) { // Chrome recently stopped displaying context menus on button right-click // with 'browser_action' as context. It's supposed to work, so maybe a bug // in Chrome, but let's fix it on our side. Firefox, meanwhile, throws if 'action' // is included in the context list. buttonContext.push('action'); + await Zotero.OffscreenManager.init(); } this._tabInfo = _tabInfo = await Zotero.Utilities.Connector.createMV3PersistentObject('tabInfo'); setInterval(async () => { diff --git a/src/browserExt/background/offscreenManager.js b/src/browserExt/background/offscreenManager.js new file mode 100644 index 000000000..769993ecf --- /dev/null +++ b/src/browserExt/background/offscreenManager.js @@ -0,0 +1,139 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2024 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +/** + * Part of background page. Manages the offscreen page + */ +Zotero.OffscreenManager = { + initPromise: null, + offscreenPageInitialized: false, + messagingDeferred: Zotero.Promise.defer(), + offscreenUrl: 'offscreen/offscreen.html', + + async init() { + const offscreenPage = await this.getOffscreenPage(); + if (!offscreenPage) { + // Make sure we're waiting for a new deferred + this.messagingDeferred = Zotero.Promise.defer(); + // Create offscreen document + await browser.offscreen.createDocument({ + url: this.offscreenUrl, + reasons: ['DOM_SCRAPING', 'DOM_PARSER'], + justification: 'Scraping the document with Zotero Translators', + }); + } + else { + // Technically the service worker can restart without the offscreen + // page being unloaded per Chrome docs, although not clear whether this would actually happen in practice. + offscreenPage.postMessage('service-worker-restarted'); + } + await this.messagingDeferred.promise; + + // Only need to set the below up once + if (this.offscreenPageInitialized) return; + this.offscreenPageInitialized = true; + + // Watch for browserext event of tab close and inform the offscreen page translate + browser.tabs.onRemoved.addListener((tabId, removeInfo) => { + this.sendMessage('tabClosed', tabId); + }); + + // Run cleanup every 15min + setInterval(() => this.cleanup(), 15*60e3); + Zotero.debug('OffscreenManager: offscreen page initialized'); + }, + + async sendMessage(message, payload, tab, frameId) { + const offscreenPage = await this.getOffscreenPage(); + if (!offscreenPage) { + await this.init(); + } + if (tab) { + payload.push(tab.id, frameId); + } + return await this._messaging.sendMessage(message, payload); + }, + + async addMessageListener(...args) { + const offscreenPage = await this.getOffscreenPage(); + if (!offscreenPage) { + await this.init(); + } + return this._messaging.addMessageListener(...args); + }, + + /** + * onTabRemoved handler should make sure offscreen doesn't hold translate instances + * that are dead and moreover the offscreen page should get killed every now and then by the browser, + * but we want to be extra sure we're not leaking memory + */ + async cleanup() { + const offscreenPage = await this.getOffscreenPage(); + if (!offscreenPage) return false; + let tabs = await browser.tabs.query({status: "complete", windowType: "normal"}); + let cleanedUpTabIds = await this.sendMessage('translateCleanup', tabs.map(tab => tab.id)); + if (cleanedUpTabIds.length > 0) { + Zotero.logError(new Error(`OffscreenManager: manually cleaned up translates that were kept ` + + `alive after onTabRemoved ${JSON.stringif(cleanedUpTabIds)}`)); + } + }, + + async getOffscreenPage() { + const matchedClients = await self.clients.matchAll(); + return matchedClients.find(client => client.url.includes(this.offscreenUrl)); + + } +} + +// Listener needs to be added at worker script initialization +self.onmessage = async (e) => { + if (e.data === 'offscreen-port') { + Zotero.debug('OffscreenManager: received the offscreen page port') + // Resolve _initMessaging() in offscreenSandbox.js + let messagingOptions = { + handlerFunctionOverrides: OFFSCREEN_BACKGROUND_OVERRIDES, + } + messagingOptions.sendMessage = (...args) => { + e.ports[0].postMessage(args) + }; + messagingOptions.addMessageListener = (fn) => { + e.ports[0].onmessage = (e) => fn(e.data); + }; + // If the offscreen document got killed by the browser and we restarted it + // we only need to set sendMessage, otherwise previously added message listeners + // will get discarded + if (Zotero.OffscreenManager._messaging) { + Zotero.OffscreenManager._messaging.reinit(messagingOptions); + } + else { + Zotero.OffscreenManager._messaging = new Zotero.MessagingGeneric(messagingOptions); + } + Zotero.debug('OffscreenManager: messaging initialized') + e.ports[0].postMessage(null); + await new Promise(resolve => Zotero.OffscreenManager._messaging.addMessageListener('offscreen-sandbox-initialized', resolve)); + Zotero.debug('OffscreenManager: offscreen sandbox initialized message received') + Zotero.OffscreenManager.messagingDeferred.resolve(); + } +} \ No newline at end of file diff --git a/src/browserExt/inject/virtualOffscreenTranslate.js b/src/browserExt/inject/virtualOffscreenTranslate.js new file mode 100644 index 000000000..245cbb2bd --- /dev/null +++ b/src/browserExt/inject/virtualOffscreenTranslate.js @@ -0,0 +1,123 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2024 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +// A virtual translate that offloads translating to the offscreen page +Zotero.VirtualOffscreenTranslate = class { + translateDoc = null; + + /** + * @returns {Promise} + */ + static async create() { + let translate = new Zotero.VirtualOffscreenTranslate(); + await translate.sendMessage('Translate.new'); + return new Proxy(translate, { + get: (target, property, ...args) => { + if (!target[property] && (property in Zotero.Translate.Web.prototype)) { + return (...args) => { + return target.sendMessage(`Translate.${property}`, args); + } + } + return Reflect.get(target, property, ...args); + } + }); + } + + constructor() { + // Handling for translate.monitorDOMChanges + let mutationObserver; + this.addMessageListener('MutationObserver.observe', ([selector, config]) => { + // We allow at most one observer, or we'll have to keep track of them. Websites + // that need this will only have one translator applying an observer anyway. + if (mutationObserver) mutationObserver.disconnect(); + mutationObserver = new MutationObserver(() => { + // We disconnect immediately because that's what monitorDOMChanges does, and if we don't + // there's an async messaging timeblock where more mutations may occur and result in + // pageModified being called multiple times. + mutationObserver.disconnect(); + return this.sendMessage('MutationObserver.trigger'); + }); + const node = this.translateDoc.querySelector(selector); + mutationObserver.observe(node, config); + }); + } + + getProxy() { + return this.sendMessage('Translate.getProxy'); + } + + async setHandler(name, callback) { + let id = Zotero.Utilities.randomString(10); + await this.sendMessage('Translate.setHandler', [name, id]); + this.addMessageListener(`Translate.onHandler.${name}`, ([remoteId, args]) => { + if (name == 'select') { + args[2] = (...args) => { + this.sendMessage('Translate.selectCallback', [id, args]); + } + } + if (remoteId == id) { + callback(...args); + } + }); + } + + setDocument(doc, updateLiveElements=false) { + this.translateDoc = doc; + if (updateLiveElements) { + for (const checkbox of doc.querySelectorAll('input[type=checkbox]')) { + if (checkbox.checked) { + checkbox.setAttribute('checked', ''); + } + else { + checkbox.removeAttribute('checked'); + } + } + } + return this.sendMessage('Translate.setDocument', [doc.documentElement.outerHTML, doc.location.href, doc.cookie]); + } + + async setTranslator(translators) { + if (!Array.isArray(translators)) { + translators = [translators]; + } + translators = translators.map(t => t.serialize(Zotero.Translator.TRANSLATOR_PASSING_PROPERTIES)); + return this.sendMessage('Translate.setTranslator', [translators]) + } + + async getTranslators(...args) { + let translators = await this.sendMessage('Translate.getTranslators', args); + return translators.map(translator => new Zotero.Translator(translator)); + } + + sendMessage(message, payload=[]) { + return Zotero.OffscreenManager.sendMessage(message, payload) + } + + addMessageListener(...args) { + // Listening for messages from bg page messaging via which OffscreenManager will send messages + // since it doesn't have the ability to send messages directly to tabs itself + return Zotero.Messaging.addMessageListener(...args) + } +} diff --git a/src/browserExt/manifest-v3.json b/src/browserExt/manifest-v3.json index 374511805..a5c906dfb 100644 --- a/src/browserExt/manifest-v3.json +++ b/src/browserExt/manifest-v3.json @@ -13,7 +13,7 @@ "default_title": "Save to Zotero" }, "host_permissions": ["http://*/*", "https://*/*"], - "permissions": ["tabs", "contextMenus", "cookies", "scripting", + "permissions": ["tabs", "contextMenus", "cookies", "scripting", "offscreen", "webRequest", "declarativeNetRequest", "webNavigation", "storage"], "declarative_net_request": { "rule_resources": [{ @@ -51,16 +51,18 @@ } ], "sandbox": { - "pages": ["translateSandbox/translateSandbox.html"] + "pages": ["offscreen/offscreenSandbox.html"] }, "web_accessible_resources": [{ "resources": [ "images/*", "progressWindow/progressWindow.html", "modalPrompt/modalPrompt.html", - "translateSandbox/translateSandbox.html", "test/data/journalArticle-single.html", - "lib/SingleFile/single-file-hooks-frames.js" + "lib/SingleFile/single-file-hooks-frames.js", + "inject/pageSaving.js", + "translateWeb.js", + "itemSaver.js" ], "matches": ["http://*/*", "https://*/*"] }], diff --git a/src/browserExt/manifest.json b/src/browserExt/manifest.json index 5d57e57c4..a843071a3 100644 --- a/src/browserExt/manifest.json +++ b/src/browserExt/manifest.json @@ -50,7 +50,10 @@ "progressWindow/progressWindow.html", "modalPrompt/modalPrompt.html", "test/data/journalArticle-single.html", - "lib/SingleFile/single-file-hooks-frames.js" + "lib/SingleFile/single-file-hooks-frames.js", + "inject/pageSaving.js", + "translateWeb.js", + "itemSaver.js" ], "content_security_policy": "script-src 'self' 'unsafe-eval'; object-src 'self'", "homepage_url": "https://www.zotero.org/", diff --git a/src/browserExt/messagingGeneric.js b/src/browserExt/messagingGeneric.js index e2d4bba7e..3c618f87f 100644 --- a/src/browserExt/messagingGeneric.js +++ b/src/browserExt/messagingGeneric.js @@ -129,6 +129,17 @@ Zotero.MessagingGeneric = class { this._initMessageListener(); } + // Reinit messaging without resetting existing message listeners. Needed if the existing connection + // gets severed for some reason. + reinit(options) { + if (!options.sendMessage || !options.addMessageListener) { + throw new Error('Zotero.MessagingGeneric: mandatory reinit() options missing'); + } + this._sendMessage = options.sendMessage; + this._addMessageListener = options.addMessageListener; + this._initMessageListener(); + } + // Initialize message handler _initMessageListener() { this._addMessageListener(async (args) => { @@ -149,7 +160,7 @@ Zotero.MessagingGeneric = class { if (this._options.supportsResponse) { return result; } - else if (result !== undefined) { + else { this._sendMessage(`response`, result, messageId); } } diff --git a/src/browserExt/offscreen/offscreen.html b/src/browserExt/offscreen/offscreen.html new file mode 100644 index 000000000..811779d9b --- /dev/null +++ b/src/browserExt/offscreen/offscreen.html @@ -0,0 +1,35 @@ + + + + + + Zotero - Offscreen Utility Page + + + + + + \ No newline at end of file diff --git a/src/browserExt/offscreen/offscreen.js b/src/browserExt/offscreen/offscreen.js new file mode 100644 index 000000000..81045f4a6 --- /dev/null +++ b/src/browserExt/offscreen/offscreen.js @@ -0,0 +1,71 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2024 Corporation for Digital Scholarship + Vienna, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +/* + * Entrypoint for offscreen page. Evals are disallowed here and we run them in a sandbox iframe instead. + * + * This script orchestrates establishing a message channel for message passing between the background + * page and the offscreen translate sandbox page. Also handles possible situations where the background + * service worker gets killed, but the offscreen page stays alive. + * + * Content scripts then communicate with translate sandbox + * by message passing via background page. + */ + +let offscreenSandboxReadyPromise = new Promise((resolve) => { + self.onmessage = async (e) => { + if (e.data === 'offscreen-sandbox-ready') { + self.onmessage = null; + resolve(); + } + } +}); + +async function init() { + console.log('Offscreen: awaiting offscreen sandbox to be ready') + await offscreenSandboxReadyPromise; + + let messageChannel = new MessageChannel(); + const iframe = document.querySelector('iframe'); + iframe.contentWindow.postMessage('offscreen-port', "*", [messageChannel.port1]); + + console.log('Offscreen: awaiting offscreen sandbox to prepare for service worker connection') + await new Promise((resolve) => { + messageChannel.port2.onmessage = resolve; + }); + messageChannel.port2.onmessage = null; + + const backgroundServiceWorker = await navigator.serviceWorker.ready; + backgroundServiceWorker.active.postMessage('offscreen-port', [messageChannel.port2]); + console.log('Offscreen: messaging ports posted'); +} + +document.addEventListener('DOMContentLoaded', () => init()); + +navigator.serviceWorker.onmessage = async (e) => { + if (e.data !== 'service-worker-restarted') return; + console.log('Offscreen: owner service worker restarted. reinitializing messaging'); + await init(); +}; \ No newline at end of file diff --git a/src/browserExt/translateSandbox/translateSandboxFunctionOverrides.js b/src/browserExt/offscreen/offscreenFunctionOverrides.js similarity index 88% rename from src/browserExt/translateSandbox/translateSandboxFunctionOverrides.js rename to src/browserExt/offscreen/offscreenFunctionOverrides.js index 07ee45ee9..4cfbb7e58 100644 --- a/src/browserExt/translateSandbox/translateSandboxFunctionOverrides.js +++ b/src/browserExt/offscreen/offscreenFunctionOverrides.js @@ -1,8 +1,8 @@ /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2021 Corporation for Digital Scholarship - Vienna, Virginia, USA + Copyright © 2024 Corporation for Digital Scholarship + Vienna, Virginia, USA http://zotero.org This file is part of Zotero. @@ -23,6 +23,7 @@ ***** END LICENSE BLOCK ***** */ + function serializeTranslator(translator, properties) { let serializedTranslator = {}; for (let key of properties) { @@ -77,7 +78,7 @@ const requestOverride = { } } -const CONTENT_SCRIPT_FUNCTION_OVERRIDES = { +const OFFSCREEN_BACKGROUND_OVERRIDES = { 'Translators.get': { handler: { preSend: async function(translator) { @@ -125,26 +126,17 @@ const CONTENT_SCRIPT_FUNCTION_OVERRIDES = { } }, }, + 'getVersion': true, + 'getExtensionURL': true, 'Debug.log': true, 'debug': true, - 'getExtensionURL': true, - 'getExtensionVersion': true, 'Errors.log': true, 'Messaging.sendMessage': true, - 'Connector.checkIsOnline': true, - 'Connector.callMethod': true, - 'Connector.callMethodWithCookies': true, - 'Connector.saveSingleFile': true, + // Translator error reporting 'Connector_Browser.isIncognito': true, 'Prefs.getAll': true, 'Prefs.getAsync': true, - 'API.authorize': true, - 'API.onAuthorizationComplete': false, - 'API.clearCredentials': false, - 'API.getUserInfo': true, - 'API.run': true, - 'API.uploadAttachment': true, - 'SingleFile.retrievePageData': true, + // Translator HTTP requests 'COHTTP.request': requestOverride, 'HTTP.request': requestOverride, }; diff --git a/src/browserExt/translateSandbox/translateSandbox.html b/src/browserExt/offscreen/offscreenSandbox.html similarity index 83% rename from src/browserExt/translateSandbox/translateSandbox.html rename to src/browserExt/offscreen/offscreenSandbox.html index 391bb46c6..99133cd72 100644 --- a/src/browserExt/translateSandbox/translateSandbox.html +++ b/src/browserExt/offscreen/offscreenSandbox.html @@ -2,9 +2,9 @@