From b7ed3b45ed18d28b99a3d162b8794467a671b0a8 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Mon, 2 Sep 2024 19:32:56 -0400 Subject: [PATCH] Add ability to directly evaluate static network filtering engine Related issue: https://github.com/uBlockOrigin/uBlock-issues/issues/3362 There used to be a way to test URL against the network filtering engine, but this was removed in a distant past during refactoring. The ability has been brought back through uBO's own developer tools, accessible through the _More_ button in the _Support_ pane in the dashboard. To query the static network filtering engine, enter the following in the text editor: snfe?url-to-test [type] [url-of-context] `snfe?` is a prompt indicating the intent to query the static network filtering engine. At a minimum there must be a URL to test. Optionally the type of the resource to match, default to `xhr` if none specified. Also optionally, the context from within which the request is made. Example: Enter: snfe?https://www.google-analytics.com/analytics.js Result: url: https://www.google-analytics.com/analytics.js blocked: ||google-analytics.com^ Enter: snfe?https://www.google-analytics.com/analytics.js script Result: url: https://www.google-analytics.com/analytics.js type: script blocked: ||google-analytics.com^ modified: ||google-analytics.com/analytics.js$script,redirect-rule=google-analytics_analytics.js:5 Enter: snfe?https://example.com/ Result: url: https://example.com/ not blocked Enter: snfe?https://example.com/ ping Result: url: https://example.com/ type: ping blocked: *$ping,3p --- .eslintrc.yml | 1 + src/js/benchmarks.js | 50 +++++++++---------- src/js/devtools.js | 43 +++++++++++++++- src/js/messaging.js | 90 ++++++++++++++++++---------------- src/js/static-net-filtering.js | 65 +++++++++++++++++++++--- 5 files changed, 171 insertions(+), 78 deletions(-) diff --git a/.eslintrc.yml b/.eslintrc.yml index e9ff21ec0234c..470a195bd90e7 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -12,6 +12,7 @@ rules: - error - 4 - ignoredNodes: + - Program > BlockStatement - Program > IfStatement > BlockStatement - Program > ExpressionStatement > CallExpression > ArrowFunctionExpression > BlockStatement - Program > ExpressionStatement > CallExpression > FunctionExpression > BlockStatement diff --git a/src/js/benchmarks.js b/src/js/benchmarks.js index 9fdc6ec104538..93d099bd448dc 100644 --- a/src/js/benchmarks.js +++ b/src/js/benchmarks.js @@ -19,26 +19,22 @@ Home: https://github.com/gorhill/uBlock */ -'use strict'; - -/******************************************************************************/ - -import cosmeticFilteringEngine from './cosmetic-filtering.js'; -import io from './assets.js'; -import scriptletFilteringEngine from './scriptlet-filtering.js'; -import staticNetFilteringEngine from './static-net-filtering.js'; -import µb from './background.js'; -import webRequest from './traffic.js'; -import { FilteringContext } from './filtering-context.js'; -import { LineIterator } from './text-utils.js'; -import { sessionFirewall } from './filtering-engines.js'; - import { domainFromHostname, entityFromDomain, hostnameFromURI, } from './uri-utils.js'; +import { FilteringContext } from './filtering-context.js'; +import { LineIterator } from './text-utils.js'; +import cosmeticFilteringEngine from './cosmetic-filtering.js'; +import io from './assets.js'; +import scriptletFilteringEngine from './scriptlet-filtering.js'; +import { sessionFirewall } from './filtering-engines.js'; +import { default as sfne } from './static-net-filtering.js'; +import webRequest from './traffic.js'; +import µb from './background.js'; + /******************************************************************************/ // The requests.json.gz file can be downloaded from: @@ -155,13 +151,13 @@ export async function benchmarkStaticNetFiltering(options = {}) { fctxt.setURL(request.url); fctxt.setDocOriginFromURL(request.frameUrl); fctxt.setType(request.cpt); - const r = staticNetFilteringEngine.matchRequest(fctxt); + const r = sfne.matchRequest(fctxt); console.info(`Result=${r}:`); console.info(`\ttype=${fctxt.type}`); console.info(`\turl=${fctxt.url}`); console.info(`\tdocOrigin=${fctxt.getDocOrigin()}`); if ( r !== 0 ) { - console.info(staticNetFilteringEngine.toLogData()); + console.info(sfne.toLogData()); } return; } @@ -180,34 +176,34 @@ export async function benchmarkStaticNetFiltering(options = {}) { fctxt.setURL(request.url); fctxt.setDocOriginFromURL(request.frameUrl); fctxt.setType(request.cpt); - staticNetFilteringEngine.redirectURL = undefined; - const r = staticNetFilteringEngine.matchRequest(fctxt); + sfne.redirectURL = undefined; + const r = sfne.matchRequest(fctxt); matchCount += 1; if ( r === 1 ) { blockCount += 1; } else if ( r === 2 ) { allowCount += 1; } if ( r !== 1 ) { - if ( staticNetFilteringEngine.transformRequest(fctxt) ) { + if ( sfne.transformRequest(fctxt) ) { redirectCount += 1; } - if ( fctxt.redirectURL !== undefined && staticNetFilteringEngine.hasQuery(fctxt) ) { - if ( staticNetFilteringEngine.filterQuery(fctxt, 'removeparam') ) { + if ( fctxt.redirectURL !== undefined && sfne.hasQuery(fctxt) ) { + if ( sfne.filterQuery(fctxt, 'removeparam') ) { removeparamCount += 1; } } if ( fctxt.type === 'main_frame' || fctxt.type === 'sub_frame' ) { - if ( staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'csp') ) { + if ( sfne.matchAndFetchModifiers(fctxt, 'csp') ) { cspCount += 1; } - if ( staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'permissions') ) { + if ( sfne.matchAndFetchModifiers(fctxt, 'permissions') ) { permissionsCount += 1; } } - staticNetFilteringEngine.matchHeaders(fctxt, []); - if ( staticNetFilteringEngine.matchAndFetchModifiers(fctxt, 'replace') ) { + sfne.matchHeaders(fctxt, []); + if ( sfne.matchAndFetchModifiers(fctxt, 'replace') ) { replaceCount += 1; } } else if ( redirectEngine !== undefined ) { - if ( staticNetFilteringEngine.redirectRequest(redirectEngine, fctxt) ) { + if ( sfne.redirectRequest(redirectEngine, fctxt) ) { redirectCount += 1; } } @@ -254,7 +250,7 @@ export async function tokenHistogramsfunction() { fctxt.setURL(request.url); fctxt.setDocOriginFromURL(request.frameUrl); fctxt.setType(request.cpt); - const r = staticNetFilteringEngine.matchRequest(fctxt); + const r = sfne.matchRequest(fctxt); for ( let [ keyword ] of request.url.toLowerCase().matchAll(reTokens) ) { const token = keyword.slice(0, 7); if ( r === 0 ) { diff --git a/src/js/devtools.js b/src/js/devtools.js index 0763b0be83b6c..629cd36b17d10 100644 --- a/src/js/devtools.js +++ b/src/js/devtools.js @@ -21,8 +21,6 @@ /* global CodeMirror, uBlockDashboard */ -'use strict'; - import { dom, qs$ } from './dom.js'; /******************************************************************************/ @@ -212,3 +210,44 @@ vAPI.messaging.send('dashboard', { }); /******************************************************************************/ + +async function snfeQuery(lineNo, query) { + const doc = cmEditor.getDoc(); + const lineHandle = doc.getLineHandle(lineNo) + const result = await vAPI.messaging.send('devTools', { + what: 'snfeQuery', + query + }); + if ( typeof result !== 'string' ) { return; } + cmEditor.startOperation(); + const nextLineNo = doc.getLineNumber(lineHandle) + 1; + doc.replaceRange(`${result}\n`, { line: nextLineNo, ch: 0 }); + cmEditor.endOperation(); +} + +cmEditor.on('beforeChange', (cm, details) => { + if ( details.origin !== '+input' ) { return; } + if ( details.text.length !== 2 ) { return; } + if ( details.text[1] !== '' ) { return; } + const lineNo = details.from.line; + const line = cm.getLine(lineNo); + if ( details.from.ch !== line.length ) { return; } + if ( line.startsWith('snfe?') === false ) { return; } + const fields = line.slice(5).split(/\s+/); + const query = {}; + for ( const field of fields ) { + if ( /[/.]/.test(field) ) { + if ( query.url === undefined ) { + query.url = field; + } else if ( query.from === undefined ) { + query.from = field; + } + } else if ( query.type === undefined ) { + query.type = field; + } + } + if ( query.url === undefined ) { return; } + snfeQuery(lineNo, query); +}); + +/******************************************************************************/ diff --git a/src/js/messaging.js b/src/js/messaging.js index 5f39af4f441dd..ebfc5c7d44fed 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -19,53 +19,50 @@ Home: https://github.com/gorhill/uBlock */ -/* globals browser */ - -'use strict'; +import * as s14e from './s14e-serializer.js'; +import * as sfp from './static-filtering-parser.js'; -/******************************************************************************/ +import { + domainFromHostname, + domainFromURI, + entityFromDomain, + hostnameFromURI, + isNetworkURI, +} from './uri-utils.js'; -import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js'; -import punycode from '../lib/punycode.js'; +import { + permanentFirewall, + permanentSwitches, + permanentURLFiltering, + sessionFirewall, + sessionSwitches, + sessionURLFiltering, +} from './filtering-engines.js'; -import { filteringBehaviorChanged } from './broadcast.js'; import cacheStorage from './cachestorage.js'; import cosmeticFilteringEngine from './cosmetic-filtering.js'; +import { denseBase64 } from './base64-custom.js'; +import { dnrRulesetFromRawLists } from './static-dnr-filtering.js'; +import { filteringBehaviorChanged } from './broadcast.js'; import htmlFilteringEngine from './html-filtering.js'; +import { i18n$ } from './i18n.js'; +import io from './assets.js'; import logger from './logger.js'; import lz4Codec from './lz4.js'; -import io from './assets.js'; +import publicSuffixList from '../lib/publicsuffixlist/publicsuffixlist.js'; +import punycode from '../lib/punycode.js'; +import { redirectEngine } from './redirect-engine.js'; import scriptletFilteringEngine from './scriptlet-filtering.js'; import staticFilteringReverseLookup from './reverselookup.js'; import staticNetFilteringEngine from './static-net-filtering.js'; -import µb from './background.js'; import webRequest from './traffic.js'; -import { denseBase64 } from './base64-custom.js'; -import { dnrRulesetFromRawLists } from './static-dnr-filtering.js'; -import { i18n$ } from './i18n.js'; -import { redirectEngine } from './redirect-engine.js'; -import * as sfp from './static-filtering-parser.js'; -import * as s14e from './s14e-serializer.js'; - -import { - permanentFirewall, - sessionFirewall, - permanentSwitches, - sessionSwitches, - permanentURLFiltering, - sessionURLFiltering, -} from './filtering-engines.js'; - -import { - domainFromHostname, - domainFromURI, - entityFromDomain, - hostnameFromURI, - isNetworkURI, -} from './uri-utils.js'; +import µb from './background.js'; /******************************************************************************/ +const hasOwnProperty = (o, p) => + Object.prototype.hasOwnProperty.call(o, p); + // https://github.com/uBlockOrigin/uBlock-issues/issues/710 // Listeners have a name and a "privileged" status. // The nameless default handler is always deemed "privileged". @@ -807,7 +804,7 @@ const onMessage = function(request, sender, callback) { }); break; - case 'shouldRenderNoscriptTags': + case 'shouldRenderNoscriptTags': { if ( pageStore === null ) { break; } const fctxt = µb.filteringContext.fromTabId(sender.tabId); if ( pageStore.filterScripting(fctxt, undefined) ) { @@ -818,7 +815,7 @@ const onMessage = function(request, sender, callback) { }); } break; - + } case 'retrieveGenericCosmeticSelectors': request.tabId = sender.tabId; request.frameId = sender.frameId; @@ -1098,7 +1095,7 @@ const restoreUserData = async function(request) { // Discard unknown setting or setting with default value. for ( const key in hiddenSettings ) { if ( - µb.hiddenSettingsDefault.hasOwnProperty(key) === false || + hasOwnProperty(µb.hiddenSettingsDefault, key) === false || hiddenSettings[key] === µb.hiddenSettingsDefault[key] ) { delete hiddenSettings[key]; @@ -1128,7 +1125,7 @@ const restoreUserData = async function(request) { }); µb.saveUserFilters(userData.userFilters); if ( Array.isArray(userData.selectedFilterLists) ) { - await µb.saveSelectedFilterLists(userData.selectedFilterLists); + await µb.saveSelectedFilterLists(userData.selectedFilterLists); } vAPI.app.restart(); @@ -1150,7 +1147,7 @@ const resetUserData = async function() { // Filter lists const prepListEntries = function(entries) { for ( const k in entries ) { - if ( entries.hasOwnProperty(k) === false ) { continue; } + if ( hasOwnProperty(entries, k) === false ) { continue; } const entry = entries[k]; if ( typeof entry.supportURL === 'string' && entry.supportURL !== '' ) { entry.supportName = hostnameFromURI(entry.supportURL); @@ -1338,7 +1335,7 @@ const getSupportData = async function() { let addedListset = {}; let removedListset = {}; for ( const listKey in lists ) { - if ( lists.hasOwnProperty(listKey) === false ) { continue; } + if ( hasOwnProperty(lists, listKey) === false ) { continue; } const list = lists[listKey]; if ( list.content !== 'filters' ) { continue; } const used = µb.selectedFilterLists.includes(listKey); @@ -1755,7 +1752,7 @@ const onMessage = (request, sender, callback) => { // Sync let response; switch ( request.what ) { - case 'getInspectorArgs': + case 'getInspectorArgs': { const bc = new globalThis.BroadcastChannel('contentInspectorChannel'); bc.postMessage({ what: 'contentInspectorChannel', @@ -1768,6 +1765,7 @@ const onMessage = (request, sender, callback) => { ), }; break; + } default: return vAPI.messaging.UNHANDLED; } @@ -2004,6 +2002,12 @@ const onMessage = function(request, sender, callback) { response = staticNetFilteringEngine.dump(); break; + case 'snfeQuery': + response = staticNetFilteringEngine.test( + Object.assign({ redirectEngine }, request.query) + ); + break; + case 'cfeDump': response = cosmeticFilteringEngine.dump(); break; @@ -2181,7 +2185,7 @@ const onMessage = function(request, sender, callback) { } break; - case 'subscribeTo': + case 'subscribeTo': { // https://github.com/uBlockOrigin/uBlock-issues/issues/1797 if ( /^(file|https?):\/\//.test(request.location) === false ) { break; } const url = encodeURIComponent(request.location); @@ -2194,8 +2198,8 @@ const onMessage = function(request, sender, callback) { select: true, }); break; - - case 'updateLists': + } + case 'updateLists': { const listkeys = request.listkeys.split(',').filter(s => s !== ''); if ( listkeys.length === 0 ) { return; } if ( listkeys.includes('all') ) { @@ -2211,7 +2215,7 @@ const onMessage = function(request, sender, callback) { }); µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: request.auto }); break; - + } default: return vAPI.messaging.UNHANDLED; } diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index d7edc01fb35d8..f30af31bea003 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -5384,17 +5384,70 @@ StaticNetFilteringEngine.prototype.enableWASM = function(wasmModuleFetcher, path /******************************************************************************/ -StaticNetFilteringEngine.prototype.test = async function(docURL, type, url) { +StaticNetFilteringEngine.prototype.test = function(details) { + const { url, type, from, redirectEngine } = details; + if ( url === undefined ) { return; } const fctxt = new FilteringContext(); - fctxt.setDocOriginFromURL(docURL); - fctxt.setType(type); fctxt.setURL(url); + fctxt.setType(type || ''); + fctxt.setDocOriginFromURL(from || ''); const r = this.matchRequest(fctxt); - console.info(`${r}`); + const out = [ `url: ${url}` ]; + if ( type ) { + out.push(`type: ${type}`); + } + if ( from ) { + out.push(`context: ${from}`); + } if ( r !== 0 ) { - console.info(this.toLogData()); + const logdata = this.toLogData(); + if ( r === 1 ) { + out.push(`blocked: ${logdata.raw}`); + } else if ( r === 2 ) { + out.push(`unblocked: ${logdata.raw}`); + } + } else { + out.push('not blocked'); } -}; + if ( r !== 1 ) { + const entries = this.transformRequest(fctxt); + if ( entries ) { + for ( const entry of entries ) { + out.push(`modified: ${entry.logData().raw}`); + } + } + if ( fctxt.redirectURL !== undefined && this.hasQuery(fctxt) ) { + const entries = this.filterQuery(fctxt, 'removeparam'); + if ( entries ) { + for ( const entry of entries ) { + out.push(`modified: ${entry.logData().raw}`); + } + } + } + if ( fctxt.type === 'main_frame' || fctxt.type === 'sub_frame' ) { + const csps = this.matchAndFetchModifiers(fctxt, 'csp'); + if ( csps ) { + for ( const csp of csps ) { + out.push(`modified: ${csp.logData().raw}`); + } + } + const pps = this.matchAndFetchModifiers(fctxt, 'permissions'); + if ( pps ) { + for ( const pp of pps ) { + out.push(`modified: ${pp.logData().raw}`); + } + } + } + } else if ( redirectEngine ) { + const redirects = this.redirectRequest(redirectEngine, fctxt); + if ( redirects ) { + for ( const redirect of redirects ) { + out.push(`modified: ${redirect.logData().raw}`); + } + } + } + return out.join('\n'); +} /******************************************************************************/