diff --git a/clearurls.js b/clearurls.js index 66a4aeeb..8cd31336 100644 --- a/clearurls.js +++ b/clearurls.js @@ -106,6 +106,12 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { urlObject = new URL(url); fields = urlObject.searchParams; + // Capture the raw query string once, with the leading '?' stripped, so that + // fields whose values use a non-UTF-8 encoding (e.g. GBK on 1688.com) are + // filtered byte-for-byte instead of being round-tripped through UTF-8. + let rawFieldsQuery = urlObject.search.startsWith('?') + ? urlObject.search.slice(1) + : urlObject.search; fragments = extractFragments(urlObject); domain = urlWithoutParamsAndHash(urlObject).toString(); @@ -114,16 +120,17 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { */ if (fields.toString() !== "" || fragments.toString() !== "") { rules.forEach(rule => { - const beforeFields = fields.toString(); + const beforeFields = rawFieldsQuery; const beforeFragments = fragments.toString(); let localChange = false; - for (const field of fields.keys()) { - if (new RegExp("^"+rule+"$", "gi").test(field)) { - fields.delete(field); - changes = true; - localChange = true; - } + // Filter fields byte-for-byte so that non-UTF-8 values (e.g. GBK) + // are preserved. A change is detected by comparing the raw strings. + const afterFields = removeFieldsFromQuery(rawFieldsQuery, [rule]); + if (afterFields !== rawFieldsQuery) { + rawFieldsQuery = afterFields; + changes = true; + localChange = true; } for (const fragment of fragments.keys()) { @@ -139,9 +146,9 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { let tempURL = domain; let tempBeforeURL = domain; - if (fields.toString() !== "") tempURL += "?" + fields.toString(); + if (rawFieldsQuery !== "") tempURL += "?" + rawFieldsQuery; if (fragments.toString() !== "") tempURL += "#" + fragments.toString(); - if (beforeFields.toString() !== "") tempBeforeURL += "?" + beforeFields.toString(); + if (beforeFields !== "") tempBeforeURL += "?" + beforeFields; if (beforeFragments.toString() !== "") tempBeforeURL += "#" + beforeFragments.toString(); if (!quiet) pushToLog(tempBeforeURL, tempURL, rule); @@ -152,10 +159,10 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { let finalURL = domain; - if (fields.toString() !== "") finalURL += "?" + urlSearchParamsToString(fields); + if (rawFieldsQuery !== "") finalURL += "?" + rawFieldsQuery; if (fragments.toString() !== "") finalURL += "#" + fragments.toString(); - url = finalURL.replace(new RegExp("\\?&"), "?").replace(new RegExp("#&"), "#"); + url = finalURL.replace(/\?&/, "?").replace(/#&/, "#"); } @@ -166,524 +173,539 @@ function removeFieldsFormURL(provider, pureUrl, quiet = false, request = null) { } } -function start() { +/** + * Initialize the JSON provider object keys. + * + * @param {object} obj + */ +function getKeys(obj) { + for (const key in obj) { + prvKeys.push(key); + } +} + +/** + * Deactivates ClearURLs, if no rules can be downloaded and also no old rules in storage + */ +function deactivateOnFailure() { + if (storage.ClearURLsData.length === 0) { + storage.globalStatus = false; + storage.dataHash = ""; + changeIcon(); + storeHashStatus(5); + saveOnExit(); + } +} + +/** + * Declare constructor + * + * @param {String} _name Provider name + * @param {boolean} _completeProvider Set URL Pattern as rule + * @param {boolean} _forceRedirection Whether redirects should be enforced via a "tabs.update" + * @param {boolean} _isActive Is the provider active? + */ +function Provider(_name, _completeProvider = false, _forceRedirection = false, _isActive = true) { + let name = _name; + let urlPattern; + let enabled_rules = {}; + let disabled_rules = {}; + let enabled_exceptions = {}; + let disabled_exceptions = {}; + let canceling = _completeProvider; + let enabled_redirections = {}; + let disabled_redirections = {}; + let active = _isActive; + let enabled_rawRules = {}; + let disabled_rawRules = {}; + let enabled_referralMarketing = {}; + let disabled_referralMarketing = {}; + let methods = []; + + if (_completeProvider) { + enabled_rules[".*"] = true; + } + /** - * Initialize the JSON provider object keys. - * - * @param {object} obj + * Returns whether redirects should be enforced via a "tabs.update" + * @return {boolean} whether redirects should be enforced */ - function getKeys(obj) { - for (const key in obj) { - prvKeys.push(key); - } - } + this.shouldForceRedirect = function () { + return _forceRedirection; + }; /** - * Initialize the providers form the JSON object. - * + * Returns the provider name. + * @return {String} */ - function createProviders() { - let data = storage.ClearURLsData; + this.getName = function () { + return name; + }; - for (let p = 0; p < prvKeys.length; p++) { - //Create new provider - providers.push(new Provider(prvKeys[p], data.providers[prvKeys[p]].getOrDefault('completeProvider', false), - data.providers[prvKeys[p]].getOrDefault('forceRedirection', false))); + /** + * Add URL pattern. + * + * @require urlPatterns as RegExp + */ + this.setURLPattern = function (urlPatterns) { + urlPattern = new RegExp(urlPatterns, "i"); + }; - //Add URL Pattern - providers[p].setURLPattern(data.providers[prvKeys[p]].getOrDefault('urlPattern', '')); + /** + * Return if the Provider Request is canceled + * @return {Boolean} isCanceled + */ + this.isCaneling = function () { + return canceling; + }; - let rules = data.providers[prvKeys[p]].getOrDefault('rules', []); - //Add rules to provider - for (let r = 0; r < rules.length; r++) { - providers[p].addRule(rules[r]); - } + /** + * Check the url is matching the ProviderURL. + * + * @return {boolean} ProviderURL as RegExp + */ + this.matchURL = function (url) { + return urlPattern.test(url) && !(this.matchException(url)); + }; - let rawRules = data.providers[prvKeys[p]].getOrDefault('rawRules', []); - //Add raw rules to provider - for (let raw = 0; raw < rawRules.length; raw++) { - providers[p].addRawRule(rawRules[raw]); - } + /** + * Apply a rule to a given tuple of rule array. + * @param enabledRuleArray array for enabled rules + * @param disabledRulesArray array for disabled rules + * @param {String} rule RegExp as string + * @param {boolean} isActive Is this rule active? + */ + this.applyRule = (enabledRuleArray, disabledRulesArray, rule, isActive = true) => { + if (isActive) { + enabledRuleArray[rule] = true; - let referralMarketingRules = data.providers[prvKeys[p]].getOrDefault('referralMarketing', []); - //Add referral marketing rules to provider - for (let referralMarketing = 0; referralMarketing < referralMarketingRules.length; referralMarketing++) { - providers[p].addReferralMarketing(referralMarketingRules[referralMarketing]); + if (disabledRulesArray[rule] !== undefined) { + delete disabledRulesArray[rule]; } + } else { + disabledRulesArray[rule] = true; - let exceptions = data.providers[prvKeys[p]].getOrDefault('exceptions', []); - //Add exceptions to provider - for (let e = 0; e < exceptions.length; e++) { - providers[p].addException(exceptions[e]); + if (enabledRuleArray[rule] !== undefined) { + delete enabledRuleArray[rule]; } + } + }; - let redirections = data.providers[prvKeys[p]].getOrDefault('redirections', []); - //Add redirections to provider - for (let re = 0; re < redirections.length; re++) { - providers[p].addRedirection(redirections[re]); - } + /** + * Add a rule to the rule array + * and replace old rule with new rule. + * + * @param {String} rule RegExp as string + * @param {boolean} isActive Is this rule active? + */ + this.addRule = function (rule, isActive = true) { + this.applyRule(enabled_rules, disabled_rules, rule, isActive); + }; - let methods = data.providers[prvKeys[p]].getOrDefault('methods', []); - //Add HTTP methods list to provider - for (let re = 0; re < methods.length; re++) { - providers[p].addMethod(methods[re]); - } + /** + * Return all active rules as an array. + * + * @return Array RegExp strings + */ + this.getRules = function () { + if (!storage.referralMarketing) { + return Object.keys(Object.assign(enabled_rules, enabled_referralMarketing)); } - } + + return Object.keys(enabled_rules); + }; /** - * Convert the external data to Objects and - * call the create provider function. + * Add a raw rule to the raw rule array + * and replace old raw rule with new raw rule. * - * @param {String} retrievedText - pure data form github + * @param {String} rule RegExp as string + * @param {boolean} isActive Is this rule active? */ - function toObject(retrievedText) { - getKeys(storage.ClearURLsData.providers); - createProviders(); - } + this.addRawRule = function (rule, isActive = true) { + this.applyRule(enabled_rawRules, disabled_rawRules, rule, isActive); + }; /** - * Deactivates ClearURLs, if no rules can be downloaded and also no old rules in storage + * Return all active raw rules as an array. + * + * @return Array RegExp strings */ - function deactivateOnFailure() { - if (storage.ClearURLsData.length === 0) { - storage.globalStatus = false; - storage.dataHash = ""; - changeIcon(); - storeHashStatus(5); - saveOnExit(); - } - } + this.getRawRules = function () { + return Object.keys(enabled_rawRules); + }; /** - * Get the hash for the rule file on GitLab. - * Check the hash with the hash form the local file. - * If the hash has changed, then download the new rule file. - * Else do nothing. + * Add a referral marketing rule to the referral marketing array + * and replace old referral marketing rule with new referral marketing rule. + * + * @param {String} rule RegExp as string + * @param {boolean} isActive Is this rule active? */ - function getHash() { - //Get the target hash from GitLab - const response = fetch(storage.hashURL).then(async response => { - return { - hash: (await response.text()).trim(), - status: response.status - } - }); + this.addReferralMarketing = function (rule, isActive = true) { + this.applyRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive); + }; - response.then(result => { - if (result.status === 200 && result.hash) { - dataHash = result.hash; + /** + * Add a exception to the exceptions array + * and replace old with new exception. + * + * @param {String} exception RegExp as string + * @param {Boolean} isActive Is this exception active? + */ + this.addException = function (exception, isActive = true) { + if (isActive) { + enabled_exceptions[exception] = true; - if (dataHash !== localDataHash.trim()) { - fetchFromURL(); - } else { - toObject(storage.ClearURLsData); - storeHashStatus(1); - saveOnDisk(['hashStatus']); - } - } else { - throw "The status code was not okay or the given hash were empty."; + if (disabled_exceptions[exception] !== undefined) { + delete disabled_exceptions[exception]; } - }).catch(error => { - console.error("[ClearURLs]: Could not download the rules hash from the given URL due to the following error: ", error); - dataHash = false; - deactivateOnFailure(); - }); - } + } else { + disabled_exceptions[exception] = true; - /* - * ################################################################## - * # Fetch Rules & Exception from URL # - * ################################################################## - */ - function fetchFromURL() { - const response = fetch(storage.ruleURL).then(async response => { - return { - data: (await response.clone().text()).trim(), - hash: await sha256((await response.text()).trim()), - status: response.status + if (enabled_exceptions[exception] !== undefined) { + delete enabled_exceptions[exception]; } - }) - - response.then(result => { - if (result.status === 200 && result.data) { - if (result.hash === dataHash.trim()) { - storage.ClearURLsData = result.data; - storage.dataHash = result.hash; - storeHashStatus(2); - } else { - storeHashStatus(3); - console.error("The hash does not match. Expected `" + result.hash + "` got `" + dataHash.trim() + "`"); - } - storage.ClearURLsData = JSON.parse(storage.ClearURLsData); - toObject(storage.ClearURLsData); - saveOnDisk(['ClearURLsData', 'dataHash', 'hashStatus']); - } else { - throw "The status code was not okay or the given rules were empty." - } - }).catch(error => { - console.error("[ClearURLs]: Could not download the rules from the given URL due to the following error: ", error); - deactivateOnFailure(); - }); + } + }; + + /** + * Add a HTTP method to methods list. + * + * @param {String} method HTTP Method Name + */ + this.addMethod = function (method) { + if (!methods.includes(method)) { + methods.push(method); + } } - // ################################################################## + /** + * Check the requests' method. + * + * @param {requestDetails} details Requests details + * @returns {boolean} should be filtered or not + */ + this.matchMethod = function (details) { + if (!methods.length) return true; + return methods.includes(details['method']); + } - /* - * ################################################################## - * # Supertyp Provider # - * ################################################################## - */ /** - * Declare constructor + * Private helper method to check if the url + * an exception. * - * @param {String} _name Provider name - * @param {boolean} _completeProvider Set URL Pattern as rule - * @param {boolean} _forceRedirection Whether redirects should be enforced via a "tabs.update" - * @param {boolean} _isActive Is the provider active? + * @param {String} url RegExp as string + * @return {boolean} if matching? true: false */ - function Provider(_name, _completeProvider = false, _forceRedirection = false, _isActive = true) { - let name = _name; - let urlPattern; - let enabled_rules = {}; - let disabled_rules = {}; - let enabled_exceptions = {}; - let disabled_exceptions = {}; - let canceling = _completeProvider; - let enabled_redirections = {}; - let disabled_redirections = {}; - let active = _isActive; - let enabled_rawRules = {}; - let disabled_rawRules = {}; - let enabled_referralMarketing = {}; - let disabled_referralMarketing = {}; - let methods = []; - - if (_completeProvider) { - enabled_rules[".*"] = true; - } + this.matchException = function (url) { + let result = false; - /** - * Returns whether redirects should be enforced via a "tabs.update" - * @return {boolean} whether redirects should be enforced - */ - this.shouldForceRedirect = function () { - return _forceRedirection; - }; + //Add the site blocked alert to every exception + if (url === siteBlockedAlert) return true; - /** - * Returns the provider name. - * @return {String} - */ - this.getName = function () { - return name; - }; + for (const exception in enabled_exceptions) { + if (result) break; - /** - * Add URL pattern. - * - * @require urlPatterns as RegExp - */ - this.setURLPattern = function (urlPatterns) { - urlPattern = new RegExp(urlPatterns, "i"); - }; + let exception_regex = new RegExp(exception, "i"); + result = exception_regex.test(url); + } - /** - * Return if the Provider Request is canceled - * @return {Boolean} isCanceled - */ - this.isCaneling = function () { - return canceling; - }; + return result; + }; - /** - * Check the url is matching the ProviderURL. - * - * @return {boolean} ProviderURL as RegExp - */ - this.matchURL = function (url) { - return urlPattern.test(url) && !(this.matchException(url)); - }; + /** + * Add a redirection to the redirections array + * and replace old with new redirection. + * + * @param {String} redirection RegExp as string + * @param {Boolean} isActive Is this redirection active? + */ + this.addRedirection = function (redirection, isActive = true) { + if (isActive) { + enabled_redirections[redirection] = true; - /** - * Apply a rule to a given tuple of rule array. - * @param enabledRuleArray array for enabled rules - * @param disabledRulesArray array for disabled rules - * @param {String} rule RegExp as string - * @param {boolean} isActive Is this rule active? - */ - this.applyRule = (enabledRuleArray, disabledRulesArray, rule, isActive = true) => { - if (isActive) { - enabledRuleArray[rule] = true; - - if (disabledRulesArray[rule] !== undefined) { - delete disabledRulesArray[rule]; - } - } else { - disabledRulesArray[rule] = true; + if (disabled_redirections[redirection] !== undefined) { + delete disabled_redirections[redirection]; + } + } else { + disabled_redirections[redirection] = true; - if (enabledRuleArray[rule] !== undefined) { - delete enabledRuleArray[rule]; - } + if (enabled_redirections[redirection] !== undefined) { + delete enabled_redirections[redirection]; } - }; + } + }; - /** - * Add a rule to the rule array - * and replace old rule with new rule. - * - * @param {String} rule RegExp as string - * @param {boolean} isActive Is this rule active? - */ - this.addRule = function (rule, isActive = true) { - this.applyRule(enabled_rules, disabled_rules, rule, isActive); - }; + /** + * Return all redirection. + * + * @return url + */ + this.getRedirection = function (url) { + let re = null; - /** - * Return all active rules as an array. - * - * @return Array RegExp strings - */ - this.getRules = function () { - if (!storage.referralMarketing) { - return Object.keys(Object.assign(enabled_rules, enabled_referralMarketing)); + for (const redirection in enabled_redirections) { + let result = (url.match(new RegExp(redirection, "i"))); + + if (result && result.length > 0 && redirection) { + re = (new RegExp(redirection, "i")).exec(url)[1]; + + break; } + } - return Object.keys(enabled_rules); - }; + return re; + }; +} - /** - * Add a raw rule to the raw rule array - * and replace old raw rule with new raw rule. - * - * @param {String} rule RegExp as string - * @param {boolean} isActive Is this rule active? - */ - this.addRawRule = function (rule, isActive = true) { - this.applyRule(enabled_rawRules, disabled_rawRules, rule, isActive); - }; +/** + * To prevent long loading on data urls + * we will check here for data urls. + * + * @type {requestDetails} + * @return {boolean} + */ +function isDataURL(requestDetails) { + const s = requestDetails.url; - /** - * Return all active raw rules as an array. - * - * @return Array RegExp strings - */ - this.getRawRules = function () { - return Object.keys(enabled_rawRules); - }; + return s.substring(0, 4) === "data"; +} - /** - * Add a referral marketing rule to the referral marketing array - * and replace old referral marketing rule with new referral marketing rule. - * - * @param {String} rule RegExp as string - * @param {boolean} isActive Is this rule active? - */ - this.addReferralMarketing = function (rule, isActive = true) { - this.applyRule(enabled_referralMarketing, disabled_referralMarketing, rule, isActive); - }; +/** + * Initialize the providers form the JSON object. + * + */ +function createProviders() { + let data = storage.ClearURLsData; - /** - * Add a exception to the exceptions array - * and replace old with new exception. - * - * @param {String} exception RegExp as string - * @param {Boolean} isActive Is this exception active? - */ - this.addException = function (exception, isActive = true) { - if (isActive) { - enabled_exceptions[exception] = true; - - if (disabled_exceptions[exception] !== undefined) { - delete disabled_exceptions[exception]; - } - } else { - disabled_exceptions[exception] = true; + for (let p = 0; p < prvKeys.length; p++) { + //Create new provider + providers.push(new Provider(prvKeys[p], data.providers[prvKeys[p]].getOrDefault('completeProvider', false), + data.providers[prvKeys[p]].getOrDefault('forceRedirection', false))); - if (enabled_exceptions[exception] !== undefined) { - delete enabled_exceptions[exception]; - } - } - }; + //Add URL Pattern + providers[p].setURLPattern(data.providers[prvKeys[p]].getOrDefault('urlPattern', '')); - /** - * Add a HTTP method to methods list. - * - * @param {String} method HTTP Method Name - */ - this.addMethod = function (method) { - if (methods.indexOf(method) === -1) { - methods.push(method); - } + let rules = data.providers[prvKeys[p]].getOrDefault('rules', []); + //Add rules to provider + for (let r = 0; r < rules.length; r++) { + providers[p].addRule(rules[r]); } - /** - * Check the requests' method. - * - * @param {requestDetails} details Requests details - * @returns {boolean} should be filtered or not - */ - this.matchMethod = function (details) { - if (!methods.length) return true; - return methods.indexOf(details['method']) > -1; + let rawRules = data.providers[prvKeys[p]].getOrDefault('rawRules', []); + //Add raw rules to provider + for (let raw = 0; raw < rawRules.length; raw++) { + providers[p].addRawRule(rawRules[raw]); } - /** - * Private helper method to check if the url - * an exception. - * - * @param {String} url RegExp as string - * @return {boolean} if matching? true: false - */ - this.matchException = function (url) { - let result = false; + let referralMarketingRules = data.providers[prvKeys[p]].getOrDefault('referralMarketing', []); + //Add referral marketing rules to provider + for (let referralMarketing = 0; referralMarketing < referralMarketingRules.length; referralMarketing++) { + providers[p].addReferralMarketing(referralMarketingRules[referralMarketing]); + } - //Add the site blocked alert to every exception - if (url === siteBlockedAlert) return true; + let exceptions = data.providers[prvKeys[p]].getOrDefault('exceptions', []); + //Add exceptions to provider + for (let e = 0; e < exceptions.length; e++) { + providers[p].addException(exceptions[e]); + } - for (const exception in enabled_exceptions) { - if (result) break; + let redirections = data.providers[prvKeys[p]].getOrDefault('redirections', []); + //Add redirections to provider + for (let re = 0; re < redirections.length; re++) { + providers[p].addRedirection(redirections[re]); + } - let exception_regex = new RegExp(exception, "i"); - result = exception_regex.test(url); - } + let methods = data.providers[prvKeys[p]].getOrDefault('methods', []); + //Add HTTP methods list to provider + for (let re = 0; re < methods.length; re++) { + providers[p].addMethod(methods[re]); + } + } +} - return result; - }; +/** + * Convert the external data to Objects and + * call the create provider function. + * + * @param {String} retrievedText - pure data form github + */ +function toObject(retrievedText) { + getKeys(storage.ClearURLsData.providers); + createProviders(); +} - /** - * Add a redirection to the redirections array - * and replace old with new redirection. - * - * @param {String} redirection RegExp as string - * @param {Boolean} isActive Is this redirection active? - */ - this.addRedirection = function (redirection, isActive = true) { - if (isActive) { - enabled_redirections[redirection] = true; - - if (disabled_redirections[redirection] !== undefined) { - delete disabled_redirections[redirection]; - } - } else { - disabled_redirections[redirection] = true; +/** + * Get the hash for the rule file on GitLab. + * Check the hash with the hash form the local file. + * If the hash has changed, then download the new rule file. + * Else do nothing. + */ +function getHash() { + //Get the target hash from GitLab + const response = fetch(storage.hashURL).then(async response => { + return { + hash: (await response.text()).trim(), + status: response.status + } + }); - if (enabled_redirections[redirection] !== undefined) { - delete enabled_redirections[redirection]; - } - } - }; + response.then(result => { + if (result.status === 200 && result.hash) { + dataHash = result.hash; - /** - * Return all redirection. - * - * @return url - */ - this.getRedirection = function (url) { - let re = null; + if (dataHash === localDataHash.trim()) { + toObject(storage.ClearURLsData); + storeHashStatus(1); + saveOnDisk(['hashStatus']); + } else { + fetchFromURL(); + } + } else { + throw "The status code was not okay or the given hash were empty."; + } + }).catch(error => { + console.error("[ClearURLs]: Could not download the rules hash from the given URL due to the following error: ", error); + dataHash = false; + deactivateOnFailure(); + }); +} - for (const redirection in enabled_redirections) { - let result = (url.match(new RegExp(redirection, "i"))); +/* +* ################################################################## +* # Fetch Rules & Exception from URL # +* ################################################################## +*/ +function fetchFromURL() { + const response = fetch(storage.ruleURL).then(async response => { + return { + data: (await response.clone().text()).trim(), + hash: await sha256((await response.text()).trim()), + status: response.status + } + }) + + response.then(result => { + if (result.status === 200 && result.data) { + if (result.hash === dataHash.trim()) { + storage.ClearURLsData = result.data; + storage.dataHash = result.hash; + storeHashStatus(2); + } else { + storeHashStatus(3); + console.error("The hash does not match. Expected `" + result.hash + "` got `" + dataHash.trim() + "`"); + } + storage.ClearURLsData = JSON.parse(storage.ClearURLsData); + toObject(storage.ClearURLsData); + saveOnDisk(['ClearURLsData', 'dataHash', 'hashStatus']); + } else { + throw "The status code was not okay or the given rules were empty." + } + }).catch(error => { + console.error("[ClearURLs]: Could not download the rules from the given URL due to the following error: ", error); + deactivateOnFailure(); + }); +} - if (result && result.length > 0 && redirection) { - re = (new RegExp(redirection, "i")).exec(url)[1]; +/** + * Function which called from the webRequest to + * remove the tracking fields from the url. + * + * @param {requestDetails} request webRequest-Object + * @return {Array} redirectUrl or none + */ +function clearUrl(request) { + const URLbeforeReplaceCount = countFields(request.url); - break; - } - } + //Add Fields form Request to global url counter + increaseTotalCounter(URLbeforeReplaceCount); - return re; + if (storage.globalStatus) { + let result = { + "changes": false, + "url": "", + "redirect": false, + "cancel": false }; - } - // ################################################################## + if (storage.pingBlocking && storage.pingRequestTypes.includes(request.type)) { + pushToLog(request.url, request.url, translate('log_ping_blocked')); + increaseBadged(false, request); + increaseTotalCounter(1); + return {cancel: true}; + } - /** - * Function which called from the webRequest to - * remove the tracking fields from the url. - * - * @param {requestDetails} request webRequest-Object - * @return {Array} redirectUrl or none - */ - function clearUrl(request) { - const URLbeforeReplaceCount = countFields(request.url); - - //Add Fields form Request to global url counter - increaseTotalCounter(URLbeforeReplaceCount); - - if (storage.globalStatus) { - let result = { - "changes": false, - "url": "", - "redirect": false, - "cancel": false - }; - - if (storage.pingBlocking && storage.pingRequestTypes.includes(request.type)) { - pushToLog(request.url, request.url, translate('log_ping_blocked')); - increaseBadged(false, request); - increaseTotalCounter(1); - return {cancel: true}; + /* + * Call for every provider the removeFieldsFormURL method. + */ + for (let i = 0; i < providers.length; i++) { + if (!providers[i].matchMethod(request)) continue; + if (providers[i].matchURL(request.url)) { + result = removeFieldsFormURL(providers[i], request.url, false, request); } /* - * Call for every provider the removeFieldsFormURL method. + * Expand urls and bypass tracking. + * Cancel the active request. */ - for (let i = 0; i < providers.length; i++) { - if (!providers[i].matchMethod(request)) continue; - if (providers[i].matchURL(request.url)) { - result = removeFieldsFormURL(providers[i], request.url, false, request); + if (result.redirect) { + if (providers[i].shouldForceRedirect() && + request.type === 'main_frame') { + browser.tabs.update(request.tabId, {url: result.url}).catch(handleError); + return {cancel: true}; } - /* - * Expand urls and bypass tracking. - * Cancel the active request. - */ - if (result.redirect) { - if (providers[i].shouldForceRedirect() && - request.type === 'main_frame') { - browser.tabs.update(request.tabId, {url: result.url}).catch(handleError); - return {cancel: true}; - } - - return { - redirectUrl: result.url - }; - } + return { + redirectUrl: result.url + }; + } - /* - * Cancel the Request and redirect to the site blocked alert page, - * to inform the user about the full url blocking. - */ - if (result.cancel) { - if (request.type === 'main_frame') { - const blockingPage = browser.runtime.getURL("html/siteBlockedAlert.html?source=" + encodeURIComponent(request.url)); - browser.tabs.update(request.tabId, {url: blockingPage}).catch(handleError); - - return {cancel: true}; - } else { - return { - redirectUrl: siteBlockedAlert - }; - } - } + /* + * Cancel the Request and redirect to the site blocked alert page, + * to inform the user about the full url blocking. + */ + if (result.cancel) { + if (request.type === 'main_frame') { + const blockingPage = browser.runtime.getURL("html/siteBlockedAlert.html?source=" + encodeURIComponent(request.url)); + browser.tabs.update(request.tabId, {url: blockingPage}).catch(handleError); - /* - * Ensure that the function go not into - * a loop. - */ - if (result.changes) { + return {cancel: true}; + } else { return { - redirectUrl: result.url + redirectUrl: siteBlockedAlert }; } } + + /* + * Ensure that the function go not into + * a loop. + */ + if (result.changes) { + return { + redirectUrl: result.url + }; + } } + } + + // Default case + return {}; +} - // Default case +/** + * Check the request. + */ +function promise(requestDetails) { + if (isDataURL(requestDetails)) { return {}; + } else { + return clearUrl(requestDetails); } +} +function start() { /** * Call loadOldDataFromStore, getHash, counter, status and log functions */ @@ -692,30 +714,6 @@ function start() { getHash(); setBadgedStatus(); - /** - * Check the request. - */ - function promise(requestDetails) { - if (isDataURL(requestDetails)) { - return {}; - } else { - return clearUrl(requestDetails); - } - } - - /** - * To prevent long loading on data urls - * we will check here for data urls. - * - * @type {requestDetails} - * @return {boolean} - */ - function isDataURL(requestDetails) { - const s = requestDetails.url; - - return s.substring(0, 4) === "data"; - } - /** * Call by each Request and checking the url. * diff --git a/core_js/tools.js b/core_js/tools.js index 8e204818..8267ade7 100644 --- a/core_js/tools.js +++ b/core_js/tools.js @@ -31,14 +31,14 @@ const logThreshold = 5000; * To support Waterfox. */ Array.prototype.rmEmpty = function () { - return this.filter(v => v); + return this.filter(Boolean); }; /* * To support Waterfox. */ Array.prototype.flatten = function () { - return this.reduce((a, b) => a.concat(b), []); + return this.flat(); }; /** @@ -226,11 +226,10 @@ function getCurrentURL() { * Check for browser. */ function getBrowser() { - if (typeof InstallTrigger !== 'undefined') { - return "Firefox"; - } else { + if (typeof InstallTrigger === 'undefined') { return "Chrome"; } + return "Firefox"; } /** @@ -287,7 +286,7 @@ function handleError(error) { */ function pushToLog(beforeProcessing, afterProcessing, rule) { const limit = Math.max(0, storage.logLimit); - if (storage.loggingStatus && limit !== 0 && !isNaN(limit)) { + if (storage.loggingStatus && limit !== 0 && !Number.isNaN(limit)) { while (storage.log.log.length >= limit || storage.log.log.length >= logThreshold) { storage.log.log.shift(); @@ -330,28 +329,70 @@ async function sha256(message) { } /** - * Generates a non-secure random ASCII string of length {@code len}. - * - * @returns non-secure random ASCII + * Generates a cryptographically-secure random ASCII string of length {@code len}, + * using the Web Crypto API (crypto.getRandomValues). The result is base-36 + * ([0-9a-z]) so it stays valid as an HTTP ETag value. + * + * @param {number} len length of the generated string + * @returns {string} random ASCII string */ function randomASCII(len) { - return [...Array(len)].map(() => (~~(Math.random() * 36)).toString(36)).join(''); + const bytes = crypto.getRandomValues(new Uint8Array(len)); + return Array.from(bytes, b => (b % 36).toString(36)).join(''); +} + +/** + * Decode a single percent-encoding into its raw byte value. + * Malformed sequences (e.g. a stray '%' or non-hex digits) are returned as-is, + * so that values are never corrupted. + * + * @param {string} str the string to decode + * @return {string} the percent-decoded string, with every %XX byte decoded + * to the character whose code unit equals that byte value + */ +function percentDecodeBytes(str) { + let rtn = ''; + + for (let i = 0; i < str.length; i++) { + if (str[i] === '%' && i + 2 < str.length) { + const hex = str.slice(i + 1, i + 3); + if (/^[0-9a-f]{2}$/i.test(hex)) { + rtn += String.fromCodePoint(Number.parseInt(hex, 16)); + i += 2; + continue; + } + } + rtn += str[i]; + } + + return rtn; } /** - * Returns an URLSearchParams as string. - * Does handle spaces correctly. + * Remove every query field whose (decoded) key matches one of the given rules + * from the raw query string, while preserving all other fields byte-for-byte. + * + * Unlike the URLSearchParams round-trip, the field values are never decoded as + * UTF-8, so values that use a non-UTF-8 encoding (e.g. GBK on Chinese sites + * such as 1688.com) survive unchanged. + * + * @param {string} query the raw query string, WITHOUT the leading '?' + * @param {Array} rules the provider rules to match against each key + * @return {string} the filtered query string, WITHOUT a leading '?' */ -function urlSearchParamsToString(searchParams) { - const rtn = [] - - searchParams.forEach((value, key) => { - if (value) { - rtn.push(key + '=' + encodeURIComponent(value)) - } else { - rtn.push(key) +function removeFieldsFromQuery(query, rules) { + const filtered = query.split('&').filter(pair => { + const rawKey = pair.split('=')[0]; + const decodedKey = percentDecodeBytes(rawKey); + + for (const rule of rules) { + if (new RegExp('^' + rule + '$', 'i').test(decodedKey)) { + return false; + } } - }) - return rtn.join('&') + return true; + }); + + return filtered.join('&'); }