Detect pending tab navigations for popup domain suggestions

package.json

@@ -42,6 +42,18 @@
     "build-th": "webpack --env browser=thunderbird",
     "build-th:watch": "webpack --watch --env browser=thunderbird --env dev=true"
   },
+  "jest": {
+    "setupFiles": [
+      "jest-webextension-mock"
+    ],
+    "testEnvironment": "jsdom",
+    "testMatch": [
+      "**/src/tests/**/*.test.ts"
+    ],
+    "transform": {
+      "^.+\\.tsx?$": "ts-jest"
+    }
+  },
   "license": "GPL-3.0",
   "dependencies": {
     "pako": "^3.0.1",

src/core/TabManager.ts

@@ -20,6 +20,7 @@ import { CompiledProxyRule, FailedRequestType, ProxyServer, TabProxyStatus } fro
 import { api, environment } from "../lib/environment";
 import { Settings } from "./Settings";
 import { ProxyRules } from "./ProxyRules";
+import { Utils } from "../lib/Utils";
 
 export class TabManager {
 
@@ -44,6 +45,20 @@ export class TabManager {
 		// listen to tab URL changes
 		api.tabs.onUpdated.addListener(TabManager.updateActiveTab);
 
+		let webNavigation = api.webNavigation;
+		if (webNavigation) {
+			if (webNavigation.onBeforeNavigate)
+				webNavigation.onBeforeNavigate.addListener(TabManager.handleNavigationStarted);
+			if (webNavigation.onCommitted)
+				webNavigation.onCommitted.addListener(TabManager.handleNavigationCommitted);
+			if (webNavigation.onHistoryStateUpdated)
+				webNavigation.onHistoryStateUpdated.addListener(TabManager.handleNavigationCommitted);
+			if (webNavigation.onReferenceFragmentUpdated)
+				webNavigation.onReferenceFragmentUpdated.addListener(TabManager.handleNavigationCommitted);
+			if (webNavigation.onErrorOccurred)
+				webNavigation.onErrorOccurred.addListener(TabManager.handleNavigationError);
+		}
+
 		api.tabs.onRemoved.addListener(TabManager.handleTabRemoved);
 
 		// listen for window switching
@@ -86,19 +101,28 @@ export class TabManager {
 		if (!tabData) {
 			tabData = TabManager.getOrSetTab(tabId, false);
 		}
-		if (tabData.proxifiedParentDocumentUrl != tabInfo.url) {
+
+		// Chrome may expose the destination as pendingUrl while url is still the old/placeholder page.
+		// Firefox only has url, and during loading that is often about:blank / about:newtab.
+		let incomingUrl = tabInfo.pendingUrl || tabInfo.url || "";
+		let keepExistingUrl = Utils.shouldPreserveTrackedUrl(tabData.url, incomingUrl);
+		let effectiveUrl = keepExistingUrl ? tabData.url : incomingUrl;
+
+		if (effectiveUrl && tabData.proxifiedParentDocumentUrl != effectiveUrl) {
 			// resettings the state
 			tabData.resetTabState();
 
 			// apply `proxified` value
-			TabManager.setRuleForProxyPerOrigin(tabData, tabInfo.url);
+			TabManager.setRuleForProxyPerOrigin(tabData, effectiveUrl);
 		}
 		tabData.updated = new Date();
 		tabData.incognito = tabInfo.incognito;
-		tabData.url = tabInfo.url;
 		tabData.index = tabInfo.index;
-		if (!tabData.proxifiedParentDocumentUrl)
-			tabData.proxifiedParentDocumentUrl = tabInfo.url;
+		if (effectiveUrl) {
+			tabData.url = effectiveUrl;
+			if (!tabData.proxifiedParentDocumentUrl)
+				tabData.proxifiedParentDocumentUrl = effectiveUrl;
+		}
 
 		// saving the tab in the storage
 		TabManager.tabs[tabId] = tabData;
@@ -192,6 +216,58 @@ export class TabManager {
 		tabData.clearFailedRequests();
 	}
 
+	private static isMainFrameNavigation(details: any): boolean {
+		return details && details.tabId > -1 && details.frameId === 0 && details.url;
+	}
+
+	private static handleNavigationStarted(details: any) {
+		if (!TabManager.isMainFrameNavigation(details))
+			return;
+
+		TabManager.updateTabUrlFromNavigation(details.tabId, details.url);
+	}
+
+	private static handleNavigationCommitted(details: any) {
+		if (!TabManager.isMainFrameNavigation(details))
+			return;
+
+		TabManager.updateTabUrlFromNavigation(details.tabId, details.url);
+	}
+
+	private static handleNavigationError(details: any) {
+		if (!TabManager.isMainFrameNavigation(details))
+			return;
+
+		let tabData = TabManager.tabs[details.tabId];
+		if (!tabData || tabData.url !== details.url)
+			return;
+
+		TabManager.loadTabData(tabData);
+	}
+
+	private static updateTabUrlFromNavigation(tabId: number, url: string) {
+		let tabData = TabManager.tabs[tabId];
+		let tabDataCreated = false;
+		if (!tabData) {
+			tabData = TabManager.getOrSetTab(tabId, false, url);
+			tabDataCreated = true;
+		}
+
+		if (!tabDataCreated && tabData.url === url)
+			return;
+
+		tabData.clearFailedRequests();
+		tabData.resetTabState();
+		TabManager.setRuleForProxyPerOrigin(tabData, url);
+		tabData.updated = new Date();
+		tabData.url = url;
+		tabData.proxifiedParentDocumentUrl = url;
+
+		if (!TabManager.currentTab || TabManager.currentTab.tabId === tabId)
+			TabManager.currentTab = tabData;
+		TabManager.onTabUpdated.trigger(tabData);
+	}
+
 	private static handleTabUpdated(tabId: number, changeInfo: any, tabInfo: any) {
 		// only if url of the page is changed
 
@@ -228,7 +304,10 @@ export class TabManager {
 			if (tabData) {
 				// reload tab data
 				tabData.clearFailedRequests();
-				TabManager.loadTabData(tabData);
+				if (changeInfo.url &&
+					!Utils.shouldPreserveTrackedUrl(tabData.url, changeInfo.url)) {
+					TabManager.updateTabUrlFromNavigation(tabId, changeInfo.url);
+				}
 				callOnUpdate = true;
 			}
 		}
@@ -300,4 +379,4 @@ export class TabDataStatuses {
 
 	/** Always enabled has bypassed and no proxy is applied */
 	public hasAlwaysEnabledByPassed: boolean;
-}
+}

src/lib/Utils.ts

@@ -272,6 +272,37 @@ export class Utils {
 		catch (e) { return false; }
 	}
 
+	/** New-tab placeholders reported by tabs.query/get while a real navigation is already in flight. */
+	public static isTransientTabUrl(url: string): boolean {
+		if (!url)
+			return true;
+
+		let value = url.toLowerCase();
+		return value === "about:blank"
+			|| value === "about:newtab"
+			|| value === "about:home"
+			|| value === "about:privatebrowsing"
+			|| value.startsWith("chrome://newtab")
+			|| value.startsWith("chrome://new-tab-page")
+			|| value.startsWith("edge://newtab");
+	}
+
+	/**
+	 * tabs.Tab.url lags webNavigation in both Firefox and Chrome.
+	 * Preserve the tracked URL only when the tabs API has no URL or is still
+	 * reporting a new-tab placeholder.
+	 */
+	public static shouldPreserveTrackedUrl(existingUrl: string, incomingUrl: string): boolean {
+		if (!existingUrl)
+			return false;
+		if (!incomingUrl)
+			return true;
+		if (incomingUrl === existingUrl)
+			return false;
+
+		return Utils.isTransientTabUrl(incomingUrl) && !Utils.isTransientTabUrl(existingUrl);
+	}
+
 	public static urlHasSchema(url: string): boolean {
 		// note: this will accept like http:/example.org/ in Chrome and Firefox
 		if (!url)

src/manifest-chrome-mv2.json

@@ -17,6 +17,7 @@
 		"<all_urls>",
 		"activeTab",
 		"tabs",
+		"webNavigation",
 		"proxy",
 		"webRequest",
 		"webRequestBlocking",

src/manifest-chrome.json

@@ -16,6 +16,7 @@
 	"permissions": [
 		"activeTab",
 		"tabs",
+		"webNavigation",
 		"proxy",
 		"webRequest",
 		"webRequestAuthProvider",

src/manifest-edge.json

@@ -17,6 +17,7 @@
 		"<all_urls>",
 		"activeTab",
 		"tabs",
+		"webNavigation",
 		"proxy",
 		"webRequest",
 		"webRequestBlocking",

src/manifest-firefox-android.json

@@ -17,6 +17,7 @@
 		"<all_urls>",
 		"activeTab",
 		"tabs",
+		"webNavigation",
 		"proxy",
 		"webRequest",
 		"webRequestBlocking",

src/manifest-firefox-unlisted.json

@@ -15,9 +15,10 @@
 	},
 	"permissions": [
 		"<all_urls>",
-		"activeTab",
-		"tabs",
-		"proxy",
+		"activeTab",
+		"tabs",
+		"webNavigation",
+		"proxy",
 		"webRequest",
 		"webRequestBlocking",
 		"storage",

src/manifest-firefox.json

@@ -15,9 +15,10 @@
 	},
 	"permissions": [
 		"<all_urls>",
-		"activeTab",
-		"tabs",
-		"proxy",
+		"activeTab",
+		"tabs",
+		"webNavigation",
+		"proxy",
 		"webRequest",
 		"webRequestBlocking",
 		"storage",

src/manifest-opera.json

@@ -17,6 +17,7 @@
 		"<all_urls>",
 		"activeTab",
 		"tabs",
+		"webNavigation",
 		"proxy",
 		"webRequest",
 		"webRequestBlocking",

src/manifest-thunderbird.json

@@ -17,6 +17,7 @@
 		"<all_urls>",
 		"activeTab",
 		"tabs",
+		"webNavigation",
 		"proxy",
 		"webRequest",
 		"webRequestBlocking",