From 20e352736243847a2033de2b35b65341f0741f63 Mon Sep 17 00:00:00 2001 From: Raymond Fowkes Date: Fri, 5 Dec 2025 15:15:30 -0800 Subject: [PATCH 1/3] Capture the Chromium DNS Info --- src/NetBlame/NetBlameDataProcessor.cs | 14 +- src/NetBlame/Providers/Chromium.cs | 214 ++++++++++++++++++++++++++ src/NetBlame/Providers/DNSClient.cs | 54 +++++++ src/WPRP/EdgeChrome.15002.wprp | 4 + 4 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 src/NetBlame/Providers/Chromium.cs diff --git a/src/NetBlame/NetBlameDataProcessor.cs b/src/NetBlame/NetBlameDataProcessor.cs index 1755754..32f5e6f 100644 --- a/src/NetBlame/NetBlameDataProcessor.cs +++ b/src/NetBlame/NetBlameDataProcessor.cs @@ -31,6 +31,7 @@ using NetBlameCustomDataSource.WinsockAFD; using NetBlameCustomDataSource.WinsockNameRes; using NetBlameCustomDataSource.WThreadPool; +using NetBlameCustomDataSource.Chromium; using static NetBlameCustomDataSource.Util; @@ -381,7 +382,14 @@ public override DataSourceInfo GetDataSourceInfo() DNSTable.guid, // Microsoft-Windows-DNS-Client WinINetTable.guid, // Microsoft-Windows-WinINet WinHttpTable.guid, // Microsoft-Windows-WinHttp - WebIOTable.guid // Microsoft-Windows-WebIO + WebIOTable.guid, // Microsoft-Windows-WebIO + // For the browser's DNS records: + Chromium.DNSInfo.rgGuid[0], // Microsoft.MSEdgeStable + Chromium.DNSInfo.rgGuid[1], // Microsoft.MSEdgeBeta + Chromium.DNSInfo.rgGuid[2], // Microsoft.MSEdgeWebView2 + Chromium.DNSInfo.rgGuid[3], // Microsoft.MSEdgeCanary + Chromium.DNSInfo.rgGuid[4], // Microsoft.MSEdgeDev + Chromium.DNSInfo.rgGuid[5] // CHROME }; // These providers will get processed in the ClassicEventConsumer callback. @@ -619,6 +627,10 @@ AllTables GenerateProviderTables(ClassicEventConsumer eventConsumer, Cancellatio // Lower frequency event for polling. if (cancellationToken.IsCancellationRequested) return null; } + else if (Array.IndexOf(Chromium.DNSInfo.rgGuid, evtGeneric.ProviderId) >= 0) + { + Chromium.DNSInfo.Dispatch(evtGeneric, allTables.dnsTable); + } } // foreach evtGeneric // Generic events were all dispatched, but some classic events may remain. diff --git a/src/NetBlame/Providers/Chromium.cs b/src/NetBlame/Providers/Chromium.cs new file mode 100644 index 0000000..1c5060b --- /dev/null +++ b/src/NetBlame/Providers/Chromium.cs @@ -0,0 +1,214 @@ +using System; +using System.Text.Json; + +using Microsoft.Windows.EventTracing.Events; + +using static NetBlameCustomDataSource.Util; // Assert + + +namespace NetBlameCustomDataSource.Chromium +{ + class ResolvedDNS + { + public string Domain; // "star-mini.c1q0r.facebook.com" + public string Alias; // "www.facebook.com" or null + public string[] rgAddress; // never null, no missing or white-space elements, no port numbers + } + + static class DNSInfo + { + /* + Parse the JSON string from the 'params' field where: 'source_type' == 'HOST_RESOLVER_IMPL_JOB' + + Simplified Input: + { + "results": + [ + { + "domain_name":"ax-0001.ax-msedge.net", + "endpoints": + [ + { "address":"150.171.27.10" }, + { "address":"150.171.28.10" } + ], + "type":"data" + }, + { + "alias_target":"c-bing-com.ax-0001.ax-msedge.net", + "domain_name":"c.bing.com", + "type":"alias" + }, + { + "alias_target":"ax-0001.ax-msedge.net", + "domain_name":"c-bing-com.ax-0001.ax-msedge.net", + "type":"alias" +   } +  ], + } + + Output: + Domain = "ax-0001.ax-msedge.net" + Alias = "c.bing.com" + rgAddress = { "150.171.27.10", "150.171.28.10" } + */ + static ResolvedDNS ParseHostResolveDNS(string json) + { + JsonDocument jd = null; + try + { + jd = JsonDocument.Parse(json); + } + catch + { + jd = null; + } + + if (jd == null) + return null; + + using (jd) + { + if (!jd.RootElement.TryGetProperty("results", out JsonElement results) || results.ValueKind != JsonValueKind.Array || results.GetArrayLength() == 0) + return null; + + bool[] rgAlias = new bool[results.GetArrayLength()]; + + string[] rgEndPoints = null; + + string sDomain = null; + int iRes = -1; + foreach (var res in results.EnumerateArray()) + { + ++iRes; + + if (!res.TryGetProperty("type", out JsonElement type) || type.ValueKind != JsonValueKind.String) + continue; + + switch (type.GetString()) + { + case "data": + break; + + case "alias": + rgAlias[iRes] = true; + continue; + + // case "metadata": + // case "error": + default: + continue; + } + + // Only one "data" element! + AssertImportant(sDomain == null); + + if (!res.TryGetProperty("domain_name", out JsonElement domain_name) || type.ValueKind != JsonValueKind.String) + continue; + + sDomain = domain_name.GetString(); + + if (string.IsNullOrWhiteSpace(sDomain)) + { + sDomain = null; + continue; + } + + if (!res.TryGetProperty("endpoints", out JsonElement endpoints) || endpoints.ValueKind != JsonValueKind.Array || endpoints.GetArrayLength() == 0) + continue; + + int iEP = 0; + rgEndPoints = new string[endpoints.GetArrayLength()]; + foreach (var endpoint in endpoints.EnumerateArray()) + { + if (!endpoint.TryGetProperty("address", out JsonElement address) || address.ValueKind != JsonValueKind.String) + continue; + + if (string.IsNullOrWhiteSpace(address.GetString())) + continue; + + rgEndPoints[iEP++] = address.GetString(); + } + + // Continue the loop only to populate: rgAlias[iRes] + } // foreach res + + if (sDomain == null) + return null; + + if (rgEndPoints == null) + return null; + + // Find and trim any trailing null strings. (None have ever been observed.) + + int iEmpty = Array.FindIndex(rgEndPoints, s => s == null); + + if (iEmpty == 0) + return null; + + if (iEmpty > 0) + rgEndPoints = rgEndPoints[..iEmpty]; + + // Now scan through the "alias" elements, matching alias to domain, and ultimately arriving at a final domain. (Truncated N^2 for small N) + + string target = sDomain; + bool fAdvance; + do + { + fAdvance = false; + iRes = -1; + foreach (var res in results.EnumerateArray()) + { + if (!rgAlias[++iRes]) continue; + + if (!res.TryGetProperty("alias_target", out JsonElement alias_target) || alias_target.ValueKind != JsonValueKind.String) + continue; + + if (alias_target.GetString() != target) continue; + + rgAlias[iRes] = false; + + if (!res.TryGetProperty("domain_name", out JsonElement domain_name) || domain_name.ValueKind != JsonValueKind.String) + continue; + + target = domain_name.GetString(); + fAdvance = true; + } + } while (fAdvance); + + if (target == sDomain) + target = null; + + return new ResolvedDNS + { + Domain = sDomain, + Alias = target, + rgAddress = rgEndPoints + }; + } // using (jd) + } // ParseHostResolveDNS + + + public static Guid[] rgGuid = + { + new Guid("{3A5F2396-5C8F-4F1F-9B67-6CCA6C990E61}"), // Microsoft.MSEdgeStable + new Guid("{BD089BAA-4E52-4794-A887-9E96868570D2}"), // Microsoft.MSEdgeBeta + new Guid("{E16EC3D2-BB0F-4E8F-BDB8-DE0BEA82DC3D}"), // Microsoft.MSEdgeWebView2 + new Guid("{C56B8664-45C5-4E65-B3C7-A8D6BD3F2E67}"), // Microsoft.MSEdgeCanary + new Guid("{D30B5C9F-B58F-4DC9-AFAF-134405D72107}"), // Microsoft.MSEdgeDev + new Guid("{d2d578d9-2936-45b6-a09f-30e32715f42d}") // CHROME + }; + + public static void Dispatch(in IGenericEvent evt, DNSClient.DNSTable dnsTable) + { + if (!evt.TaskName.Equals("HOST_RESOLVER_DNS_TASK_EXTRACTION_RESULTS")) + return; + + AssertImportant(evt.GetString("source_type").Equals("HOST_RESOLVER_IMPL_JOB")); + + ResolvedDNS rdns = ParseHostResolveDNS(evt.GetString("params")); + + if (rdns != null) + dnsTable.AddServerAndAddress(rdns.Domain, rdns.Alias, rdns.rgAddress); + } + } // DNSInfo +} // namespace NetBlameCustomDataSource.DNSClient \ No newline at end of file diff --git a/src/NetBlame/Providers/DNSClient.cs b/src/NetBlame/Providers/DNSClient.cs index 6fef872..fc44cdc 100644 --- a/src/NetBlame/Providers/DNSClient.cs +++ b/src/NetBlame/Providers/DNSClient.cs @@ -598,6 +598,60 @@ public string ConnectNameResolution(in IGenericEvent evt, string strServer) } // ConnectNameResolution + /* + The Chromium DNS provider has been parsed from JSON to provide these values: + domain1 // "star-mini.c1q0r.facebook.com" + domain2 // "www.facebook.com" or null + rgAddress // never null, no missing or white-space elements, no port numbers + */ + public void AddServerAndAddress(string domain1, string domain2, string[] rgAddress) + { + string strServerName; + string strCanonical; + + if (domain2 != null) + { + strServerName = domain2; // "www.facebook.com" + strCanonical = domain1; // "star-mini.c1q0r.facebook.com" + } + else + { + strServerName = domain1; // "star-mini.c1q0r.facebook.com" + strCanonical = null; + } + + uint iDNS = 0; + + foreach (string strAddr in rgAddress) + { + if (TryParseEx(strAddr, out IPAddress ipAddress)) + { + uint iAddr = this.AddDNSEntry(strServerName, ipAddress, ref iDNS); + AssertImportant(iAddr > 0); // 1-based + } + else + { + // All of these address strings should be parseable! + AssertImportant(false); + } + } + + if (iDNS != 0 && strCanonical != null) + { + DNSClient.DNSEntry dnsEntry = this.DNSEntryFromI(iDNS); + if (dnsEntry.strNameAlt == null) + { + AssertImportant(!String.Equals(dnsEntry.strServer, strCanonical, StringComparison.OrdinalIgnoreCase)); + dnsEntry.strNameAlt = strCanonical; + } + else + { + AssertInfo(String.Equals(dnsEntry.strNameAlt, strCanonical, StringComparison.OrdinalIgnoreCase)); + } + } + } // AddServerAndAddress + + /* Parse the given Address/Port string, and return the 1-based iDNS, 1-based iAddr, and Port. */ diff --git a/src/WPRP/EdgeChrome.15002.wprp b/src/WPRP/EdgeChrome.15002.wprp index 950d1e0..31f9dce 100644 --- a/src/WPRP/EdgeChrome.15002.wprp +++ b/src/WPRP/EdgeChrome.15002.wprp @@ -113,6 +113,7 @@ + @@ -128,6 +129,7 @@ + @@ -198,6 +200,7 @@ + @@ -211,6 +214,7 @@ + From 9cbc6b8bfa42248382163bf9c59b9c197eed0a04 Mon Sep 17 00:00:00 2001 From: Raymond Fowkes Date: Thu, 23 Jul 2026 15:20:56 -0700 Subject: [PATCH 2/3] NetBlame Chromium Processing --- src/NetBlame/Tables/NetBlameTable.Chromium.cs | 448 ++++++++++++++++++ .../Tables/NetBlameTable.ChromiumStreams.cs | 429 +++++++++++++++++ src/WPAP/ChromiumRegions.wpaProfile | 82 ++++ src/WPAP/EDGE/Chromium-Request.regions.xml | 121 +++++ 4 files changed, 1080 insertions(+) create mode 100644 src/NetBlame/Tables/NetBlameTable.Chromium.cs create mode 100644 src/NetBlame/Tables/NetBlameTable.ChromiumStreams.cs create mode 100644 src/WPAP/ChromiumRegions.wpaProfile create mode 100644 src/WPAP/EDGE/Chromium-Request.regions.xml diff --git a/src/NetBlame/Tables/NetBlameTable.Chromium.cs b/src/NetBlame/Tables/NetBlameTable.Chromium.cs new file mode 100644 index 0000000..42121d3 --- /dev/null +++ b/src/NetBlame/Tables/NetBlameTable.Chromium.cs @@ -0,0 +1,448 @@ +// Copyright(c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +using Microsoft.Performance.SDK.Processing; + +using NetBlameCustomDataSource.Chromium; + +using QWord = System.UInt64; + + +namespace NetBlameCustomDataSource.Tables +{ + [Table] + public sealed class NetBlameTableChromium : NetBlameTableBase + { + public NetBlameTableChromium(PendingSources sources, AllTables tables, IApplicationEnvironment environ) : base(sources, tables, environ) { } + + public static TableDescriptor TableDescriptor => new TableDescriptor( + new Guid("12a5df39-85b6-42e1-8e6a-0b25498ab4aa"), // The GUID must be unique across all tables. + "NetBlame Chromium Requests", // The Table must have a name. + "NetBlame Network Analyzer - Chromium Requests", // The Table must have a description. + "Network" // Optional category for grouping different types of tables in WPA UI. + ); + + + // Declare columns here. You can do this using the ColumnConfiguration class. + // It is possible to declaratively describe the table configuration as well. Please refer to our Advanced Topics Wiki page for more information. + // + // The Column metadata describes each column in the table. + // Each column must have a unique GUID and a unique name. The GUID must be unique globally; the name only unique within the table. + static class Columns + { + // These are created via ColumnsCommon(), below: + // colProcessName, colProcess, colThread, colStack, colDuration, colOpenTime, colCloseTime + + public static readonly ColumnConfiguration colServer = + DeclareColumn + ( + "Server", + "Base Server Name", + width: 180, + visible: true + ); + + public static readonly ColumnConfiguration colServerAlt = + DeclareColumn + ( + "Alt DNS Name", + "Alternate Server DNS Name", + width: 180, + visible: false + ); + + public static readonly ColumnConfiguration colAnonKey = + DeclareColumn + ( + "Origin Context", + "Network Anonymization Key", + width: 180, + visible: false + ); + + public static readonly ColumnConfiguration colMethod = + DeclareColumn + ( + "Method", + "HTTP Method", + width: 82, + visible: true + ); + + public static readonly ColumnConfiguration colUrlPath = + DeclareColumn + ( + "URL", + "Full Url Path", + width: 500, + visible: true + ); + + public static readonly ColumnConfiguration colPriority = + DeclareColumn + ( + "Priority", + "URL Connection Priority", + width: 54, + visible: true + ); + + public static readonly ColumnConfiguration colSend = + DeclareColumn + ( + "Send (B)", + "Bytes Sent (estimted)", + mode: AggregationMode.Sum, + width: 70, + align: TextAlignment.Right, + format: ColumnFormats.NumberFormat, + visible: true + ); + + public static readonly ColumnConfiguration colRecv = + DeclareColumn + ( + "Recv (B)", + "Bytes Received (estimated)", + mode: AggregationMode.Sum, + width: 70, + align: TextAlignment.Right, + format: ColumnFormats.NumberFormat, + visible: true + ); + + public static readonly ColumnConfiguration colAddr = + DeclareColumn + ( + "IP Address", + "Remote IP Address", + width: 110, + visible: true + ); + + public static readonly ColumnConfiguration colPort = + DeclareColumn + ( + "Port", + "Remote IP Address Port:\r\n80=http, 443=https, etc.", + width: 38, + align: TextAlignment.Center, + visible: true + ); + + public static readonly ColumnConfiguration colSocket = + DeclareColumn + ( + "Socket ID", + "TCP / WinSock / Chromium Socket ID", + width: 78, + align: TextAlignment.Center, + visible: false + ); + + public static readonly ColumnConfiguration colConnect = + DeclareColumn + ( + "Connection ID", + "Winsock Connection ID (reusable)", + width: 128, + format: ColumnFormats.HexFormat, + visible: false + ); + + public static readonly ColumnConfiguration colRequest = + DeclareColumn + ( + "Request ID", + "Chromium Request ID (reusable)", + width: 148, + align: TextAlignment.Right, + format: ColumnFormats.NumberFormat, + visible: false + ); + + public static readonly ColumnConfiguration colSession = + DeclareColumn + ( + "Session ID", + "ID of the Session creation event", + width: 150, + align: TextAlignment.Right, // number + format: ColumnFormats.NumberFormat, + visible: false + ); +#if DEBUG + public static readonly ColumnConfiguration colIDs = + DeclareColumn + ( + "IDs", + "ID1; ID2; ...", + width: 160, + visible: false + ); + + public static readonly ColumnConfiguration colSourceDeps = + DeclareColumn + ( + "Source Dependencies", + "\"source_dependency\":\"id\":#", // "\"source_dependency\":{\"id\":#}" // {Curly Braces} in the description can crash WPA 11.8.423.12582 + width: 130, + visible: false + ); +#endif // DEBUG + public static readonly ColumnConfiguration colStatus = + DeclareColumn + ( + "Status", + "HTTP Status Value", + width: 48, + visible: true + ); + + public static readonly ColumnConfiguration colError = + DeclareColumn + ( + "Error", + "Error value or condition", + width: 64, + visible: false + ); + + public static readonly ColumnConfiguration colTransport = + DeclareColumn + ( + "Transport", + "Session Transport Type: QUIC / HTTP2 / HTTP1", + width: 64, + visible: true + ); + +/* + public static readonly ColumnConfiguration colNAME = + DeclareColumn + ( + "NAME", + "DESC", + width: 180, + visible: true + ); +*/ + } // Columns + + + // Generators for use in Projections + // Static functions are more efficient than non-static, inline lambdas. + // None of these should return null. + static class Generators + { + // These generators are in GeneratorCommon<>: + // ProcessData, ProcName, ProcFullName, Thread, OpenStack, OpenTime, CloseTime, Duration + + public static string Server(Request req) => req.Domain; + + public static string ServerAlt(Request req) => req.Canon; + + public static string AnonKey(Request req) => Chromium.Util.ScrubAnonKey(req.anon_key); + + public static string Method(Request req) => req.method ?? String.Empty; + + public static string UrlPath(Request req) => req.URL ?? String.Empty; + + public static string Priority(Request req) => req.priority.ToString(); + + public static string Port(Request req) => req.port.ToStringOrBlank(); + +#if DEBUG + private static readonly System.Text.StringBuilder sb = new(160); + + public static string IDs(Request req) + { + sb.Clear(); + + foreach (System.UInt64 uid in req.rguidDB) + sb.AppendFormat("{0:#,#} ", uid); + + return sb.ToString(); + } + + public static string SourceDep(Request req) + { + sb.Clear(); + + foreach (int srcdep in req.rgsrcdepDB) + sb.AppendFormat("{0:#} ", srcdep); + + return sb.ToString(); + } +#endif // DEBUG + + public static uint Send(Request req) => req.stream?.CbSend() ?? 0; + + public static uint Recv(Request req) => req.stream?.CbRecv() ?? 0; + + public static string IPAddress(Request req) => req.AddressAndPort(); + + public static string Socket(Request req) => (req.Session?.socket?.WSSocket()).ToStringOrBlank(); + + public static string Status(Request req) => req.stream?.strHTTPStatus ?? string.Empty; + + public static string Error(Request req) => req.Error(); + + public static string Transport(Request req) => req.Transport(); + + public static QWord Request(Request req) => req.UIDRequest; + + public static QWord Session(Request req) => req.UIDSession; + + public static QWord Connect(Request req) => req.WSConnection; + } // Generators + + + public override void Build(ITableBuilder tableBuilder) + { + // Implement your columns here. + // Columns are implemented via Projections, which are simply functions that map a row index to a data point. + // Create projection for each column by composing the base projection with another projection that maps to the data point as needed. + + var tableBase = this.Tables?.chromiumTable; + + if (tableBase == null) return; + + var chromiumBaseProjector = Projection.Index(tableBase); + + // int -> string: URL, Server, Method, Protocol, Status + var chromiumServerProjector = Projection.Project(chromiumBaseProjector, Generators.Server); + var chromiumServerAltProjector = Projection.Project(chromiumBaseProjector, Generators.ServerAlt); + var chromiumAnonKeyProjector = Projection.Project(chromiumBaseProjector, Generators.AnonKey); + var chromiumMethodProjector = Projection.Project(chromiumBaseProjector, Generators.Method); + var chromiumPathProjector = Projection.Project(chromiumBaseProjector, Generators.UrlPath); + var chromiumPriorityProjector = Projection.Project(chromiumBaseProjector, Generators.Priority); + var chromiumStatusProjector = Projection.Project(chromiumBaseProjector, Generators.Status); + var chromiumErrorProjector = Projection.Project(chromiumBaseProjector, Generators.Error); + var chromiumTransportProjector = Projection.Project(chromiumBaseProjector, Generators.Transport); + + // int -> uint: Send, Recv, Port, Socket + var chromiumSendProjector = Projection.Project(chromiumBaseProjector, Generators.Send); + var chromiumRecvProjector = Projection.Project(chromiumBaseProjector, Generators.Recv); + var chromiumPortProjector = Projection.Project(chromiumBaseProjector, Generators.Port); + var chromiumSocketProjector = Projection.Project(chromiumBaseProjector, Generators.Socket); + + // int -> IPAddress + var chromiumAddressProjector = Projection.Project(chromiumBaseProjector, Generators.IPAddress); + + // int -> QWord + var chromiumRequestProjector = Projection.Project(chromiumBaseProjector, Generators.Request); + var chromiumConnectProjector = Projection.Project(chromiumBaseProjector, Generators.Connect); + var chromiumSessionProjector = Projection.Project(chromiumBaseProjector, Generators.Session); + +#if DEBUG + // int -> "1,234; 5,678; ..." + var chromiumIDProjector = Projection.Project(chromiumBaseProjector, Generators.IDs); + + // int -> "123; 456" + var chromiumSourceDepProjector = Projection.Project(chromiumBaseProjector, Generators.SourceDep); +#endif // DEBUG + + // int -> common projectors: process, thread, stack, start/end time, duration + var commonProjectors = new ProjectorCommon(this.Sources, in chromiumBaseProjector, tableBase.Count); + + + // Table Configurations describe how your table should be presented to the user: + // the columns to show, what order to show them, which columns to aggregate, and which columns to graph. + // You may provide a number of columns in your table, but only want to show a subset of them by default so as not to overwhelm the user. + // The user can still open the table properties in WPA to turn on or off columns. + // The table configuration class also exposes four (4) columns that WPA explicitly recognizes: Pivot Column, Graph Column, Left Freeze Column, Right Freeze Column + // For more information about what these columns do, go to "Advanced Topics" -> "Table Configuration" in our Wiki. Link can be found in README.md + + // Common columns: colProcessName, colProcess, colStack, colDuration, colOpenTime, colCloseTime + ColumnsCommon commonColumns = new ColumnsCommon(); + + var config = new TableConfiguration("Chromium Info") + { + Columns = new[] + { + commonColumns.colProcessName, + commonColumns.colProcess, + Columns.colServer, + commonColumns.colStack, + TableConfiguration.PivotColumn, /*------------*/ + commonColumns.colCount, + TableConfiguration.LeftFreezeColumn, /*------*/ + commonColumns.colThread, + Columns.colServerAlt, + Columns.colAnonKey, + Columns.colMethod, + Columns.colPriority, + Columns.colSend, + Columns.colRecv, + Columns.colUrlPath, + Columns.colAddr, + Columns.colPort, + Columns.colSocket, + Columns.colConnect, + Columns.colSession, + Columns.colRequest, +#if DEBUG + Columns.colIDs, + Columns.colSourceDeps, +#endif // DEBUG + Columns.colTransport, + Columns.colStatus, + Columns.colError, + TableConfiguration.RightFreezeColumn, /*------*/ + commonColumns.colDuration, + TableConfiguration.GraphColumn, /*------------*/ + commonColumns.colOpenTime, + commonColumns.colCloseTime, + } + }; +#if !DEBUG +/* + When open/close timestamps are given this meta-data, zeros get eliminated. +*/ + // Advanced Settings / Graph Configuration + config.AddColumnRole(ColumnRole.StartTime, commonColumns.colOpenTime.Metadata.Guid); + config.AddColumnRole(ColumnRole.EndTime, commonColumns.colCloseTime.Metadata.Guid); +#endif // !DEBUG + config.AddColumnRole(ColumnRole.Duration, commonColumns.colDuration.Metadata.Guid); + + // Use the table builder to build the table. + // Add and set table configuration if applicable. + // Then set the row count and then add the columns using AddColumn. + + tableBuilder + .AddTableConfiguration(config) + .SetDefaultTableConfiguration(config) + .SetRowCount(tableBase.Count) + .AddCommonColumns(commonColumns, commonProjectors, null) // Process, Thread, Duration, Open/CloseTime + + .AddHierarchicalColumn(commonColumns.colStack, commonProjectors.stackProjector, Sources.stackAccessProvider) + .AddColumn(Columns.colServer, chromiumServerProjector) + .AddColumn(Columns.colServerAlt, chromiumServerAltProjector) + .AddColumn(Columns.colAnonKey, chromiumAnonKeyProjector) + .AddColumn(Columns.colMethod, chromiumMethodProjector) + .AddColumn(Columns.colPriority, chromiumPriorityProjector) + .AddColumn(Columns.colSend, chromiumSendProjector) + .AddColumn(Columns.colRecv, chromiumRecvProjector) + .AddColumn(Columns.colUrlPath, chromiumPathProjector) + .AddColumn(Columns.colAddr, chromiumAddressProjector) + .AddColumn(Columns.colPort, chromiumPortProjector) + .AddColumn(Columns.colSocket, chromiumSocketProjector) + .AddColumn(Columns.colSession, chromiumSessionProjector) + .AddColumn(Columns.colRequest, chromiumRequestProjector) + .AddColumn(Columns.colConnect, chromiumConnectProjector) +#if DEBUG + .AddColumn(Columns.colIDs, chromiumIDProjector) + .AddColumn(Columns.colSourceDeps, chromiumSourceDepProjector) +#endif // DEBUG + .AddColumn(Columns.colTransport, chromiumTransportProjector) + .AddColumn(Columns.colStatus, chromiumStatusProjector) + .AddColumn(Columns.colError, chromiumErrorProjector) + ; + + // this.Sources.Release(); + } // Build + } // NetBlameTableChromium +} diff --git a/src/NetBlame/Tables/NetBlameTable.ChromiumStreams.cs b/src/NetBlame/Tables/NetBlameTable.ChromiumStreams.cs new file mode 100644 index 0000000..550e5dd --- /dev/null +++ b/src/NetBlame/Tables/NetBlameTable.ChromiumStreams.cs @@ -0,0 +1,429 @@ +// Copyright(c) Microsoft Corporation. +// Licensed under the MIT License. + +#if AUX_TABLES +#if DEBUG // most valuable for debugging + +using System; + +using Microsoft.Performance.SDK; +using Microsoft.Performance.SDK.Processing; + +using TimestampUI = Microsoft.Performance.SDK.Timestamp; +using QWord = System.UInt64; + + +namespace NetBlameCustomDataSource.Tables +{ + [Table] + public sealed class NetBlameTableChromiumStreams : NetBlameTableBase + { + public NetBlameTableChromiumStreams(PendingSources sources, AllTables tables, IApplicationEnvironment environ) : base(sources, tables, environ) { } + + public static TableDescriptor TableDescriptor => new TableDescriptor( + new Guid("9f63dd43-7265-45d9-8c2b-24872f039e2e"), // The GUID must be unique across all tables. + "NetBlame Chromium Streams", // The Table must have a name. + "NetBlame Network Analyzer - Chromium Streams", // The Table must have a description. + "Network", // Optional category for grouping different types of tables in WPA UI. + false, // Not Metadata + TableLayoutStyle.GraphAndTable // .Table // Chart & Table or Table Only + ); + + + // Declare columns here. You can do this using the ColumnConfiguration class. + // It is possible to declaratively describe the table configuration as well. Please refer to our Advanced Topics Wiki page for more information. + // + // The Column metadata describes each column in the table. + // Each column must have a unique GUID and a unique name. The GUID must be unique globally; the name only unique within the table. + static class Columns + { + public static readonly ColumnConfiguration colProcessId = + DeclareColumn + ( + "PID", + "Process ID", + width: 56, + align: TextAlignment.Right, + format: ColumnFormats.NumberFormat, + visible: true + ); + + public static readonly ColumnConfiguration colThreadId = + DeclareColumn + ( + "TID", + "Thread ID", + width: 56, + align: TextAlignment.Right, + format: ColumnFormats.NumberFormat, + visible: true + ); + + public static readonly ColumnConfiguration colType = + DeclareColumn + ( + "Type", + "Session Type: QUIC / HTTP2 / HTTP1", + width: 58, + visible: true + ); + + public static readonly ColumnConfiguration colDomain = + DeclareColumn + ( + "Domain", + "Base Server Name", + width: 180, + visible: true + ); + + public static readonly ColumnConfiguration colStreamID = + DeclareColumn + ( + "#", + "Stream ID Number within the Session", + width: 34, + align: TextAlignment.Right, // number + visible: true + ); + + public static readonly ColumnConfiguration colSessionID = + DeclareColumn + ( + "Session ID", + "ID of the Session creation event", + width: 150, + align: TextAlignment.Right, // number + format: ColumnFormats.NumberFormat, + visible: false + ); + + public static readonly ColumnConfiguration colSessionSourceID = + DeclareColumn + ( + "Src ID", + "Session: source_dependency:id", + width: 48, + align: TextAlignment.Right, // number + visible: false + ); + + public static readonly ColumnConfiguration colRequestID = + DeclareColumn + ( + "Request ID", + "Stream's Chromium Request ID (reusable)", + width: 148, + align: TextAlignment.Right, + format: ColumnFormats.NumberFormat, + visible: false + ); + + public static readonly ColumnConfiguration colAnonKey = + DeclareColumn + ( + "Origin Context", + "Network Anonymization Key", + width: 180, + visible: false + ); + + public static readonly ColumnConfiguration colMethod = + DeclareColumn + ( + "Method", + "HTTP Method", + width: 82, + visible: true + ); + + public static readonly ColumnConfiguration colUrlPath = + DeclareColumn + ( + "URL", + "Full Url Path", + width: 2000, + visible: true + ); + + public static readonly ColumnConfiguration colSend = + DeclareColumn + ( + "Send (B)", + "Bytes Sent (estimted)", + mode: AggregationMode.Sum, + width: 70, + align: TextAlignment.Right, // 1 number + format: ColumnFormats.NumberFormat, + visible: true + ); + + public static readonly ColumnConfiguration colRecv = + DeclareColumn + ( + "Recv (B)", + "Bytes Received (estimated)", + mode: AggregationMode.Sum, + width: 70, + align: TextAlignment.Right, // 1 number + format: ColumnFormats.NumberFormat, + visible: true + ); + + public static readonly ColumnConfiguration colAddr = + DeclareColumn + ( + "IP Address", + "Remote IP Address", + width: 118, + visible: true + ); + + public static readonly ColumnConfiguration colWSSocket = + DeclareColumn + ( + "Socket", + "Winsock Socket ID = Local IP Address Port", + width: 54, + align: TextAlignment.Right, // 1 number + visible: false + ); + + public static readonly ColumnConfiguration colPort = + DeclareColumn + ( + "Port", + "Remote IP Address Port:\r\n80=http, 443=https, etc.", + width: 38, + align: TextAlignment.Center, + visible: true + ); + + public static readonly ColumnConfiguration colStatus = + DeclareColumn + ( + "Status", + "HTTP Status Number", + width: 46, + visible: false + ); + + public static readonly ColumnConfiguration colError = + DeclareColumn + ( + "Error", + "Error value or condition", + width: 60, + visible: false + ); + + public static readonly ColumnConfiguration colFirstTime = + NetBlameTableBase.DeclareColumn + ( + "First Time", + "The Stream's SEND HEADERS event time", + width: 102, + mode: AggregationMode.Min, + align: TextAlignment.Right, + format: TimestampFormatter.FormatSecondsGrouped, + visible: true + ); + + public static readonly ColumnConfiguration colLastTime = + NetBlameTableBase.DeclareColumn + ( + "Last Time", + "The Stream's last SEND or RECEIVE event time", + width: 102, + mode: AggregationMode.Min, + align: TextAlignment.Right, + format: TimestampFormatter.FormatSecondsGrouped, + visible: true + ); + +/* + public static readonly ColumnConfiguration colNAME = + DeclareColumn + ( + "NAME", + "DESC", + width: 180, + visible: true + ); +*/ + } // Columns + + + // Generators for use in Projections + // Static functions are more efficient than non-static, inline lambdas. + // None of these should return null. + static class Generators + { + public static string IStream(StreamEntry stream) => (stream.stream_id >= 0) ? stream.stream_id.ToString() : string.Empty; + + public static QWord SessionID(StreamEntry stream) => stream.id; + + public static int SourceID(StreamEntry stream) => stream.source_id; + + public static QWord RequestID(StreamEntry stream) => stream.request_id; + + public static int PID(StreamEntry stream) => stream.pid; + + public static int TID(StreamEntry stream) => stream.tid; + + public static string Type(StreamEntry stream) => stream.type; + + public static string Domain(StreamEntry stream) => stream.domain; + + public static string UrlPath(StreamEntry stream) => stream.url; + + public static string AnonKey(StreamEntry stream) => Chromium.Util.ScrubAnonKey(stream.anon_key); + + public static string Method(StreamEntry stream) => stream.method; + + public static string Status(StreamEntry stream) => stream.status; + + public static string Error(StreamEntry stream) => stream.error; + + public static string IPAddress(StreamEntry stream) => stream.addrRemote.ToGraphable(); + + public static uint WSSocket(StreamEntry stream) => stream.socket; + + public static uint Port(StreamEntry stream) => stream.port; + + public static uint Send(StreamEntry stream) => stream.cbSend; + + public static uint Recv(StreamEntry stream) => stream.cbRecv; + + public static TimestampUI FirstTime(StreamEntry stream) => stream.timeFirst; + public static TimestampUI LastTime(StreamEntry stream) => stream.timeLast; + } // Generators + + + public override void Build(ITableBuilder tableBuilder) + { + // Implement your columns here. + // Columns are implemented via Projections, which are simply functions that map a row index to a data point. + // Create projection for each column by composing the base projection with another projection that maps to the data point as needed. + + var tableBase = this.Tables?.streamTable; + + if (tableBase == null) return; + + var streamBaseProjector = Projection.Index(tableBase); + + // int -> string: Type, Domain, URL, Key, Method, IPAddress + var streamTypeProjector = Projection.Project(streamBaseProjector, Generators.Type); + var streamDomainProjector = Projection.Project(streamBaseProjector, Generators.Domain); + var streamURLProjector = Projection.Project(streamBaseProjector, Generators.UrlPath); + var streamAnonKeyProjector = Projection.Project(streamBaseProjector, Generators.AnonKey); + var streamMethodProjector = Projection.Project(streamBaseProjector, Generators.Method); + var streamStatusProjector = Projection.Project(streamBaseProjector, Generators.Status); + var streamErrorProjector = Projection.Project(streamBaseProjector, Generators.Error); + var streamAddressProjector = Projection.Project(streamBaseProjector, Generators.IPAddress); + + // int -> int: PID, TID, stream_id + var streamPIDProjector = Projection.Project(streamBaseProjector, Generators.PID); + var streamTIDProjector = Projection.Project(streamBaseProjector, Generators.TID); + var streamIDProjector = Projection.Project(streamBaseProjector, Generators.IStream); + var streamSourceIDProjector = Projection.Project(streamBaseProjector, Generators.SourceID); + var streamRequestIDProjector = Projection.Project(streamBaseProjector, Generators.RequestID); + + // int -> uint: Send, Recv, port, socket + var streamSendProjector = Projection.Project(streamBaseProjector, Generators.Send); + var streamRecvProjector = Projection.Project(streamBaseProjector, Generators.Recv); + var streamWSSocketProjector = Projection.Project(streamBaseProjector, Generators.WSSocket); + var streamPortProjector = Projection.Project(streamBaseProjector, Generators.Port); + + // int -> QWord + var streamUIDProjector = Projection.Project(streamBaseProjector, Generators.SessionID); + + // int -> TimestampUI + var streamFirstTimeProjector = Projection.Project(streamBaseProjector, Generators.FirstTime); + var streamLastTimeProjector = Projection.Project(streamBaseProjector, Generators.LastTime); + + + // Table Configurations describe how your table should be presented to the user: + // the columns to show, what order to show them, which columns to aggregate, and which columns to graph. + // You may provide a number of columns in your table, but only want to show a subset of them by default so as not to overwhelm the user. + // The user can still open the table properties in WPA to turn on or off columns. + // The table configuration class also exposes four (4) columns that WPA explicitly recognizes: Pivot Column, Graph Column, Left Freeze Column, Right Freeze Column + // For more information about what these columns do, go to "Advanced Topics" -> "Table Configuration" in our Wiki. Link can be found in README.md + + var config = new TableConfiguration("Chromium Stream Info") + { + Columns = new[] + { + Columns.colProcessId, + Columns.colThreadId, + Columns.colType, + Columns.colDomain, + TableConfiguration.PivotColumn, /*------------*/ + Columns.colStreamID, + Columns.colMethod, + Columns.colAnonKey, + Columns.colSend, + Columns.colRecv, + Columns.colAddr, + Columns.colPort, + Columns.colWSSocket, + Columns.colSessionID, + Columns.colRequestID, + Columns.colSessionSourceID, + Columns.colStatus, + Columns.colError, + Columns.colUrlPath, // Last, Widest + TableConfiguration.RightFreezeColumn, /*------*/ + TableConfiguration.GraphColumn, /*------------*/ + Columns.colFirstTime, + Columns.colLastTime + } + }; + +#if !DEBUG +/* + When open/close timestamps are given this meta-data, zeros get eliminated. +*/ + // Advanced Settings / Graph Configuration + config.AddColumnRole(ColumnRole.StartTime, colFirstTime.Metadata.Guid); + config.AddColumnRole(ColumnRole.EndTime, colLastTime.Metadata.Guid); +#endif // !DEBUG +// config.AddColumnRole(ColumnRole.Duration, colDuration.Metadata.Guid); + + // Use the table builder to build the table. + // Add and set table configuration if applicable. + // Then set the row count and then add the columns using AddColumn. + + tableBuilder + .AddTableConfiguration(config) + .SetDefaultTableConfiguration(config) + .SetRowCount(tableBase.Count) + .AddColumn(Columns.colProcessId, streamPIDProjector) + .AddColumn(Columns.colThreadId, streamTIDProjector) + .AddColumn(Columns.colType, streamTypeProjector) + .AddColumn(Columns.colDomain, streamDomainProjector) + .AddColumn(Columns.colStreamID, streamIDProjector) + .AddColumn(Columns.colSessionID, streamUIDProjector) + .AddColumn(Columns.colSessionSourceID, streamSourceIDProjector) + .AddColumn(Columns.colRequestID, streamRequestIDProjector) + .AddColumn(Columns.colUrlPath, streamURLProjector) + .AddColumn(Columns.colAnonKey, streamAnonKeyProjector) + .AddColumn(Columns.colMethod, streamMethodProjector) + .AddColumn(Columns.colStatus, streamStatusProjector) + .AddColumn(Columns.colError, streamErrorProjector) + .AddColumn(Columns.colWSSocket, streamWSSocketProjector) + .AddColumn(Columns.colPort, streamPortProjector) + .AddColumn(Columns.colAddr, streamAddressProjector) + .AddColumn(Columns.colSend, streamSendProjector) + .AddColumn(Columns.colRecv, streamRecvProjector) + .AddColumn(Columns.colFirstTime, streamFirstTimeProjector) + .AddColumn(Columns.colLastTime, streamLastTimeProjector) + ; + + // this.Sources.Release(); + } // Build + } // NetBlameTableChromium +} // NetBlameCustomDataSource.Tables + +#endif // DEBUG +#endif // AUX_TABLES diff --git a/src/WPAP/ChromiumRegions.wpaProfile b/src/WPAP/ChromiumRegions.wpaProfile new file mode 100644 index 0000000..d85aec5 --- /dev/null +++ b/src/WPAP/ChromiumRegions.wpaProfile @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + \ No newline at end of file diff --git a/src/WPAP/EDGE/Chromium-Request.regions.xml b/src/WPAP/EDGE/Chromium-Request.regions.xml new file mode 100644 index 0000000..4ad19c0 --- /dev/null +++ b/src/WPAP/EDGE/Chromium-Request.regions.xml @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From e4e4e46b26210a3d6e0f832ff7409104a780d22e Mon Sep 17 00:00:00 2001 From: Raymond Fowkes Date: Thu, 23 Jul 2026 15:46:01 -0700 Subject: [PATCH 3/3] NetBlame Chromium Processing --- src/BETA/TraceNetwork.ps1 | 2 +- src/NetBlame/.editorconfig | 9 + src/NetBlame/Auxiliary/CallStack.cs | 41 +- src/NetBlame/Auxiliary/Extensions.cs | 12 +- src/NetBlame/Auxiliary/NetUtil.cs | 215 +- src/NetBlame/GatherTables.cs | 332 +- src/NetBlame/NetBlameDataProcessor.cs | 30 +- src/NetBlame/Providers/Chromium.cs | 6035 ++++++++++++++++- src/NetBlame/Providers/DNSClient.cs | 8 +- src/NetBlame/Providers/TcpIp.cs | 6 +- src/NetBlame/Providers/WinsockAFD.cs | 19 +- src/NetBlame/Tables/NetBlameTable.URL.cs | 21 +- .../Tables/NetBlameTable.WebIO.Request.cs | 5 +- src/NetBlame/Tables/NetBlameTable.Winsock.cs | 2 +- src/NetBlame/Tables/TableBase.cs | 2 +- src/TraceNetwork.ps1 | 2 +- src/WPRP/EdgeChrome.15002.wprp | 132 +- src/WPRP/EdgeChrome.wprp | 54 +- 18 files changed, 6682 insertions(+), 245 deletions(-) diff --git a/src/BETA/TraceNetwork.ps1 b/src/BETA/TraceNetwork.ps1 index 75d5734..bf6bc71 100644 --- a/src/BETA/TraceNetwork.ps1 +++ b/src/BETA/TraceNetwork.ps1 @@ -130,7 +130,7 @@ Param( @{ # The configuration files define the data tabs in the WPA viewer. # https://learn.microsoft.com/en-us/windows-hardware/test/wpt/view-profiles - ViewerConfig = "..\WPAP\BasicInfo.wpaProfile", "..\WPAP\EdgeRegions.wpaProfile", ".\WPAP\Network.wpaProfile" + ViewerConfig = "..\WPAP\BasicInfo.wpaProfile", ".\WPAP\Network.wpaProfile" # The trace file name is: .etl TraceName = $TraceParams.InstanceName diff --git a/src/NetBlame/.editorconfig b/src/NetBlame/.editorconfig index f08d38e..41d2752 100644 --- a/src/NetBlame/.editorconfig +++ b/src/NetBlame/.editorconfig @@ -2,3 +2,12 @@ # IDE0024: Use block body for operators csharp_style_expression_bodied_operators = false:silent + +# Flag methods that can be static as warnings +# dotnet_diagnostic.CA1822.severity = warning + +# Require 'this.' for methods and show a warning if it is missing +# dotnet_style_qualification_for_method = true:warning +# dotnet_style_qualification_for_field = true:warning +# dotnet_style_qualification_for_property = true:warning + diff --git a/src/NetBlame/Auxiliary/CallStack.cs b/src/NetBlame/Auxiliary/CallStack.cs index 7a51a20..79e5bb6 100644 --- a/src/NetBlame/Auxiliary/CallStack.cs +++ b/src/NetBlame/Auxiliary/CallStack.cs @@ -56,7 +56,7 @@ Office Idle Manager namespace NetBlameCustomDataSource.Stack { static class PID { public const IDVal Unknown = -1; } // Process ID - static class TID { public const IDVal Unknown = -1; } // Thrread ID + static class TID { public const IDVal Unknown = -1; } // Thread ID static class PROC { public const IDVal Unknown = -1; } // Processor ID public class MyStackSnapshot : IStackSnapshot @@ -344,7 +344,11 @@ protected static bool IsEmpty(MyStackSnapshot stack) return Array.TrueForAll(stack.rgStack, s => s?.Frames == null); } - public bool IsNull(IStackSnapshot stack) => IsEmpty((MyStackSnapshot)stack); + public bool IsNull(IStackSnapshot stack) + { + MyStackSnapshot myStack = stack as MyStackSnapshot; + return (myStack != null) ? IsEmpty(myStack) : GetFrameCount(stack) == 0; + } protected static int CountAll(MyStackSnapshot stack) => stack?.rgStack?.Length ?? 0; @@ -394,11 +398,18 @@ public string _GetValue(IStackSnapshot stack, in MyStackSnapshot.Attributes attr // Invoked for Last Stack public int GetCount(IStackSnapshot stack) { - MyStackSnapshot myStack = (MyStackSnapshot)stack; + MyStackSnapshot myStack = stack as MyStackSnapshot; - AssertCritical(myStack?.StackLast == null || Equals(stack, myStack.StackLast)); + if (myStack != null) + { + AssertCritical(myStack.StackLast == null || Equals(stack, myStack.StackLast)); - if (IsEmpty(myStack)) return 0; + if (IsEmpty(myStack)) return 0; + } + else + { + if (GetFrameCount(stack) == 0) return 0; + } int count = _GetCount(stack); #if StackTitle @@ -410,17 +421,27 @@ public int GetCount(IStackSnapshot stack) return count; } + static readonly MyStackSnapshot.Attributes s_attrib = default; + // Invoked for Last Stack public string GetValue(IStackSnapshot stack, int index) { - MyStackSnapshot myStack = (MyStackSnapshot)stack; + MyStackSnapshot myStack = stack as MyStackSnapshot; - AssertCritical(myStack?.StackLast == null || Equals(stack, myStack.StackLast)); + if (myStack != null) + { + AssertCritical(myStack.StackLast == null || Equals(stack, myStack.StackLast)); - if (IsEmpty(myStack)) return PastEndValue; + if (IsEmpty(myStack)) return PastEndValue; - return _GetValue(stack, in ((MyStackSnapshot)stack).rgAttrib[^1], index); + return _GetValue(stack, in myStack.rgAttrib[^1], index); + } + else + { + if (GetFrameCount(stack) == 0) return PastEndValue; + return _GetValue(stack, in s_attrib, index); + } } @@ -666,8 +687,6 @@ public FullStackSnapshotAccessProvider(SymLoadProgress _symLoadProgress) : base( public new string GetValue(IStackSnapshot stack, int index) { - AssertImportant(stack != null); - MyStackSnapshot myStack = (MyStackSnapshot)stack; if (IsEmpty(myStack)) return PastEndValue; diff --git a/src/NetBlame/Auxiliary/Extensions.cs b/src/NetBlame/Auxiliary/Extensions.cs index 8a91dab..426eab9 100644 --- a/src/NetBlame/Auxiliary/Extensions.cs +++ b/src/NetBlame/Auxiliary/Extensions.cs @@ -42,10 +42,14 @@ public static class Extensions public static string ToGraphable(this IPAddress ipAddr) => ipAddr?.ToString() ?? String.Empty; - public static string ToGraphable(this IPEndPoint ipEndPoint) => !ipEndPoint.Empty() ? ipEndPoint.ToString() : String.Empty; - public static string AddrGraphable(this IPEndPoint ipEndPoint) => !ipEndPoint.Empty() ? ipEndPoint.Address.ToString() : String.Empty; + public static string ToGraphable(this IPEndPoint ipEndPoint) => !ipEndPoint.Empty() ? ((ipEndPoint.Port != 0) ? ipEndPoint.ToString() : ipEndPoint.AddrGraphable()) : String.Empty; + public static string AddrGraphable(this IPEndPoint ipEndPoint) => !(ipEndPoint?.Address).Empty() ? ipEndPoint.Address.ToString() : String.Empty; public static uint PortGraphable(this IPEndPoint ipEndPoint) => (uint?)ipEndPoint?.Port ?? 0; + public static string ToStringOrBlank(this ushort val) => (val > 0) ? val.ToString() : string.Empty; + public static string ToStringOrBlank(this ushort? val) => (val > 0) ? val.ToString() : string.Empty; + public static string ToHexOrBlank(this uint val) => (val > 0) ? val.ToString("X") : string.Empty; + public static bool Empty(this SocketAddress socket) => (socket == null || socket[0]/*family*/ == 0); // SocketAddress.Equals may give false negatives when the socket contains random padding bytes! @@ -111,6 +115,8 @@ public static bool IsAddrZero(this SocketAddress socket) public static SocketAddress GetRemoteAddress(this IGenericEvent evt) => (evt.GetUInt32("RemoteAddressLength") != 0) ? evt.GetSocketAddress("RemoteAddress") : null; + public static bool Contains(this ReadOnlySpan span, string substr) => span.IndexOf(substr) >= 0; + // Simple hash of all addresses in the stack. Returns 0 if none. public static int Hash(this IStackSnapshot ss) { @@ -122,6 +128,6 @@ public static int Hash(this IStackSnapshot ss) return hash; } - } + } // Extensions } // NetBlameCustomDataSource.Events diff --git a/src/NetBlame/Auxiliary/NetUtil.cs b/src/NetBlame/Auxiliary/NetUtil.cs index cc055a9..41bce01 100644 --- a/src/NetBlame/Auxiliary/NetUtil.cs +++ b/src/NetBlame/Auxiliary/NetUtil.cs @@ -19,11 +19,12 @@ public enum Protocol : byte // Order of increasing priority Unknown = 0, // anomalous Rundown = 1, // preexisting connection - TCP = 2, - UDP = 4, + TCP = 2, + UDP = 4, Winsock = 8, WinINet = 16, - WinHTTP = 32 // WebIO + WinHTTP = 32, // WebIO + Chromium = 64 }; public static class Util @@ -65,6 +66,9 @@ public static void AssertCritical(bool c) } } + [Conditional("DEBUG")] + public static void DEBUG(T e) { } + #if DEBUG // Must remove references to build RELEASE. [DebuggerStepThrough()] @@ -83,8 +87,6 @@ public static void BreakWhen(bool c) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool FImplies(bool a, bool b) => !a || b; - public static uint BitFromI(uint i) => (uint)1 << ((int)i - 1); // i: 1-based - // https://github.com/dotnet/runtime/issues/58378 public static AddressFamily AF_HYPERV = (AddressFamily)34; public static AddressFamily AF_VSOCK = (AddressFamily)40; @@ -177,32 +179,20 @@ public static IPEndPoint NewEndPoint(in Microsoft.Windows.EventTracing.Events.IG } - static public readonly char[] rgchURLSplit = new char[] { ':', '/' }; static public readonly char[] rgchEOLSplit = new char[] { '\r', '\n' }; - /* - Return the string "ServerName" from "http:// ServerName / Path" - Else return String.Empty (never null). - */ + static public string ServerNameFromURL(string strURL) { - if (strURL.IsNA()) - return String.Empty; - - // Skip past the "http://" - - string[] rgstrURL = strURL.Split(rgchURLSplit, StringSplitOptions.RemoveEmptyEntries); - - if (rgstrURL.Length == 0) - return String.Empty; - - if (rgstrURL.Length > 1 && rgstrURL[0].StartsWith("http", StringComparison.OrdinalIgnoreCase)) - return rgstrURL[1].ToLowerInvariant(); + if (!strURL.IsNA()) + { + if (Uri.TryCreate(strURL, UriKind.Absolute, out Uri uri)) + return uri.Host; + } - return rgstrURL[0].ToLowerInvariant(); + return string.Empty; } - /* Return true if the server part of the URL strings is the same (case insensitive). */ @@ -259,89 +249,46 @@ Name some common TCP/UDP ports. */ static public string ServiceFromPort(int port) { - switch (port) + return port switch { - case 20: - case 21: - return "FTP"; - case 22: - return "SSH/SCP/SFTP"; - case 23: - return "TELNET"; - case 25: - return "SMTP"; - case 53: - return "DNS"; - case 80: - case 8080: - case 8081: - return "HTTP"; - case 88: - return "Kerberos"; - case 110: - return "POP3"; - case 119: - return "NNTP"; - case 123: - return "NTP"; - case 135: - return "DCE/DHCP/DNS/WINS/DCOM"; - case 137: - return "NetBIOS"; - case 143: - return "IMAP"; - case portLDAP: // 389 - return "LDAP"; - case 443: - case 8443: - return "HTTPS"; - case 445: - return "AD/SMB"; - case 465: - return "SMTPS"; - case 546: - case 547: - return "DHCPv6"; - case 554: - return "RTSP"; - case 563: - return "NNTPS"; - case 636: - return "LDAPs"; - case 989: - case 990: - return "FTPS"; - case 993: - return "IMAPS"; - case 995: - return "POP3S"; - case 1433: - return "MSSQL"; - case 1900: - return "SSDP"; - case 2555: - case 2869: - case 5000: - return "UPnP"; - case 3268: - return "LDAP/AD"; - case 3269: - return "LDAPs/AD"; - case 3389: - return "TS/RDP"; - case 5353: - return "mDNS"; - case 5355: - return "LLMNR"; - case 5985: - return "CIM/DMTF"; - case 7680: - return "DeliveryOpt"; - case 8888: - return "HTTP/LocalHost"; - default: - return null; - } + 20 or 21 => "FTP", + 22 => "SSH/SCP/SFTP", + 23 => "TELNET", + 25 => "SMTP", + 53 => "DNS", + 80 or 8080 or 8081 => "HTTP", + 88 => "Kerberos", + 110 => "POP3", + 119 => "NNTP", + 123 => "NTP", + 135 => "DCE/DHCP/DNS/WINS/DCOM", + 137 => "NetBIOS", + 143 => "IMAP", + portLDAP => "LDAP", // 389 + 443 or 8443 => "HTTPS", + 445 => "AD/SMB", + 465 => "SMTPS", + 546 or 547 => "DHCPv6", + 554 => "RTSP", + 563 => "NNTPS", + 636 => "LDAPs", + 853 => "DNS/TLS", + 989 or 990 => "FTPS", + 993 => "IMAPS", + 995 => "POP3S", + 1433 => "MSSQL", + 1900 => "SSDP", + 2555 or 2869 or 5000 => "UPnP", + 3268 => "LDAP/AD", + 3269 => "LDAPs/AD", + 3389 => "TS/RDP", + 5353 => "mDNS", + 5355 => "LLMNR", + 5985 => "CIM/DMTF", + 7680 => "DeliveryOpt", + 8888 => "HTTP/LocalHost", + _ => null + }; } @@ -369,9 +316,39 @@ static public string ComposeMethod(this WinsockAFD.Connection cxn) return service; } + static class IPSpecial + { + const string strGoogleDNS = "Google DNS"; + const string strCFlareDNS = "Cloudflare DNS"; + const string strQuad9DNS = "Quad9 DNS"; + const string strOpenDNS = "OpenDNS"; + + private static readonly System.Collections.Generic.Dictionary _map = + new() + { + { IPAddress.Parse("1.0.0.1"), strCFlareDNS }, + { IPAddress.Parse("1.1.1.1"), strCFlareDNS }, + { IPAddress.Parse("[2606:4700:4700::1001]"), strCFlareDNS }, + { IPAddress.Parse("[2606:4700:4700::1111]"), strCFlareDNS }, + { IPAddress.Parse("8.8.4.4"), strGoogleDNS}, + { IPAddress.Parse("8.8.8.8"), strGoogleDNS}, + { IPAddress.Parse("[2001:4860:4860::8844]"), strGoogleDNS }, + { IPAddress.Parse("[2001:4860:4860::8888]"), strGoogleDNS }, + { IPAddress.Parse("9.9.9.9"), strQuad9DNS}, + { IPAddress.Parse("[2620:fe::fe]"), strQuad9DNS}, + { IPAddress.Parse("[2620:fe::9]"), strQuad9DNS}, + { IPAddress.Parse("208.67.222.222"), strOpenDNS }, + { IPAddress.Parse("[2620:0:ccc::2]"), strOpenDNS }, + { IPAddress.Parse("[2620:0:ccd::2]"), strOpenDNS }, + { IPAddress.Parse("255.255.255.255"), "Broadcast"} + }; + + public static string Get(IPAddress ip) => _map.TryGetValue(ip, out var v) ? v : null; + } + static public string AddressType(IPAddress addr) { - if (addr == null) + if (addr.Empty()) return null; if (addr.IsIPv4MappedToIPv6) @@ -398,16 +375,12 @@ static public string AddressType(IPAddress addr) if (bytes[1] == 168) return "Private Network (C)"; break; - case 255: - if (bytes[1] == 255 && bytes[2] == 255 && bytes[3] == 255) - return "Broadcast"; - break; default: if (bytes[0] >= 224 && bytes[0] < 240) return "Multicast"; break; } - return null; + return IPSpecial.Get(addr); // usually null } if ((int)addr.AddressFamily == 34) @@ -428,23 +401,7 @@ static public string AddressType(IPAddress addr) if (addr.IsIPv6UniqueLocal) return "Unique Local"; - return null; - } - - /* - qw.RotateLeft() - .NET Core 3:0 - System.Numerics.BitOperations.RotateLeft() - */ - static public void RotateLeft(ref this QWord ul) => ul = (ul << 1) | (ul >> 63); - - /* - Return the 1-based index of the lowest bit, else return 0. - */ - static public uint BitScanLeft(uint grbit) - { - uint i = 0; - for (grbit &= (uint)-grbit /*isolate low bit*/; grbit != 0; grbit >>= 1) ++i; - return i; + return IPSpecial.Get(addr); // usually null } } // class Util } diff --git a/src/NetBlame/GatherTables.cs b/src/NetBlame/GatherTables.cs index 43a2763..056e6ba 100644 --- a/src/NetBlame/GatherTables.cs +++ b/src/NetBlame/GatherTables.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Net; // IPAddress - using Microsoft.Windows.EventTracing.Symbols; // IStackSnapshot using NetBlameCustomDataSource.Link; @@ -99,6 +98,31 @@ public int Compare(ThreadPoolItem tp1, ThreadPoolItem tp2) } // TPCompare + public class StreamEntry + { + public string type; + public string url; + public string domain; + public string anon_key; + public string method; + public string origin; + public string referer; + public string error; // Socket.Status() + public string status; // HTTP Status + public ushort socket; // Socket.WSSocket() + public ushort port; + public uint cbSend, cbRecv; + public IDVal pid, tid; + public int stream_id; + public int source_id; + public QWord request_id; + public QWord id; + public IPEndPoint addrRemote; + public TimestampUI timeFirst; + public TimestampUI timeLast; + } + + /* This is the master table entry, accumulating data from all other tables. */ @@ -179,15 +203,17 @@ public URLTable(in AllTables _allTables) } - URL AddURL(string strURL, string strMethod, IDVal pid, IDVal tid, uint cbSend, uint cbRecv, in TcbRecord tcb, Protocol netType) + URL AddURL(string strURL, string strMethod, IDVal pid, IDVal tid, uint cbSend, uint cbRecv, TcbRecord tcb, Protocol netType) { if (tcb != null) + { + tcb.SetType(netType); tcb.fGathered = true; + } URL url = new URL(strURL, strMethod, pid, tid, in tcb, netType); this.Add(url); - // TODO: compare TCB, tcb.cbSend/Recv ? url.cbSend += cbSend; url.cbRecv += cbRecv; @@ -203,7 +229,7 @@ void AddURLConnection(WebIO.Request req, WebIO.Connection cxn) AssertCritical(tcbR == null || tcbR.pid == req.pid); } - URL url = AddURL(req.strURL, req.strMethod, req.pid, req.tidStack, cxn.cbSend, cxn.cbRecv, in tcbR, Protocol.WinHTTP); + URL url = AddURL(req.strURL, req.strMethod, req.pid, req.tidStack, cxn.cbSend, cxn.cbRecv, tcbR, Protocol.WinHTTP); url.strServer = this.allTables.dnsTable.GetServerNameAndAlt(tcbR?.addrRemote?.Address, req.strURL, req.strServer, out url.strServerAlt); url.strStatus = cxn.strHeader ?? String.Empty; url.qwConnection = cxn.qwConnection; @@ -242,7 +268,7 @@ void AddURL(WebIO.Request req) void AddURL(in WinINet.Request req, in TcbRecord tcb) { - URL url = AddURL(req.strURL, req.strMethod, req.pid, req.tid1, req.cbSend, req.cbRecv, in tcb, Protocol.WinINet); + URL url = AddURL(req.strURL, req.strMethod, req.pid, req.tid1, req.cbSend, req.cbRecv, tcb, Protocol.WinINet); url.strServer = this.allTables.dnsTable.GetServerNameAndAlt(req.addrRemote?.Address, req.strURL, req.strServerName, out url.strServerAlt); url.strStatus = req.Status; @@ -269,32 +295,170 @@ void AddURL(in WinINet.Request req, in TcbRecord tcb) #endif // DEBUG } + + void AddURL(Chromium.Request req) + { + AssertCritical(!req.IsSpeculative); + + Chromium.Socket socket = req.Session?.socket; + TcbRecord tcbr = this.allTables.tcpTable.TcbrFromI(socket?.cxn?.iTCB ?? 0); + + URL url = AddURL(req.URL, req.method, req.pid, req.tid, req.cbUpload, req.cbDownload, tcbr, Protocol.Chromium); + + url.strServer = req.Domain; + url.strServerAlt = req.Canon; + url.timeOpen = req.timeStampBeginJob; + url.timeClose = req.timeStampEndJob; + url.timeRef = req.timeRef; + url.qwRequest = req.uidRequest; + url.strStatus = req.Error(); + + ushort port = req.port; + + if (socket != null) + { + port = (ushort)socket.addrRemote.PortGraphable(); + url.qwConnection = socket.cxn?.qwEndpoint ?? 0; + url.dwSocket = socket.WSSocket(); // ushort + + socket.fGathered = true; + } + + IPAddress ipAddr = req.IPAddress(socket); + if (!ipAddr.Empty()) + url.ipAddrPort = new IPEndPoint(ipAddr, port); + + url.myStack = new MyStackSnapshot(in req.stack, in req.xlink, req.tid); // aggregated stacks +#if DEBUG + req.fGathered = true; +#endif // DEBUG + } + + void AddURL(Chromium.Session session, Chromium.Session.Stream stream) + { + AssertCritical(!stream.fAbandoned && !stream.fIgnore); + AssertInfo(stream.request != null); // handled, but... late trace start? + + Chromium.Request req = stream.request; + Chromium.Socket socket = session.socket; + WinsockAFD.Connection cxn = socket?.cxn; + TcbRecord tcbr = this.allTables.tcpTable.TcbrFromI(cxn?.iTCB ?? 0); + + URL url = null; + + if (req != null) + { + url = AddURL(req.URL, req.method, req.pid, req.tid, stream.CbSend(), stream.CbRecv(), tcbr, Protocol.Chromium); + + url.strServer = req.Domain; + url.strServerAlt = req.Canon; + url.timeOpen = req.timeStampBeginJob; + url.timeClose = req.timeStampEndJob; + url.timeRef = req.timeRef; + url.qwRequest = req.uidRequest; + + url.myStack = new MyStackSnapshot(in req.stack, in req.xlink, req.tid); // aggregated stacks + } + else + { + url = AddURL(stream.strURL, stream.strMethod, session.pid, session.tid, stream.CbSend(), stream.CbRecv(), tcbr, Protocol.Chromium); + + string strCanon = session.resolver?.rgstrCanon?[0] ?? string.Empty; + url.strServerAlt = !strCanon.Equals(session.domain) ? strCanon : string.Empty; + url.strServer = session.domain; + url.timeRef = session.timeReference; + url.timeOpen = stream.timeFirst; + url.timeClose = stream.timeLast; + } + + url.strStatus = stream.strHTTPStatus; + + IPEndPoint ipEP = socket?.addrRemote; + AssertImportant(FImplies(ipEP.Empty(), cxn?.addrRemote == null)); + + if (ipEP.Empty()) + { + IPAddress ipAddr = req?.IPAddress(socket); + ushort port = (req?.port > session.port) ? req.port : session.port; + if (!ipAddr.Empty()) + ipEP = new IPEndPoint(ipAddr, port); + } + url.ipAddrPort = ipEP; + + if (socket != null) + { + url.qwConnection = cxn?.qwEndpoint ?? 0; + url.dwSocket = socket.WSSocket(); // ushort + + socket.fGathered = true; + } + } + + // This Socket is attached to no Request. + void AddURL(Chromium.Socket socket) + { + WinsockAFD.Connection cxn = socket.cxn; + AssertImportant(cxn != null); + if (cxn == null) + return; + + TcbRecord tcbr = this.allTables.tcpTable.TcbrFromI(cxn.iTCB); + + URL url = AddURL(string.Empty, Util.ComposeMethod(cxn), socket.pid, socket.tid, cxn.cbSend, cxn.cbRecv, tcbr, Protocol.Chromium); + + url.timeOpen = socket.timeStampConnect; + url.timeClose = socket.timeStampClose; + url.timeRef = cxn.timeRef; + url.qwConnection = cxn.qwEndpoint; + url.dwSocket = socket.WSSocket(); // ushort + + AssertImportant(socket.addrRemote.Equals(cxn.addrRemote)); + IPEndPoint ipEP = socket.addrRemote; + if (ipEP.Empty()) + ipEP = cxn.addrRemote; + + url.ipAddrPort = ipEP; + + if (!ipEP.Empty() && !ipEP.Address.Empty()) + url.strServer = this.allTables.dnsTable.GetServerNameAndAlt(ipEP.Address, null, strNA, out url.strServerAlt); + + url.myStack = new MyStackSnapshot(in cxn.stack, in cxn.xlink, cxn.tidOpen); // aggregated stacks + + socket.fGathered = true; + } + void AddURL(WinsockAFD.Connection cxn) { TcbRecord tcb = this.allTables.tcpTable.TcbrFromI(cxn.iTCB); // WinSock is a mid-level protocol. This TCB may have already been 'gathered' by a higher level protocol. - if (tcb?.fGathered ?? false) + if (tcb == null) + { + if (cxn.Protocol > Protocol.Winsock) + return; + } + else if (tcb.fGathered) { - AssertImportant(Prominent((Protocol)cxn.grbitType) > Protocol.Winsock); + AssertImportant(cxn.Protocol > Protocol.Winsock); return; } - AssertImportant(Prominent((Protocol)cxn.grbitType) == Protocol.Winsock); + // Any Connection with another protocol should have already been 'gathered'. + AssertImportant(cxn.Protocol == Protocol.Winsock); - URL url = AddURL(null, Util.ComposeMethod(cxn), cxn.pid, cxn.tidOpen, cxn.cbSend, cxn.cbRecv, in tcb, Protocol.Winsock); + URL url = AddURL(null, Util.ComposeMethod(cxn), cxn.pid, cxn.tidOpen, cxn.cbSend, cxn.cbRecv, tcb, Protocol.Winsock); uint iAddr = this.allTables.dnsTable.IFindAddress(cxn.iDNS, uint.MaxValue, cxn.addrRemote.Address); // 1-based url.strServer = this.allTables.dnsTable.GetServerNameAndAlt(cxn.iDNS, iAddr, null, strNA, out url.strServerAlt); url.timeOpen = cxn.timeCreate; url.timeClose = cxn.timeClose; url.timeRef = cxn.timeRef; - url.qwRequest = cxn.qwEndpoint; + AssertCritical(url.TimeRef.HasValue); + url.qwConnection = cxn.qwEndpoint; url.dwSocket = cxn.socket; // ushort - url.strStatus = cxn.status.ToString("X"); + url.strStatus = cxn.status.ToHexOrBlank(); url.xlink = cxn.xlink; url.myStack = new MyStackSnapshot(in cxn.stack, in cxn.xlink, cxn.tidOpen); - AssertCritical(url.TimeRef.HasValue); if (tcb == null) { @@ -330,7 +494,7 @@ void AddURL(TcbRecord tcb) } } - URL url = AddURL(null, strMethod, tcb.Pid, tcb.TidOpen, tcb.cbSend, tcb.cbRecv, in tcb, Prominent((Protocol)tcb.grbitProtocol)); + URL url = AddURL(null, strMethod, tcb.Pid, tcb.TidOpen, tcb.cbSend, tcb.cbRecv, tcb, Prominent((Protocol)tcb.grbitProtocol)); url.strServer = this.allTables.dnsTable.GetServerNameAndAlt(tcb.addrRemote?.Address, null, strNA, out url.strServerAlt); @@ -406,6 +570,79 @@ void GatherWebIO() } + /* + Finalize the Chromium records. + Incorporate into the URL table. + */ + void GatherChromium() + { + TimestampUI timeStampEnd = this.allTables.traceMetadata.LastEventTime.ToGraphable(); + + foreach (var session in this.allTables.chromiumTable.sessionTable) + { + if (session.FMigrated) + session.AdjustForMigration(this.allTables.chromiumTable); + + IPEndPoint _addrRemote = session.RemoteAddress(); + + foreach (var kvp in session.rgStream) + { + var stream = kvp.Value; + AssertImportant(stream != null); + if (stream == null) continue; + + if (!session.ValidStream(stream)) continue; + + if (stream.request != null) + { + AssertCritical(stream.request.FAttachedToStream); // else the Request shows up twice + AssertCritical(FImplies(!session.fPreMigrate, stream.request.Session == session)); + stream.request.Close(session.resolver, this.allTables.dnsTable, in timeStampEnd); +#if DEBUG + stream.request.fGathered = true; +#endif // DEBUG + } + + AddURL(session, stream); + } +#if DEBUG + session.fGathered = true; +#endif // DEBUG + } // foreach session + + // Now add incomplete Requests not added above (not linked to a Stream). + foreach (var req in this.allTables.chromiumTable) + { +#if DEBUG + AssertCritical(req.fGathered == (req.stream != null)); +#endif // DEBUG + if (req.stream != null) continue; + + // REVIEW: Should these Speculative Preconnect placeholder Requests be removed from the list for the Chromium Requests table? + if (req.IsSpeculative) continue; + + // It should be only incomplete or preconnect placeholder Requests (or some error conditions) that have no Stream. + AssertImportant(req.Type == Chromium.StreamType.Unknown || req.IsPreconnect || req.iError != 0 || req.fCanceled || req.fRedirect); + + req.Close(in timeStampEnd); + + AddURL(req); + } // foreach req + + // Gather the Sockets not enumerated with the Requests above. + + foreach (Chromium.Socket sock in this.allTables.chromiumTable.socketTable) + { + sock.Close(timeStampEnd); + + if (!sock.fGathered) + AddURL(sock); + } + + this.allTables.chromiumTable.EmitViewProfileAnnotationQueriesDB(); + } // GatherChromium + + void GatherWinSock() { var wsTable = this.allTables.wsTable; @@ -537,12 +774,81 @@ public void GatherThreadPools() } + /* + Populate the Chromium Streams table. + */ + [Conditional("AUX_TABLES")] + void GatherChromiumStreams() + { + int count = 0; + foreach (var session in this.allTables.chromiumTable.sessionTable) + count += session.rgStream.Count; + + var streamTable = new List(count); + + foreach (var session in this.allTables.chromiumTable.sessionTable) + { + if (session.rgStream.Count == 0) continue; + + IPEndPoint _addrRemote = session.RemoteAddress(); + + string _anon_key = (session.anon_key != null) ? session.anon_key : session.resolver?.anon_key; + + string _type = session.Type.ToString(); + + ushort _socket = session.socket?.WSSocket() ?? 0; + + foreach (var kvp in session.rgStream) + { + var stream = kvp.Value; + AssertImportant(stream != null); + if (stream == null) continue; + + if (!session.ValidStream(stream)) continue; + + StreamEntry entry = new StreamEntry() + { + stream_id = kvp.Key, + error = session.Error(stream), + domain = session.domain, + port = session.port, + pid = session.pid, + tid = session.tid, + id = session.uidVal, + source_id = session.srcdep, + url = stream.strURL, + method = stream.strMethod, + origin = stream.strOrigin, + referer = stream.strReferer, + cbSend = stream.CbSend(), + cbRecv = stream.CbRecv(), + status = stream.strHTTPStatus, + request_id = stream.request?.uidRequest ?? 0, + timeFirst = stream.timeFirst, + timeLast = stream.timeLast, + addrRemote = _addrRemote, + socket = _socket, + anon_key = _anon_key, + type = _type + }; + + streamTable.Add(entry); + } + } + + this.allTables.streamTable = streamTable; + } + + public void GatherAll() { GatherDNS(); GatherThreadPools(); + GatherChromium(); + GatherChromiumStreams(); + GatherWinINet(); GatherWebIO(); diff --git a/src/NetBlame/NetBlameDataProcessor.cs b/src/NetBlame/NetBlameDataProcessor.cs index 32f5e6f..f2bfbca 100644 --- a/src/NetBlame/NetBlameDataProcessor.cs +++ b/src/NetBlame/NetBlameDataProcessor.cs @@ -208,6 +208,7 @@ public class AllTables public ODispatchQTable odqTable; public IdleManTable idleTable; public ThreadTable threadTable; + public ChromiumTable chromiumTable; // Final Aggregated Results public URLTable urlTable; @@ -216,14 +217,18 @@ public class AllTables public List dnsIndexTable; // References to all threadpool objects: Office TP, WinHTTP TP, Windows TP, Windows Timers public List tpTable; + // Aggegation of Chromium Streams + public List streamTable; #endif // AUX_TABLES // Get a stack for a Classic event: stackSource?.GetStack(timestampETW, tid); public IStackDataSource stackSource; + public ITraceMetadata2 traceMetadata; + public AllTables() { - // TODO: smart capacity in each of these? + // empirical table capacity values // DNS Info Tables this.dnsTable = new DNSTable(64); this.wsName = new WinsockNameResolution(this); @@ -232,6 +237,7 @@ public AllTables() this.tcpTable = new TcpTable(1024, this); this.webioTable = new WebIOTable(512, this); this.winetTable = new WinINetTable(128, this); + this.chromiumTable = new ChromiumTable(4096, this); // Thread/TaskPool Tables this.httpTable = new WinHttpTable(4096, this); this.wtpTable = new WThreadPoolTable(1024, this); @@ -383,13 +389,13 @@ public override DataSourceInfo GetDataSourceInfo() WinINetTable.guid, // Microsoft-Windows-WinINet WinHttpTable.guid, // Microsoft-Windows-WinHttp WebIOTable.guid, // Microsoft-Windows-WebIO - // For the browser's DNS records: - Chromium.DNSInfo.rgGuid[0], // Microsoft.MSEdgeStable - Chromium.DNSInfo.rgGuid[1], // Microsoft.MSEdgeBeta - Chromium.DNSInfo.rgGuid[2], // Microsoft.MSEdgeWebView2 - Chromium.DNSInfo.rgGuid[3], // Microsoft.MSEdgeCanary - Chromium.DNSInfo.rgGuid[4], // Microsoft.MSEdgeDev - Chromium.DNSInfo.rgGuid[5] // CHROME + // For the Chromium events and their DNS records: + ChromiumTable.rgGuid[0], // Microsoft.MSEdgeStable + ChromiumTable.rgGuid[1], // Microsoft.MSEdgeBeta + ChromiumTable.rgGuid[2], // Microsoft.MSEdgeWebView2 + ChromiumTable.rgGuid[3], // Microsoft.MSEdgeCanary + ChromiumTable.rgGuid[4], // Microsoft.MSEdgeDev + ChromiumTable.rgGuid[5] // Google.Chrome }; // These providers will get processed in the ClassicEventConsumer callback. @@ -400,7 +406,7 @@ public override DataSourceInfo GetDataSourceInfo() }; - bool FSymbolsEnabled() + static bool FSymbolsEnabled() { // If both of these are null/empty then LoadSymbolsAsync won't do anything anyway, it appears. // But if _NT_SYMCACHE_PATH or _NT_SYMBOL_PATH were empty then default values will appear here. @@ -537,6 +543,8 @@ AllTables GenerateProviderTables(ClassicEventConsumer eventConsumer, Cancellatio allTables.threadTable.SetThreadRundown(eventConsumer.threadEventConsumer.FHaveRundown); + allTables.traceMetadata = this.sources.traceMetadata as ITraceMetadata2; + var traceWThreadPool = eventConsumer.wtpEventConsumer.traceWThreadPool; var threadEventQueue = eventConsumer.threadEventConsumer.threadEventQueue; @@ -627,9 +635,9 @@ AllTables GenerateProviderTables(ClassicEventConsumer eventConsumer, Cancellatio // Lower frequency event for polling. if (cancellationToken.IsCancellationRequested) return null; } - else if (Array.IndexOf(Chromium.DNSInfo.rgGuid, evtGeneric.ProviderId) >= 0) + else if (Array.IndexOf(Chromium.ChromiumTable.rgGuid, evtGeneric.ProviderId) >= 0) { - Chromium.DNSInfo.Dispatch(evtGeneric, allTables.dnsTable); + allTables.chromiumTable.PreDispatch(evtGeneric); } } // foreach evtGeneric diff --git a/src/NetBlame/Providers/Chromium.cs b/src/NetBlame/Providers/Chromium.cs index 1c5060b..7784542 100644 --- a/src/NetBlame/Providers/Chromium.cs +++ b/src/NetBlame/Providers/Chromium.cs @@ -1,22 +1,422 @@ +// Copyright(c) Microsoft Corporation. +// Licensed under the MIT License. + +// cf. https://github.com/microsoft/MSO-Scripts/issues/50 +// cf. https://source.chromium.org/chromium/chromium/src/+/main:net/docs/crash-course-in-net-internals.md +// cf. https://source.chromium.org/chromium/chromium/src/+/main:net/docs/life-of-a-url-request.md +// cf. chrome://net-export OR edge://net-export AND https://chromium.googlesource.com/catapult/+/HEAD/netlog_viewer/ + +// This code is intended to be Assert-Clean when trace collection begins before launching any instance of Chrome or Edge. + using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Text; using System.Text.Json; using Microsoft.Windows.EventTracing.Events; +using Microsoft.Windows.EventTracing.Symbols; + +using TimestampETW = Microsoft.Windows.EventTracing.TraceTimestamp; +using TimestampUI = Microsoft.Performance.SDK.Timestamp; +using NetBlameCustomDataSource.Link; using static NetBlameCustomDataSource.Util; // Assert +using static NetBlameCustomDataSource.Chromium.JSON_Util; +using IDVal = System.Int32; // ProcessId, ThreadId +using QWord = System.UInt64; + +#if DEBUG +using UIDVal = System.UInt64; +#else +using UIDVal = System.UInt32; +#endif namespace NetBlameCustomDataSource.Chromium { - class ResolvedDNS + static class JSON_Util { - public string Domain; // "star-mini.c1q0r.facebook.com" - public string Alias; // "www.facebook.com" or null - public string[] rgAddress; // never null, no missing or white-space elements, no port numbers - } + public const int jsonIntDefault = 0; + + // Convert a string kind to string, else "". + public static string MyGetString(in this JsonElement jsonE) => (jsonE.ValueKind == JsonValueKind.String) ? jsonE.GetString() : string.Empty; + + // Convert a number kind to int, else jsonIntDefault=0 (or the given default). + public static int MyGetNumber(in this JsonElement jsonE, int iDefault = jsonIntDefault) => (jsonE.ValueKind == JsonValueKind.Number) ? jsonE.GetInt32() : iDefault; + public static uint MyGetUNumber(in this JsonElement jsonE, uint iDefault = (uint)jsonIntDefault) => (jsonE.ValueKind == JsonValueKind.Number) ? jsonE.GetUInt32() : iDefault; + public static decimal MyGetDecimal(in this JsonElement jsonE, decimal iDefault = (decimal)jsonIntDefault) => (jsonE.ValueKind == JsonValueKind.Number) ? jsonE.GetDecimal() : iDefault; + + // Convert an array kind (of String) to string[] + public static string[] MyGetStringArray(in this JsonElement jsonE) + { + if (jsonE.ValueKind != JsonValueKind.Array || jsonE.GetArrayLength() == 0 || jsonE[0].ValueKind != JsonValueKind.String) + return Array.Empty(); + + string[] rgstr = new string[jsonE.GetArrayLength()]; + for (int isz = 0; isz < jsonE.GetArrayLength(); ++isz) + rgstr[isz] = jsonE[isz].MyGetString(); + + return rgstr; + } + + // Convert an array kind (of Object) to string[] by extracting the given property from each object: [{"prop1":"string1", "prop2":"string2"}, {"prop1":"string1", "prop2":"string2"}] + public static string[] MyGetStringArray(in this JsonElement jsonE, string strProp) + { + if (jsonE.ValueKind != JsonValueKind.Array || jsonE.GetArrayLength() == 0 || jsonE[0].ValueKind != JsonValueKind.Object) + return Array.Empty(); + + string[] rgstr = new string[jsonE.GetArrayLength()]; + for (int isz = 0; isz < jsonE.GetArrayLength(); ++isz) + rgstr[isz] = jsonE[isz].TryGetProperty(strProp, out JsonElement jsonT) ? jsonT.MyGetString() : string.Empty; + + return rgstr; + } + + // Convert an array kind (of String) of the form [":name1: value1", "name2: value2", ...] to a correspondingly ordered array of selected strings { "value1", "value2", ... } + public static string[] MyGetStringArray(in this JsonElement jsonE, string[] rgstrProp) + { + if (jsonE.ValueKind != JsonValueKind.Array || jsonE.GetArrayLength() == 0 || jsonE[0].ValueKind != JsonValueKind.String) + return Array.Empty(); + + string[] rgstr = new string[rgstrProp.Length]; + + foreach (JsonElement je in jsonE.EnumerateArray()) + { + string strKeyVal = je.MyGetString(); + + // There should be exactly one colon ':' (property name terminator) before the first space, but accept a leading colon like: ":method:" + + int iColon = strKeyVal.IndexOf(':', 1); + AssertImportant(iColon > 0); + if (iColon <= 0) continue; + + int iSpace = strKeyVal.IndexOf(' ', 0, iColon); + AssertImportant(iSpace < 0); // no ' ' found before the first ':' + if (iSpace >= 0) continue; + + AssertImportant(iColon+1 < strKeyVal.Length && char.IsWhiteSpace(strKeyVal[iColon+1])); + + // Creating a span is much more efficient than allocating a temporary substring. + ReadOnlySpan spanKeyVal = strKeyVal.AsSpan(0, iColon); + int istr = rgstrProp.IndexOf(spanKeyVal); + if (istr >= 0) + { + // Skip the colon and the intervening space(s). + for (++iColon; iColon < strKeyVal.Length; ++iColon) + if (!char.IsWhiteSpace(strKeyVal[iColon])) break; + + rgstr[istr] = strKeyVal[iColon..]; + } + } + + return rgstr; + } + + + static bool FFailedDB() { AssertImportant(false); return false; } + + // Convert a True or False kind boolean, else false. + public static bool MyGetBool(in this JsonElement jsonE) => (jsonE.ValueKind == JsonValueKind.True) ? true : (jsonE.ValueKind == JsonValueKind.False ? false : FFailedDB()); + + // Convert a string kind (or a number kind) to int, else: jsonIntDefault + public static QWord MyGetStringAsNumber(in this JsonElement jsonE) => (jsonE.ValueKind == JsonValueKind.String) ? (QWord.TryParse(jsonE.GetString(), out QWord value) ? value : jsonIntDefault) : (QWord)jsonE.MyGetNumber(); + + // Return true if JSON element kind can be use with: MyGetString, MyGetNumber, or MyGetStringAsNumber + private static bool IsMyReturnKind(in this JsonElement jsonE) => (jsonE.ValueKind >= JsonValueKind.Array && jsonE.ValueKind < JsonValueKind.Null); + + /* + Parse a simple JSON string. + Extract a list (array) of properties where at least is one is nested: "/Nest1/Nest2.../Property" + Return an array of JsonElement, OR null on failure. An array element may be empty: default(JsonElement) + Caller resolves using: MyGetString, MyGetNumber, MyGetStringAsNumber, ... + */ + public static JsonElement[] ParseSimpleJsonDeepString(string json, string[] rgstrProperty) + { + const int depthMax = 4; // max nesting depth + + int cTarget = rgstrProperty.Length; + JsonElement[] results = new JsonElement[cTarget]; + string[][] targets = new string[cTarget][]; + + // Normalize property strings into segment arrays + // "url" -> {"url"} + // "/source_dependency/id" -> {"source_dependency", "id"} + for (int iTarget = 0; iTarget < cTarget; iTarget++) + { + targets[iTarget] = rgstrProperty[iTarget].Split('/', StringSplitOptions.RemoveEmptyEntries); + AssertCritical(targets[iTarget].Length <= depthMax); + } + + // Rent a buffer to avoid allocating a new byte[] on the GC heap for every ETW event. + int cbRent = Encoding.UTF8.GetByteCount(json); + byte[] buffer = System.Buffers.ArrayPool.Shared.Rent(cbRent); + + try + { + int cbWritten = Encoding.UTF8.GetBytes(json, 0, json.Length, buffer, 0); + AssertImportant(cbWritten == cbRent); + + var reader = new Utf8JsonReader(new ReadOnlySpan(buffer, 0, cbWritten)); + + int cFound = 0; + int depthCur = 0; + string propertyCur = null; + string[] pathCur = new string[depthMax]; + + while (reader.Read()) + { + switch (reader.TokenType) + { + case JsonTokenType.StartObject: + if (propertyCur != null) + { + if (depthCur < depthMax) + pathCur[depthCur] = propertyCur; + + depthCur++; + propertyCur = null; + } + break; + + case JsonTokenType.EndObject: + if (depthCur > 0) + depthCur--; + + break; + + case JsonTokenType.PropertyName: + bool fPrefix = false; + int iMatch = -1; + + // Evaluate if the current property matches any of our targets + for (int iTarget = 0; iTarget < cTarget; iTarget++) + { + if (results[iTarget].ValueKind != JsonValueKind.Undefined) + continue; // already found + + string[] pathTarget = targets[iTarget]; + bool fPathMatch = true; + + // Ensure we are in the correct parent path before evaluating the property name + if (depthCur > 0) + { + if (pathTarget.Length > depthCur) + { + for (int j = 0; j < depthCur; j++) + { + if (pathCur[j] != pathTarget[j]) + { + fPathMatch = false; + break; + } + } + } + else + { + fPathMatch = false; + } + } - static class DNSInfo + if (fPathMatch) + { + // ValueTextEquals avoids allocating a string! + if (reader.ValueTextEquals(pathTarget[depthCur])) + { + if (pathTarget.Length == depthCur + 1) + { + iMatch = iTarget; + break; // We found the exact property. + } + else + { + fPrefix = true; // We are on the right path to a nested property. + } + } + } + } + + if (iMatch >= 0) + { + if (reader.Read()) + { + // Parse ONLY this specific token/subtree into a JsonDocument. + using (var doc = JsonDocument.ParseValue(ref reader)) + { + if (doc.RootElement.IsMyReturnKind()) + { + results[iMatch] = doc.RootElement.Clone(); + + // Early exit once all properties are found. + if (++cFound == cTarget) + return results; + } + } + } + } + else if (fPrefix) + { + // We must record this property name because we will step into its object. + propertyCur = reader.GetString(); + } + else + { + // Neither a match nor a prefix. Skip the entire subtree. + reader.Skip(); + } + break; + } // switch reader.TokenType + } // while reader.Read() + + return results; + } + catch + { + AssertCritical(false); + return null; + } + finally + { + System.Buffers.ArrayPool.Shared.Return(buffer); + } + } // ParseSimpleJsonDeepString + + /* + Efficiently parse a simple JSON string. + Extract a list (array) of properties, none of which are nested. + Return an array of JsonElement, OR null on failure. An array element may be empty: default(JsonElement) + Caller resolves using: MyGetString, MyGetNumber, MyGetStringAsNumber, ... + */ + public static JsonElement[] ParseSimpleJsonShallowString(string json, string[] rgstrProperty) + { + int cTarget = rgstrProperty.Length; + JsonElement[] results = new JsonElement[cTarget]; + + // Rent a buffer to avoid allocating a new byte[] on the GC heap for every ETW event. + int cbRent = Encoding.UTF8.GetByteCount(json); + byte[] buffer = System.Buffers.ArrayPool.Shared.Rent(cbRent); + + try + { + int cbWritten = Encoding.UTF8.GetBytes(json, 0, json.Length, buffer, 0); + AssertImportant(cbWritten == cbRent); + + var reader = new Utf8JsonReader(new ReadOnlySpan(buffer, 0, cbWritten)); + + bool fStarted = false; + int cFound = 0; + + while (reader.Read()) + { + switch (reader.TokenType) + { + case JsonTokenType.StartObject: + fStarted = true; + break; + + case JsonTokenType.EndObject: + fStarted = false; + break; + + case JsonTokenType.PropertyName: + if (!fStarted) + { + // exceptional + reader.Skip(); + break; + } + + int iMatch = -1; + + // Evaluate if the current property matches any of our targets + for (int iTarget = 0; iTarget < cTarget; iTarget++) + { + if (results[iTarget].ValueKind != JsonValueKind.Undefined) + continue; // already found + + // ValueTextEquals avoids allocating a string! + if (reader.ValueTextEquals(rgstrProperty[iTarget])) + { + iMatch = iTarget; + break; // We found the exact property. + } + } + + if (iMatch < 0) + { + reader.Skip(); + break; + } + + if (reader.Read()) + { + // Parse ONLY this specific token/subtree into a JsonDocument. + using (var doc = JsonDocument.ParseValue(ref reader)) + { + if (doc.RootElement.IsMyReturnKind()) + { + results[iMatch] = doc.RootElement.Clone(); + + // Early exit once all properties are found. + if (++cFound == cTarget) + return results; + } + } + } + break; + } // switch reader.TokenType + } // while reader.Read() + + return results; + } + catch + { + AssertCritical(false); + return null; + } + finally + { + System.Buffers.ArrayPool.Shared.Return(buffer); + } + } // ParseSimpleJsonShallowString + + /* + Parse the JSON string and return an array of JsonElement corresponding to the array of property names. + If any of the property names begins with a forward slash (eg. "/source_dependency/id") + then do a deep parse for nested properties (eg. "source_dependency":{"id":123} ). + This may return null on failure. An array element may be empty: default(JsonElement) + Caller resolves using: MyGetString, MyGetNumber, MyGetBool, MyGetStringAsNumber, ... + */ + public static JsonElement[] ParseSimpleJsonString(string json, string[] rgstrProperty) + { + AssertCritical(rgstrProperty?.Length > 0); + + if (string.IsNullOrWhiteSpace(json)) + return null; + + // Test for any nested properties: "/Nest1/Nest2.../Property" + foreach (string strProp in rgstrProperty) + { + if (strProp[0] == '/') + return ParseSimpleJsonDeepString(json, rgstrProperty); + } + + return ParseSimpleJsonShallowString(json, rgstrProperty); + } + } // class JSON_Util + + + public static class DNSInfo { + public class ResolvedDNS + { + public string Domain; // "star-mini.c1q0r.facebook.com" + public string Alias; // "www.facebook.com" or null + public string[] rgAddress; // never null, no missing or white-space elements, no port numbers + } + /* Parse the JSON string from the 'params' field where: 'source_type' == 'HOST_RESOLVER_IMPL_JOB' @@ -42,8 +442,8 @@ static class DNSInfo "alias_target":"ax-0001.ax-msedge.net", "domain_name":"c-bing-com.ax-0001.ax-msedge.net", "type":"alias" -   } -  ], + } + ], } Output: @@ -51,9 +451,9 @@ static class DNSInfo Alias = "c.bing.com" rgAddress = { "150.171.27.10", "150.171.28.10" } */ - static ResolvedDNS ParseHostResolveDNS(string json) + public static ResolvedDNS ParseHostResolveDNS(string json) { - JsonDocument jd = null; + JsonDocument jd; try { jd = JsonDocument.Parse(json); @@ -63,6 +463,8 @@ static ResolvedDNS ParseHostResolveDNS(string json) jd = null; } + AssertCritical(jd != null); + if (jd == null) return null; @@ -86,17 +488,17 @@ static ResolvedDNS ParseHostResolveDNS(string json) switch (type.GetString()) { - case "data": - break; + case "data": + break; - case "alias": - rgAlias[iRes] = true; - continue; + case "alias": + rgAlias[iRes] = true; + continue; - // case "metadata": - // case "error": - default: - continue; + // case "metadata": + // case "error": + default: + continue; } // Only one "data" element! @@ -186,29 +588,5590 @@ static ResolvedDNS ParseHostResolveDNS(string json) }; } // using (jd) } // ParseHostResolveDNS + } // DNSInfo - public static Guid[] rgGuid = + public enum Priority : byte + { + THROTTLED = 0, + IDLE = 1, + LOWEST = 2, + LOW = 3, + MEDIUM = 4, + HIGHEST = 5, + Unknown = 42 + } + + + static class Util + { + public static QWord GetUID64(this IGenericEvent evt) => evt.GetUInt64("Id"); + // Return the low part of the "Id" field, which is stored as a 64-bit number. + public static UIDVal GetUID(this IGenericEvent evt) => (UIDVal)evt.GetUID64(); + + public static string GetParams(this IGenericEvent evt) => evt.GetString("params"); + + static bool CheckPhase(this IGenericEvent evt, string strPhase) => evt.GetString("Phase").Equals(strPhase); + public static bool IsBeginPhase(this IGenericEvent evt) => CheckPhase(evt, "Begin"); + public static bool IsEndPhase(this IGenericEvent evt) => CheckPhase(evt, "End"); + public static bool IsInstantPhase(this IGenericEvent evt) => CheckPhase(evt, "Instant"); + + // Only when evt.IsBeginPhase() + public static string GetSourceType(this IGenericEvent evt) => evt.GetString("source_type"); + public static bool CheckSourceType(this IGenericEvent evt, string strType) => evt.GetSourceType().Equals(strType); + + public static bool TestResolverSourceType(this IGenericEvent evt) { - new Guid("{3A5F2396-5C8F-4F1F-9B67-6CCA6C990E61}"), // Microsoft.MSEdgeStable - new Guid("{BD089BAA-4E52-4794-A887-9E96868570D2}"), // Microsoft.MSEdgeBeta - new Guid("{E16EC3D2-BB0F-4E8F-BDB8-DE0BEA82DC3D}"), // Microsoft.MSEdgeWebView2 - new Guid("{C56B8664-45C5-4E65-B3C7-A8D6BD3F2E67}"), // Microsoft.MSEdgeCanary - new Guid("{D30B5C9F-B58F-4DC9-AFAF-134405D72107}"), // Microsoft.MSEdgeDev - new Guid("{d2d578d9-2936-45b6-a09f-30e32715f42d}") // CHROME - }; + switch (evt.GetSourceType()) + { + case "SSL_CONNECT_JOB": + case "TRANSPORT_CONNECT_JOB": + case "QUIC_SESSION_POOL_DIRECT_JOB": + case "NETWORK_SERVICE_HOST_RESOLVER": + return true; +#if DEBUG + case "NETWORK_QUALITY_ESTIMATOR": + case "PAC_FILE_DECIDER": + case "CERT_VERIFIER_JOB": + case "SOCKS_CONNECT_JOB": + case "HTTP_PROXY_CONNECT_JOB": + return false; +#endif // DEBUG + default: + AssertImportant(false); // add to approved cases? + return false; + } + } + + public static Priority GetPriority(this string strPri) => Priority.TryParse(strPri, out Priority pri) ? pri : Priority.Unknown; - public static void Dispatch(in IGenericEvent evt, DNSClient.DNSTable dnsTable) + public static int GetJSONNumber(string strJSON, string[] rgstrNumber) { - if (!evt.TaskName.Equals("HOST_RESOLVER_DNS_TASK_EXTRACTION_RESULTS")) - return; + JsonElement[] rgje = JSON_Util.ParseSimpleJsonString(strJSON, rgstrNumber); + if (rgje == null) return jsonIntDefault; + return rgje[0].MyGetNumber(); + } - AssertImportant(evt.GetString("source_type").Equals("HOST_RESOLVER_IMPL_JOB")); + public static int GetJSONNumber(this IGenericEvent evt, string[] rgstrNumber) => GetJSONNumber(evt.GetParams(), rgstrNumber); - ResolvedDNS rdns = ParseHostResolveDNS(evt.GetString("params")); + // Returns { source_dependency: id } or jsonIntDefault + public static int TryGetSourceId(this IGenericEvent evt) + { + return evt.GetJSONNumber(ChromiumTable.rgstrSourceId); + } - if (rdns != null) - dnsTable.AddServerAndAddress(rdns.Domain, rdns.Alias, rdns.rgAddress); + public static int GetSourceId(this IGenericEvent evt) + { + int srcdep = evt.TryGetSourceId(); + AssertCritical(srcdep != jsonIntDefault); + return srcdep; } - } // DNSInfo -} // namespace NetBlameCustomDataSource.DNSClient \ No newline at end of file + + public static readonly string[] rgstrNetError = { "net_error" }; + public static readonly string[] rgstrQuicError = { "quic_error" }; + + public static int GetNetError(this IGenericEvent evt) => evt.GetJSONNumber(rgstrNetError); + public static int GetQuicError(this IGenericEvent evt) => evt.GetJSONNumber(rgstrQuicError); + + public static string ScrubAnonKey(string anonkey) + { +#if !DEBUG + if (anonkey?.Equals("null") ?? true) + return string.Empty; +#endif // !DEBUG + return anonkey; + } + + public static JsonElement[] ParseSimpleJsonString(this IGenericEvent evt, string[] rgstr) => JSON_Util.ParseSimpleJsonString(evt.GetParams(), rgstr); + + + // Implement: Array.IndexOf(string[], ReadOnlySpan) + public static int IndexOf(this string[] rgstr, ReadOnlySpan span) + { + for (int i = 0; i < rgstr.Length; ++i) + { + if (span.SequenceEqual(rgstr[i].AsSpan())) + return i; + } + return -1; + } + + + // Efficiently compare two base URLs, either of which may or may not end with '/'. + public static bool Equal2(this string url1, string url2) + { + int cch1 = url1.Length; + int cch2 = url2.Length; + + // The length difference must be: -1, 0, 1 + if ((uint)(cch1 - cch2 + 1) > 2) return false; // common + + if (url1[^1] == '/') + --cch1; + + if (url2[^1] == '/') + --cch2; + + if (cch1 != cch2) return false; + + return string.Compare(url1, 0, url2, 0, cch1, StringComparison.Ordinal) == 0; + } + + + /* + Parse strings such as these: + "https://www.google.com \u003Chttps://google.com same_site>" + "pm/https://fonts.gstatic.com \u003Chttps://example.com cross_site>" + And return base URLs: + "https://www.google.com" + "https://fonts.gstatic.com" + */ + public static string BaseURLFromGroupId(this string strURL) + { + int t1Start = -1, t1Len = 0; + + int len = strURL.Length; + for (int i = 0; i < len;) + { + // Skip separators + while (i < len && (strURL[i] == '/' || strURL[i] == ' ')) + i++; + + if (i >= len) break; + + // Find the length of the current token + int start = i; + while (i < len && strURL[i] != '/' && strURL[i] != ' ') + i++; + + int lenCur = i - start; + + // Track tokens and evaluate + if (t1Start == -1) + { + // First valid token found; store its bounds + t1Start = start; + t1Len = lenCur; + } + else + { + // We have a previous token (t1) and a current token (domain). + // Check if t1 meets the scheme criteria: ends with ':' and starts with a letter. + if (strURL[t1Start + t1Len - 1] == ':' && char.IsLetter(strURL[t1Start])) + return strURL[t1Start..(start+lenCur)]; + + // Shift the current token back to become the new t1 + t1Start = start; + t1Len = lenCur; + } + } + + // What URL string is this!? + AssertImportant(false); + return strURL; + } + + /* + Given a URL determine if it is of the type "https://123.4.56.789:9876/..." + Returns false if not. + Else use uri.Host or uri.DnsSafeHost (no [IPv6] brackets), and url.Port + */ + public static Uri CreateURI(this string strURL, bool fIPOnly = false) + { + if (!Uri.TryCreate(strURL, UriKind.Absolute, out Uri uri)) + return null; + + if (fIPOnly) + { + UriHostNameType type = uri.HostNameType; + if (type != UriHostNameType.IPv4 && type != UriHostNameType.IPv6) + return null; + } + + return uri; + } + + /* + Parse "Host:Port" into a string and a ushort. + */ + public static string GetHostAndPort(this string strHostPort, out ushort port) + { + port = 0; + if (string.IsNullOrWhiteSpace(strHostPort)) return null; + + ReadOnlySpan span = strHostPort; + int iPort = span.IndexOf(':'); + AssertCritical(iPort > 0); // host:port + if (iPort <= 0) return null; + + if (!ushort.TryParse(span[(iPort+1)..], out port)) return null; + + return span[0..iPort].ToString(); + } + + /* + Headers: {"headers":["HTTP/1.1 304 Not Modified",...]} + Extract: "304 Not Modified" + */ + public static string GetStatusJSON(this string strJSON) + { + AssertImportant(strJSON.StartsWith("{\"headers\":[\"HTTP/")); + + int iStart = strJSON.IndexOf(":[\""); + if (iStart < 0) + return string.Empty; + + iStart = strJSON.IndexOf(' ', iStart+3); + if (iStart < 0) + return string.Empty; + + int iEnd = strJSON.IndexOf('\"', iStart); + if (iEnd <= iStart) + return string.Empty; + + return strJSON[(iStart+1)..iEnd]; + } + + + // net_error_list.h (negative numbers) + // quic_error_codes.h (positive numbers) + public static string ErrorFromI(int iError) + { + if (iError == 0) + return string.Empty; + + if (iError > 0) + return iError.ToString(); + + string strError = iError switch + { + -2 => "-2: FAILED", + -3 => "-3: ABORTED", + -21 => "-21: NETWORK_CHANGED", + -100 => "-100: CONNECTION_CLOSED", + -101 => "-101: CONNECTION_RESET", + -103 => "-103: CONNECTION_ABORTED", + -105 => "-105: NAME_NOT_RESOLVED", + -106 => "-106: ERR_INTERNET_DISCONNECTED", + -109 => "-109: ERR_ADDRESS_UNREACHABLE", + -118 => "-118: CONNECTION_TIMED_OUT", + -173 => "-173: WS_UPGRADE", + -400 => "-400: CACHE MISS", + _ => null + }; + + if (strError != null) + return strError; + + int uError = (-iError) / 100; + strError = iError.ToString(); + + return uError switch + { + 0 => strError + ": System Error", // 1- 99 + 1 => strError + ": Connecton Error", // 100-199 + 2 => strError + ": Certificate Error", // 200-299 + 3 => strError + ": HTTP Error", // 300-399 + 4 => strError + ": Cache Error", // 400-499 + 5 => strError + ": Misc Error", // 500-599 + 6 => strError + ": FTP Error", // 600-699 + 7 => strError + ": Cert Manager Error", // 700-799 + 8 => strError + ": DNS Resolver Error", // 800-899 + 9 => strError + ": Blob Error", // 900-999 + _ => strError + }; + } + + + /* + Garbage Collect + Make a list of all List items which are 'Gone' and therefore garbage collectable. + Use the new list to delete items from the main list. + */ + public static void DoGC(this Hash hash) where V : class, IGCollectable + { + List keyRemove = null; + + foreach (var kvp in hash) + { + if (kvp.Value?.Gone ?? true) + { + if (keyRemove == null) + keyRemove = new List(hash.Count / 2); + + keyRemove.Add(kvp.Key); + } + } + + if (keyRemove == null) return; + + if (keyRemove.Count == hash.Count) + { + hash.Clear(); + } + else + { + foreach (var key in keyRemove) + hash.Remove(key); + } + } + } // Util + + + /* + For GarbageCollect / DoGC() + */ + public interface IGCollectable + { + bool Gone { get; set; } + } + + + public class ResolverManager : IGCollectable + { + public string host; // "https://www.google.com" + public string anon_key; // "https://google.com same_site" + public string[] rgstrAddress; // { "XX.XX.XX.XX", ... } + public string[] rgstrCanon; // { "www.google.com" } // or alternate DNS name + + public bool Gone { get; set; } + } // ResolverManager + + + public class Socket : IGCollectable + { + // If Type==TCP at the end then it's probably HTTP1.1. If it's HTTP2 then the event SSL_CONNECT would set it. + public StreamType Type { get; set; } + + readonly public IDVal pid; + readonly public IDVal tid; + + public IPEndPoint addrLocal; + public IPEndPoint addrRemote; + + public TimestampUI timeStampCreate; // SOCKET_ALIVE + public TimestampUI timeStampConnect; // TCP_CONNECT.Begin + public TimestampUI timeStampClose; // TCP_CONNECT.End & SOCKET_CLOSED.Instant + + public WinsockAFD.Connection cxn; + + public int iError; + + public bool fSSL; + public bool fCanceled; + public bool fGathered; +#if DEBUG + public bool fTCP; + + public int cref; // attached to how many HTTP2/3 Sessions? + public int crefH1; // attached to HTTP1 Session(s) + + public QWord uidDB; // for debugging +#endif // DEBUG + public UIDVal uidBound; // from SOCKET_POOL_BOUND_TO_SOCKET + + public Socket(StreamType type, in IGenericEvent evt) + { + AssertCritical(type != StreamType.Unknown && type != StreamType.CACHE); + this.Type = type; + this.pid = evt.ProcessId; + this.tid = evt.ThreadId; + this.timeStampCreate = evt.Timestamp.ToGraphable(); + this.timeStampConnect = this.timeStampCreate; // may be overwritten later + this.timeStampClose.SetMaxValue(); + this.fSSL = (type == StreamType.QUIC || type == StreamType.HTTP2); // if HTTP1, set it later +#if DEBUG + this.uidDB = evt.GetUID(); + this.fTCP = (type != StreamType.QUIC); +#endif // DEBUG + } + + public string Error() + { + if (this.iError != 0) + return Util.ErrorFromI(this.iError); + + if (this.fCanceled) + return "Canceled"; + + return string.Empty; // "" + } + + public ushort WSSocket() + { + ushort wssocket = (ushort)this.addrLocal.PortGraphable(); + if (wssocket == 0 && this.cxn != null) + wssocket = this.cxn.socket; + + AssertImportant(this.cxn == null || this.cxn.socket == wssocket); + return wssocket; + } + + public void SetAddrLocalRemote(string strLocal, string strRemote) + { + ushort port; + IPAddress ipAddress; + + AssertImportant(this.addrLocal.Empty()); + + if (strLocal != null && DNSClient.DNSTable.TryParseWithPort(strLocal, out ipAddress, out port)) + this.addrLocal = new IPEndPoint(ipAddress, port); + + AssertImportant(FImplies(strLocal != null, !this.addrLocal.Empty())); + + if (strRemote != null && DNSClient.DNSTable.TryParseWithPort(strRemote, out ipAddress, out port)) + { + if (this.addrRemote.Empty()) + this.addrRemote = new IPEndPoint(ipAddress, port); + else + AssertImportant(this.addrRemote.Equals(new IPEndPoint(ipAddress, port))); + } + + AssertImportant(!this.addrRemote.Empty()); + } + + public bool Closed => !this.timeStampClose.HasMaxValue(); + + public void Close(in TimestampUI timeStamp) + { + if (this.Closed) return; + + this.timeStampClose = timeStamp; + + this.Gone = true; + } + + // Implement IGCollectable + public bool Gone { get; set; } + } // Socket + + + /* + Stream Types: + - CACHE : filesystem + - TCP : HTTP1 or HTTP2 TBD + - HTTP1 / TCP : HTTP/1.1 probably + - HTTP2 / TCP + - HTTP3 / QUIC / UDP + */ + public enum StreamType { Unknown = 0, CACHE, TCP, HTTP1, HTTP2, QUIC } + + /* + Placeholder Sessions have placeholder Streams wth special IDs. + */ + public enum IStreamSpecial : int + { + HTTP1 = -1, + CACHE = -2, + UNKNOWN = -3, + MIN = UNKNOWN + }; + + + public class Request : Gatherable, IGraphableEntry, IGCollectable + { + public StreamType Type { get; set; } // default: Unknown + + public StreamType TypeTCP { get; set; } // If the Type of the bound socket/session is a TCP type (HTTP1/2) then it will be this. + + readonly public IDVal pid; + readonly public IDVal tid; + + private string _domain; + private string _canon = string.Empty; // canonical server name (if different from domain) + private string _url; + private string _urlScrub; // #fragment identifier stripped + + public string method; + + public string anon_key; // network_anonymization_key + public string strTaskStash; // StashStalledRequest + + // Alternates to what's in the Socket + public string ipAddr; + public ushort port; + + public int hidGroup; // hash of group_id string: TCP_CLIENT_SOCKET_POOL_REQUESTED_SOCKET & SOCKET_POOL_CONNECT_JOB_CREATED + public int iError; // "net_error":-173 + + // Corresponding xfer byte counts are in the Stream, but these may be posted before the Stream is created. + public uint cbUpload, cbDownload; + public bool fChunkedUpload; + + public Priority priority; + + public UIDVal uidQUIC; // UID of HTTP_STREAM_JOB: use_quic:true + public UIDVal uidTCP; // UID of HTTP_STREAM_JOB: use_quic:false + + private readonly QWord uidCreate64; + + public UIDVal uidRequest; // 64-bit ID of URL_REQUEST_START_JOB + + public TimestampUI timeStampBeginJob; // IRL_REQUEST_START_JOB.Begin ETW Timestamp + public TimestampUI timeStampEndJob; // IRL_REQUEST_START_JOB.End ETW Timestamp + public TimestampETW timeRef; + + public IStackSnapshot stack; +#if DEBUG + // For HTTP1: + private Socket _socketTCP; // HTTP1 / TCP + public Socket SocketTCP + { + get => _socketTCP; + set + { + AssertCritical(value?.fTCP ?? true); + if (_socketTCP == null || value == null) + _socketTCP = value; + else + AssertImportant(_socketTCP == value); + } + } +#endif // DEBUG + + private Session _sessionQUIC; // QUIC / HTTP3 / UDP + private Session _sessionHTTP2; // HTTP2 / TCP + private Session _sessionOther; // HTTP1 / CACHE + + public Session SessionQUIC + { + get => this._sessionQUIC; + + set + { + AssertCritical(value == null || value.Type == StreamType.QUIC); + if (this._sessionQUIC == null || value == null) + this._sessionQUIC = value; + else + AssertImportant(this._sessionQUIC == value); + } + } + + public Session SessionHTTP2 + { + get => this._sessionHTTP2; + + set + { + AssertCritical(value == null || value.Type == StreamType.HTTP2); + if (this._sessionHTTP2 == null || value == null) + this._sessionHTTP2 = value; + else + AssertImportant(this._sessionHTTP2 == value); + } + } + + public Session SessionOther + { + get => this._sessionOther; + } + + // CHROMIUM regularly creates or reuses two simultaneous Sessions: HTTP2/TCP and QUIC/HTTP3/UDP + // In most cases one or the other is chosen by the event: HTTP_STREAM_JOB_BOUND_TO_REQUEST + // This is the Session which contains the Stream which is linked to this Request. + public Session Session + { + get + { + AssertImportant(FImplies(!this.IsSessionEmpty, this.Type != StreamType.Unknown && this.Type != StreamType.TCP)); + + Session sessionRet; + switch (this.Type) + { + case StreamType.QUIC: + sessionRet = this._sessionQUIC; + break; + + case StreamType.HTTP2: + sessionRet = this._sessionHTTP2; + break; + + case StreamType.HTTP1: + case StreamType.CACHE: + sessionRet = this._sessionOther; + break; + + default: + return null; + } + + // If there is a Stream, it must be attached to the new Session. + AssertCritical(this.stream == null || (sessionRet.rgStream?.ContainsValue(this.stream) ?? false)); + return sessionRet; + } + + set + { + StreamType type = value?.Type ?? this.Type; + AssertImportant(FImplies(value != null, type != StreamType.Unknown && type != StreamType.TCP)); + + switch (type) + { + case StreamType.QUIC: + this.SessionQUIC = value; + break; + + case StreamType.HTTP2: + this.SessionHTTP2 = value; + break; + + case StreamType.HTTP1: + case StreamType.CACHE: + AssertImportant(this._sessionOther == null || this._sessionOther == value); + this._sessionOther = value; + break; + + default: + AssertCritical(false); + break; + } + + DEBUG(this.Session); // Do the 'get' DEBUG checks. + } + } + + /* + Set both this.Type and this.Session + this.Session will afterward return the set value (non-null). + */ + public Session SessionSet + { + set + { + AssertCritical(value != null); + if (value == null) return; + + AssertCritical(this.Type == value.Type || this.Type == StreamType.Unknown || this.Type == StreamType.TCP); + + this.Type = value.Type; + this.Session = value; // Do the 'set' and 'get' DEBUG checks. + } + } + + public void SessionReset() + { + this.Type = StreamType.Unknown; + this.TypeTCP = StreamType.Unknown; + + this._sessionQUIC = null; + this._sessionHTTP2 = null; + this._sessionOther = null; + this.stream = null; +#if DEBUG + this._socketTCP = null; +#endif // DEBUG + } + + public bool IsSessionEmpty => this._sessionQUIC == null && this._sessionHTTP2 == null && this._sessionOther == null; + + public Session.Stream stream; + + public XLink xlink; + + public bool fWebSocket; // WEBSOCKET_ALIVE + public bool fSSL; + public bool fRedirect; + public bool fCanceled; + + public string URL + { + get => this._url; + + set + { + this._url = value; + int iHash = value.IndexOf('#'); + this._urlScrub = (iHash < 0) ? value : value.Substring(0, iHash); + + // Convert: https://123.456.78.90:432/ + Uri uri = this._urlScrub.CreateURI(true); + if (uri != null) + { + this.ipAddr = uri.Host; + this.port = (ushort)uri.Port; + } + } + } + + public string URLScrub => this._urlScrub; + + public string Canon + { + get => this._canon; + + set + { + AssertImportant(!string.IsNullOrWhiteSpace(this.Domain)); + AssertImportant(!string.IsNullOrWhiteSpace(value)); + AssertImportant(!value.IsNA()); + + if (!value.Equals(this.Domain)) + this._canon = value; + } + } + + public string Domain + { + get + { + if (this._domain != null) return this._domain; + if (this._urlScrub == null) return null; + return this._urlScrub.CreateURI()?.Host; + } + + set => this._domain = value; + } + + public string AddressAndPort() + { + Socket soc = this.Session?.socket; + if (!(soc?.addrRemote).Empty()) + return soc.addrRemote.ToGraphable(); + + string strAddr = NetBlameCustomDataSource.Util.strNA; + + if (!string.IsNullOrWhiteSpace(this.ipAddr)) + { + strAddr = this.ipAddr; + if (this.port != 0) + strAddr += ":" + this.port.ToString(); + } + + return strAddr; + } + + public static readonly string strPreconnect = "preconnect"; + + public bool IsPreconnect => string.ReferenceEquals(this.method, strPreconnect); + + public bool IsSpeculative => this.IsPreconnect && this.PreKey() == 0; // See PreconnectRequest() + + public QWord WSConnection => this.Session?.socket?.cxn?.qwEndpoint ?? 0; // Winsock Connection ID + + public bool FAttachedToStream => this.stream != null; + + public int PreKey() + { + AssertCritical(this.IsPreconnect); +#if DEBUG + switch (this.Type) + { + case StreamType.QUIC: + case StreamType.HTTP2: + case StreamType.HTTP1: + AssertImportant(this.Session != null); + break; + + case StreamType.TCP: + AssertImportant(this.hidGroup != 0); + break; + + case StreamType.Unknown: + AssertImportant(this.TypeTCP != StreamType.Unknown || this.stream == null); + break; + + default: + AssertImportant(false); + break; + } +#endif // DEBUG + // May return 0 (IsSpeculative) + return this.Session?.PreKey ?? this.hidGroup; + } // PreKey + + +#if DEBUG + public List rguidDB = new(16); + + public List rgsrcdepDB = new(8); + + public void AddSrcDep(int srcdep) + { + if (!this.rgsrcdepDB.Contains(srcdep)) + this.rgsrcdepDB.Add(srcdep); + } + + private static QWord ReconstructUID(/*QWord*/UIDVal uid) => uid; +#else // !DEBUG + + /*[Conditional("DEBUG")]*/ + public List rguidDB; + + // The Event Id originates as: QWord Id = (DWord)ProcessEventGroupCounter ^ (QWord)ProcessRandomKey; + private QWord ReconstructUID(/*DWord*/UIDVal uid) => (QWord)uid | (this.uidCreate64 & 0xFFFFFFFF00000000); +#endif // !DEBUG + + [Conditional("DEBUG")] + public void AddUID(UIDVal uid) + { + if (!this.rguidDB.Contains(uid)) + this.rguidDB.Add(uid); + } + + // Final UI rendering of the Event ID value for the WPA Request Table. + public QWord UIDRequest => ReconstructUID(this.uidRequest); + public QWord UIDSession => (this.Session != null) ? ReconstructUID(this.Session.uidVal) : 0; + + + public Request(in IGenericEvent evt) + { + this.pid = evt.ProcessId; + this.tid = evt.ThreadId; + this.timeRef = evt.Timestamp; + this.stack = evt.Stack; + this.priority = Priority.Unknown; + this.timeStampEndJob.SetMaxValue(); + this.uidCreate64 = evt.GetUID64(); + } + + public Request(string strURL, in IGenericEvent evt) : this(in evt) + { + this.URL = strURL; + } + + // URL_REQUEST_START_JOB: The Request may close and reopen when redirected. Preconnect Requests don't really close. + public bool Closed => !this.timeStampEndJob.HasMaxValue() && !this.IsPreconnect; + + // CORS_REQUEST.End: The Request is truly closed. + public bool Gone { get; set; } + + public string Error() + { + if (this.fRedirect) + return "Redirected"; + + if (this.iError != 0) + return Util.ErrorFromI(this.iError); + + if (this.fCanceled) + return "Canceled"; + + if (this.IsSpeculative) + return "Speculative"; + + if (this.Session?.socket != null) + return this.Session.socket.Error(); + + if (this.Session?.FMigrated ?? false) + return "Migrated"; + + return string.Empty; // "" + } + + public string Transport() + { + if (!this.fWebSocket) + { + if (this.Type == StreamType.Unknown) + return string.Empty; + + return this.Type.ToString(); + } + else + { + if (this.Type == StreamType.Unknown) + return "WebSocket"; + + return "WebSocket/" + this.Type.ToString(); + } + } + + /* + HTTP_STREAM_JOB_BOUND_TO_REQUEST / HTTP_STREAM_JOB + The ID is of a HTTP_STREAM_JOB event, + which had the attribute: use_quic: true or false + The ID that matches determines the type of stream chosen. + */ + public void SetStreamType(UIDVal uid) + { + StreamType type; + + AssertCritical(uid != 0); + AssertImportant(this.uidQUIC != this.uidTCP); + + if (uid == this.uidQUIC) + type = StreamType.QUIC; + else if (uid == this.uidTCP) + type = (this.TypeTCP == StreamType.HTTP2) ? StreamType.HTTP2 : StreamType.HTTP1; + else + type = this.Type; // probably Unknown + + AssertImportant(FImplies(this.Type != StreamType.Unknown, this.Type == type)); + this.Type = type; + } + + /* + Set the ID of the HTTP_STREAM_JOB for use later by SetStreamType when one of them 'wins'. + A previous HTTP_STREAM_JOB could have been canceled, so overwrites happen. + */ + public void SetStreamUID(UIDVal uid, bool fQuic) + { + if (fQuic) + { + AssertImportant(this.uidTCP != uid); + this.uidQUIC = uid; + } + else + { + AssertImportant(this.uidQUIC != uid); + this.uidTCP = uid; + } + } + + + /* + Return an IP address associated with this Request. + (In reality there may be multiple associated addresses.) + */ + public IPAddress IPAddress(Socket soc) + { + if (soc != null && !soc.addrRemote.Empty()) + return soc.addrRemote.Address; + + if (string.IsNullOrWhiteSpace(this.ipAddr)) + return null; + + if (System.Net.IPAddress.TryParse(this.ipAddr, out IPAddress ipAddr)) + return ipAddr; + + return null; + } + + /* + Get the "net_error" from this event, if any, and assign it to this Request. + */ + public void SetNetError(in IGenericEvent evt) + { + int error = evt.GetNetError(); + if (error == jsonIntDefault) return; // common + + if (this.iError == 0) + this.iError = error; + } + + /* + Create a placeholder Session/Stream for HTTP1 code simplicity. + If true then caller invokes: sessionTable.Add(req.session); + */ + public bool FAttachPlaceholderSessionAndStream(Socket soc, in IGenericEvent evt) + { + AssertImportant(soc.Type != StreamType.Unknown && soc.Type != StreamType.QUIC); + + soc.fSSL |= this.fSSL; + + // If the Socket is SSL then its type was set by: SSL_CONNECT.End + // If it's not SSL then it MUST be HTTP1. + StreamType type = soc.fSSL ? soc.Type : StreamType.HTTP1; + AssertImportant(this.Type == soc.Type || this.Type == type || this.Type == StreamType.Unknown); + + if (type == StreamType.TCP) return false; // handle this later + + AssertImportant(type == StreamType.HTTP1 || type == StreamType.HTTP2); // else what? + + // The this.Type may eventually be QUIC/HTTP3/UDP, but if it's TCP then it's this TypeTCP: HTTP1 or HTTP2 + this.TypeTCP = type; + soc.Type = type; +#if DEBUG + AssertCritical(soc.fTCP); + + // In Preconnect scenarios, multiple Sockets may spin up within the context of this Request. + // See: TCP_CLIENT_SOCKET_POOL_REQUESTED_SOCKETS, SOCKET_POOL_CONNECTING_N_SOCKETS ("num_sockets":2+) + + if (this.SocketTCP == null || this.SocketTCP.iError != 0) + this.SocketTCP = soc; + else + AssertCritical(FImplies(!this.IsPreconnect, this.SocketTCP == soc)); +#endif // DEBUG + + if (type != StreamType.HTTP1) return false; + + // If it already has a Session, and not an error-Socket, then nothing more to do. + if (this.Session != null && (this.Session.socket?.iError ?? 0) == 0) return false; + + // HTTP1 does not have a Session/Stream. + // Create a dummy Session/Stream for this Request for simpler code overall. + + Session session = new Session(StreamType.HTTP1, in evt) + { + domain = this.Domain, + port = (ushort)soc.addrRemote.PortGraphable(), + uidVal = evt.GetUID() + }; + + Session.Stream stream = session.EnsureStream((int)IStreamSpecial.HTTP1, evt.Timestamp.ToGraphable()); + stream.strURL = this.URLScrub; + stream.strMethod = this.method; + stream.strDomain = this.Domain; + stream.Attach(this); + + session.Attach(soc); + this.SessionSet = session; + + AssertCritical(this.Type == StreamType.HTTP1); + AssertCritical(stream.request.Session == session); + + return true; + } + + /* + Use the Request to create a placeholder Session when no Socket is available (for code simplicity). + Maybe tracing started too late to capture the real Session. + A CACHE Request has no Socket, since data comes from the filesystem. + If true then caller invokes: sessionTable.Add(req.Session); + */ + public bool FAttachPlaceholderSessionAndStream(StreamType type, in IGenericEvent evt) + { + if (this.Session != null) + return false; + + // If this Request is already a different type, don't create a new placeholder Session. + + AssertImportant(this.Type == type || this.Type == StreamType.Unknown); + if (this.Type != type && this.Type != StreamType.Unknown) + return false; + + // CACHE type does not have a Session/Stream. + // Create a dummy Session/Stream for this Request for simpler code overall. + + Session session = new Session(type, in evt) + { + domain = this.Domain, + port = this.port, + uidVal = evt.GetUID() + }; + + IStreamSpecial iStream = type switch + { + StreamType.CACHE => IStreamSpecial.CACHE, + StreamType.HTTP1 => IStreamSpecial.HTTP1, + _ => IStreamSpecial.UNKNOWN + }; + + Session.Stream stream = session.EnsureStream((int)iStream, evt.Timestamp.ToGraphable()); + stream.strURL = this.URLScrub; + stream.strMethod = this.method; + stream.strDomain = this.Domain; + stream.Attach(this); + + this.SessionSet = session; + + AssertCritical(this.Type == type); + AssertCritical(stream.request.Session == session); + + return true; + } + + /* + Create a place holder Session based on this Request. + Caller: sessionTable.Add(session) + */ + public Session NewPlaceholderSession(StreamType type, in IGenericEvent evt) + { + AssertCritical(type == StreamType.HTTP2 || type == StreamType.QUIC); // else FAttachPlaceholderSessionAndStream + + return new Session(type, in evt) + { + domain = this.Domain, + port = this.port, + uidVal = evt.GetUID(), + fRecovered = true + }; + } + + + /* + The corresponding Session/Stream was migrated, and activity happened both before and after. + This Request followed new Stream, but we also need a Request that references the old Stream. + Return a copy of the Request with the necessary adjustments. Also adjust 'this'. + ** The caller must set request2.session / .stream ** + */ + public Request Migrate(in TimestampUI timeMigrate) + { + AssertCritical(timeMigrate.Between(this.timeStampBeginJob, this.timeStampEndJob)); + + Request request2 = (Request)this.MemberwiseClone(); // shallow + + this.timeStampBeginJob = request2.timeStampEndJob = timeMigrate; + + request2.SessionReset(); +#if DEBUG + AssertImportant(!request2.fGathered); +#endif // DEBUG + return request2; + } + + + /* + Set the close time, and fill in missing fields. + */ + public void Close(in TimestampUI timeStamp) + { + // Close the Request. + if (this.timeStampEndJob.HasMaxValue()) + this.timeStampEndJob = timeStamp; + } // Close + + public void Close(ResolverManager resolver, DNSClient.DNSTable dnsTable, in TimestampUI timeStamp) + { + Close(in timeStamp); + + AssertCritical(!string.IsNullOrWhiteSpace(this.URL)); + + Socket socket = this.Session?.socket; + + if (socket != null) + { + // These values are columns in the WPA Network Table. + + if (this.port == 0 && !socket.addrRemote.Empty()) + this.port = (ushort)socket.addrRemote.PortGraphable(); + } + + if (resolver != null) + { + if (string.IsNullOrWhiteSpace(this.Canon) && resolver.rgstrCanon?.Length > 0) + this.Canon = resolver.rgstrCanon[0]; + + if (string.IsNullOrWhiteSpace(this.ipAddr) && resolver.rgstrAddress?.Length > 0) + this.ipAddr = resolver.rgstrAddress[0]; + } + + if (!string.IsNullOrWhiteSpace(this.ipAddr) && !string.IsNullOrWhiteSpace(this.Canon)) return; + + // Resort to the DNS Table to fill in missing information. + + IPAddress ipAddress = this.IPAddress(socket); + + if (ipAddress.Empty()) + { + uint iDNS = dnsTable.IFindDNSEntryByServer(this.Domain); + + // NOTE: There may be multiple addresses for this domain. Choose the first one since we don't know otherwise. + ipAddress = dnsTable.AddressFromI(iDNS, 1); + if (!ipAddress.Empty()) + this.ipAddr = ipAddress.ToString(); + } + + if (!ipAddress.Empty()) + { + string strServerAlt = NetBlameCustomDataSource.Util.strNA; + string strServer = dnsTable.DNSNameAndAlt(ipAddress, ref strServerAlt); + + AssertImportant(!string.IsNullOrWhiteSpace(this.Domain)); + AssertInfo(this.Domain.Equals(strServer)); + + if (!strServerAlt.IsNA()) + this.Canon = strServerAlt; + } + } // Close + + + // Implement IGraphableEntry for the Chromium Requests graph/table. + public IDVal Pid => this.pid; + public IDVal TidOpen => this.tid; + public TimestampETW TimeRef => this.timeRef; + public TimestampUI TimeOpen => this.timeStampBeginJob; + public TimestampUI TimeClose => this.timeStampEndJob; + public IStackSnapshot Stack => this.stack; + public XLinkType LinkType => this.xlink.typeNext; + public uint LinkIndex => this.xlink.IFromNextLink; + } // Request + + + /* + A Session contains Streams and refers to a Socket and to a ResolverManager. + A REAL Session can be one of two types: + - HTTP2 / TCP + - HTTP3 / QUIC / UDP + For convenience, we may also fabricate other types: + - HTTP1 / TCP + - CACHE + */ + public class Session : IGCollectable + { + public readonly StreamType Type; + + public string domain; + public string anon_key; // network_anonymization_key + + public ushort port; + + readonly public IDVal pid; + readonly public IDVal tid; + + public UIDVal uidVal; + public int srcdep; + + public int iError; + + public bool fPreMigrate; // This Session is cloned from a migrated Session. + public bool fRecovered; // Recreated after the original Session was not found. + + public Session sessionPreMigrate; // This references the Session that was cloned. + + public TimestampUI timeMigrate; + + public TimestampETW timeReference; + + public ResolverManager resolver; + /* + HTTP2 Sessions can have at most one Socket. + QUIC Sessions can have multiple Sockets if the current Socket degrades, etc. + In that case, we'll Clone the Session so that there is still one active Socket per Session. + */ + public Socket socket; + public Socket socketPreMigrate; + + // Requests waiting to attach to a Stream + public List rgRequestPending; +#if DEBUG + public bool fGathered; +#endif // DEBUG + + public bool Gone { get; set; } + + public int PreKey => (int)this.uidVal; // for FindPreconnectRequest + + /* + QUIC_SESSION.End + HTTP2_SESSION.End + */ + public void Shutdown() + { + AssertImportant(this.Closed); + + this.Gone = true; + if (this.resolver != null) + this.resolver.Gone = true; + } + + public class Stream + { + public uint cbSend; + public uint cbRecv; + public uint cbUpload; // UPLOAD_DATA_STREAM_INIT + public uint cbDownload; // URL_REQUEST_JOB_FILTERED_BYTES_READ + public string strURL; + public string strMethod; + public string strOrigin; + public string strReferer; + public string strDomain; + public string strHTTPStatus; + + // The SEND HEADERS event creates the Stream. + public TimestampUI timeFirst; + // A Stream closes when there is a "fin":true for both Send and Receive events (Headers or Data). + // But for now we just capture the timestamp for every event, last one wins. + public TimestampUI timeLast; + + public Request request; + + public int iError; + public bool fAbandoned; + public bool fIgnore; + public bool fChunkedUpload; // cbUpload is likely invalid + + public uint CbSend() => Math.Max(this.cbSend, this.cbUpload); + public uint CbRecv() => Math.Max(this.cbRecv, this.cbDownload); + public bool HasDataTraffic() => (this.cbSend | this.cbRecv | this.cbUpload | this.cbDownload) != 0; + + public Stream Clone() => (Stream)this.MemberwiseClone(); + + // Mark the last time of an event of interest on this Stream. + public void SetLastTime(TimestampUI timeStamp) + { + AssertImportant(!this.timeLast.HasMaxValue()); + AssertImportant(this.timeLast.Between(this.timeFirst, timeStamp)); + this.timeLast = timeStamp; + } + + /* + Validate the Request and attach it to the Stream with references in both directions. + The caller should set the Session of the Request. + */ + public void Attach(Request req) + { + AssertCritical(req != null); + + if (this.request == req) + { + AssertCritical(req.stream == this); + return; + } +#if DEBUG + AssertImportant(this.request == null); + AssertImportant(req.stream == null); + AssertImportant(req.method?.EndsWith(this.strMethod) ?? false); + AssertImportant(req.Domain?.Equals(this.strDomain) ?? false); + AssertImportant(req.URLScrub?.Equals(this.strURL) ?? false); +#endif // DEBUG + this.request = req; + req.stream = this; + + // Reset past errors, such as from HTTP_TRANSACTION_RESTART_AFTER_ERROR or HTTP_TRANSACTION_RESTART_MISDIRECTED_REQUEST + this.iError = 0; + req.iError = 0; + } + + /* + The Stream was speculative, but it didn't work out, + so mark it as abandoned and null the references. + */ + public void Abandon(bool fHard) + { + // Let the Request refer to a different Stream in the future. + if (this.request != null) + this.request.stream = null; + + // Leave this.request.session for reference in some cases, unless there was no data transferred. + if (fHard || !this.HasDataTraffic()) + { + // Fully abandon this Stream! + this.request = null; + this.fAbandoned = true; + } + } + } // Stream + + + public SortedList rgStream; + + /* + Stream indices are sparse. + Return the Stream, creating a new one (within this Session) if needed. + Update the timestamp. + */ + public Stream EnsureStream(int iStream, in TimestampUI timeStamp) + { + AssertCritical(iStream >= (int)IStreamSpecial.MIN); + + if (this.rgStream.TryGetValue(iStream, out Stream stream) && stream != null) + { + stream.SetLastTime(timeStamp); + return stream; + } + + this.rgStream[iStream] = stream = new Stream() + { + timeFirst = timeStamp, + timeLast = timeStamp + }; + + return stream; + } + + public Session(StreamType _type, in IGenericEvent evt) + { + AssertCritical(_type != StreamType.Unknown); + this.Type = _type; + this.pid = evt.ProcessId; + this.tid = evt.ThreadId; + this.timeReference = evt.Timestamp; + this.rgStream = new(8); // In fact, there could be 10s or 100s of Streams. + if (_type == StreamType.QUIC) + this.rgRequestPending = new(8); // There should be at most a few pending Requests to attach to Streams. + } + + /* + QUIC_SESSION_CLOSED + QUIC_SESSION_CLOSE_ON_ERROR + HTTP2_SESSION_CLOSE + */ + public bool Closed { get; set; } + + public bool FQuic => this.Type == StreamType.QUIC; + + public bool FMigrated => this.sessionPreMigrate != null; + + /* + Get the Remote Address:Port either from the Socket + or by parsing the Address string array[0] from the attached ResolverManager. + Might return null. + */ + public IPEndPoint RemoteAddress() + { + IPEndPoint addrRemote; + if (this.socket == null && this.resolver?.rgstrAddress?.Length > 0) + addrRemote = new(IPAddress.Parse(this.resolver.rgstrAddress[0]), this.port); + else + addrRemote = this.socket?.addrRemote; + + return addrRemote; + } + + /* + Return an error code string from: + - the Stream + - the Stream's Request + - this Session + - "Migrated" + */ + public string Error(Stream stream) + { + // Stream + if (stream.iError != 0) + return Util.ErrorFromI(stream.iError); + + // Request + string status = stream.request?.Error(); + if (!string.IsNullOrEmpty(status)) + return status; + + // Session + if (this.iError != 0) + return Util.ErrorFromI(this.iError); + + if (this.FMigrated) + return "Migrated"; + + if (this.fRecovered) + return "Recovered Session"; + + // Socket + return string.Empty; + } + + /* + When is a Stream not 'valid' for processing? + - fIgnore: It was created only for handshaking overhead. + - fAbandoned: It was speculatively created, then abandoned. + - migrated: It was cloned with a migration event, but this copy saw no action. + */ + public bool ValidStream(Stream stream) + { + AssertImportant(FImplies(stream.fIgnore || stream.fAbandoned, stream.request == null)); + + if (stream.fIgnore) return false; + if (stream.fAbandoned) return false; + + if (this.timeMigrate == stream.timeLast) + { + // This Session migrated (was cloned, and Streams reset), and nothing further happened on this Stream. + AssertImportant(!stream.HasDataTraffic()); + AssertImportant(stream.timeFirst == stream.timeLast); + return false; + } + + return true; + } + + /* + Validate the Socket and attach it to this Session. + */ + public void Attach(Socket sock) + { + if (this.socket == sock) return; +#if DEBUG + AssertCritical(sock.fTCP == !this.FQuic); + if (sock.Type == StreamType.HTTP1) + { + // This is a placeholder HTTP1/TCP Session for algorithmic convenience. + // There is only one Stream (-1), and the Socket may be reused across related HTTP1 Sessions. + AssertImportant(this.rgStream.Count == 1); + AssertImportant(sock.cref == 0); + ++sock.crefH1; + } + else + { + // This is a real HTTP2/TCP or HTTP3/QUIC Session, with potentially multiple Streams. + // The Socket is unique per Session (unless 'migrated' under HTTP3). + AssertImportant(++sock.cref == 1); + } +#endif // DEBUG + AssertImportant(sock.Type == this.Type); + AssertImportant(this.socket == null); + AssertImportant(this.port == sock.addrRemote.Port); + + this.socket = sock; + } + + /* + Add the Request to the QUIC Session's RequestPending list, to later be paired with a Stream. + But do nothing if this Request is already on the RequestPending list or paired with a Stream. + */ + public void AttachQUIC(Request req) + { + // Only attach the Request as pending (to link to a QUIC Stream) when the Session is definitively QUIC. + AssertImportant(req.Type == StreamType.QUIC); + + AssertCritical(req.pid == this.pid); + AssertCritical(req.tid == this.tid); + AssertCritical(this.Type == StreamType.QUIC); + AssertImportant(req.Type == StreamType.Unknown || req.Type == this.Type); + AssertImportant(FImplies(req.FAttachedToStream, req.Type == this.Type)); + + if (req.FAttachedToStream) return; + + if (this.rgRequestPending.FindIndex(r => r == req) >= 0) return; + + this.rgRequestPending.Add(req); + + req.SessionQUIC = this; + } + + /* + After a Request has been matched/attached to a Stream, remove it from the Session's list of pending Requests. + Do session.Finalize with stream.Attach UNLESS the Request came from: session.MatchRequest or this.MatchRequest + */ + public void Finalize(Request req) + { + AssertCritical((this.Type == StreamType.QUIC) == (this.rgRequestPending != null)); + + this.rgRequestPending?.Remove(req); + AssertImportant(req.stream != null); // from Stream.Attach(Request) + + if (this.Type == StreamType.QUIC) + req.SessionQUIC = this; + else + req.SessionHTTP2 = this; + } + + + static readonly string[] rgstrAttrib = + { + ":method", // GET, POST, etc. + ":scheme", ":authority", ":path", // Reconstruct the URL from these three. + "origin", // https://google.com (may be absent) + "referer" // https://google.com/ (may be absent) + }; + + /* + Given a string array derived from JSON "headers" attribute, + reconstruct the original URL from: ':scheme', ':authority', ':path' + */ + static string URLFromHeaders(string[] rgstrHeaders) + { + // rgstrHeaders[] elements correspond to rgstrAttrib[] + AssertCritical(rgstrHeaders.Length >= rgstrAttrib.Length); + + if (rgstrHeaders[1] != null && rgstrHeaders[2] != null && rgstrHeaders[3] != null) + { + System.Text.StringBuilder sb = new(rgstrHeaders[1].Length + rgstrHeaders[2].Length + rgstrHeaders[3].Length + 4); + sb.AppendFormat("{0}://{1}{2}", rgstrHeaders[1], rgstrHeaders[2], rgstrHeaders[3]); + return sb.ToString(); + } + return null; + } + + /* + Certain SEND HEADER events contain information sufficient to construct a new Stream: + Stream id, URL, Method, etc. + */ + public Stream PopulateStreamFromHeader(in IGenericEvent evt) + { + string strHeader = evt.GetParams(); + JsonElement[] rgje = ParseSimpleJsonString(strHeader, ChromiumTable.rgstrStreamId_QStreamId_Headers); + if (rgje == null) return null; + + // stream_id or quic_stream_id or nothing + int iStream = rgje[0].MyGetNumber((int)IStreamSpecial.UNKNOWN); + if (iStream < 0) + iStream = rgje[1].MyGetNumber((int)IStreamSpecial.UNKNOWN); + + Stream stream = this.EnsureStream(iStream, evt.Timestamp.ToGraphable()); + if (stream.strURL == null) + { + string[] rgstrHeaders = rgje[2].MyGetStringArray(rgstrAttrib); + + stream.strURL = URLFromHeaders(rgstrHeaders); + stream.strMethod = rgstrHeaders[0]; + stream.strOrigin = rgstrHeaders[4]; + stream.strReferer = rgstrHeaders[5]; + stream.strDomain = rgstrHeaders[2]; + } + else + { +#if DEBUG + string[] rgstrHeaders = rgje[2].MyGetStringArray(rgstrAttrib); + AssertImportant(stream.strURL == URLFromHeaders(rgstrHeaders)); + AssertImportant(stream.strMethod == rgstrHeaders[0]); + AssertImportant(stream.strOrigin == rgstrHeaders[4]); + AssertImportant(stream.strReferer == rgstrHeaders[5]); + AssertImportant(stream.strDomain == rgstrHeaders[2]); + AssertCritical(stream.request?.Session == this); +#endif // DEBUG + } + + return stream; + } // PopulateStreamFromHeader + + static readonly string[] rgstrHeaderStatus = { ":status" }; + + /* + JSON: {"headers":["status: 200", ...],"stream_id":3} + Get the stream_id and the status values. + Set the Stream's status. + */ + public void SetHTTPStatus(in IGenericEvent evt) + { + string strJSON = evt.GetParams(); + JsonElement[] rgje = ParseSimpleJsonString(strJSON, ChromiumTable.rgstrStreamId_Headers); + if (rgje == null) return; + + string[] rgstrStatus = rgje[1].MyGetStringArray(rgstrHeaderStatus); + AssertImportant(rgstrStatus?.Length == 1); + if (!(rgstrStatus?.Length > 0)) return; + + int iStream = rgje[0].MyGetNumber(-1); + AssertCritical(iStream >= 0); + if (iStream < 0) return; + + Stream stream = this.EnsureStream(iStream, evt.Timestamp.ToGraphable()); + + stream.strHTTPStatus = rgstrStatus[0]; + } + + + public int LookupPendingRequestByURL(string strURL) => this.rgRequestPending.FindIndex(r => r.URLScrub.Equals(strURL)); + + /* + Given a recently created Stream, find the corresponding Request from the Pending Request list. + Find it by matching the 'scrubbed' URL (with no #fragment identifier). + Remove the Request from the list and return it. + */ + public Request MatchRequest(Stream stream) + { + if (stream.request != null) + return stream.request; + + int iReq = this.LookupPendingRequestByURL(stream.strURL); + if (iReq < 0) return null; + + Request req = this.rgRequestPending[iReq]; + + AssertCritical(req.pid == this.pid); + AssertCritical(req.tid == this.tid); + AssertImportant(req.method.EndsWith(stream.strMethod)); // "REDIRECT/GET" === "GET" + + // It's no longer pending, so remove it. + this.rgRequestPending.RemoveAt(iReq); + + // Was there also another potential match!? + AssertImportant(this.LookupPendingRequestByURL(stream.strURL) < 0); + + return req; + } + + + /* + Create a Deep Clone for migrating the Session. + A QUIC Session will need to migrate in the rare case when its UDP Socket degrades and is replaced with a new one. + (In the HTTP2 case the TCP Socket is tied to the Session, so a new one should be created automatically if needed.) + Create and return a pre-migration clone of the Session. + Reset the Streams of the original Session and give it the new, post-migration Socket. + */ + public Session Migrate(in TimestampUI timeStamp) + { + AssertCritical(this.FQuic); + AssertCritical(this.socketPreMigrate != null); + + Session sessionBefore = (Session)this.MemberwiseClone(); // shallow + var rgStreamBefore = sessionBefore.rgStream = new SortedList(this.rgStream); // shallow + /* + This Session is picking up where the original left off. + Reset the byte counts so that the old and new Sockets each get correct xmission attribution. + Request.stream will continue to refer to the elements of this.rgStream, not rgStreamBefore. + */ + foreach (var kvp in this.rgStream) + { + Stream stream = kvp.Value; + rgStreamBefore[kvp.Key] = stream.Clone(); + + stream.cbSend = stream.cbRecv = 0; + stream.cbUpload = stream.cbDownload = 0; + stream.iError = 0; + stream.timeFirst = stream.timeLast = timeStamp; + AssertImportant(!stream.HasDataTraffic()); + } + + this.socket = this.socketPreMigrate; + this.socketPreMigrate = sessionBefore.socket; + this.timeMigrate = timeStamp; + + sessionBefore.socketPreMigrate = this.socket; + sessionBefore.fPreMigrate = true; + + sessionBefore.sessionPreMigrate = this.sessionPreMigrate; // null unless multiple migrations + this.sessionPreMigrate = sessionBefore; + + return sessionBefore; + } + + /* + Do this work after parsing and before 'gathering'. + + When a QUIC Socket's connection degrades, a new one spins up to replace it. + If the new Socket successfully connects then a Migration event is triggered. + Since NetBlame is very Socket-oriented, we want to track both the old and the new Sockets. + + We created a copy of this Session (this.sessionPreMigrate) and its Streams. + Then we updated the Socket on this Session and zeroed cbSend/Recv on its Streams. + (Note that multiple migrations could theoretically happen on a Session.) + + Now we need to adjust the Requests which refer to one or both of these Sessions/Streams. + (This is important for the Request-oriented 'Chromium Requests' graph/table.) + There are three cases: + - A migrated Request refers back to the pre-migration Stream copy because nothing happened on it post-migration. + - A migrated Request remains with the post-migration Stream because not much happened on it pre-migration (cbSend/Recv==0). + - A migrated Request gets cloned to refer to both the pre- and post-migration Streams because there was activity both pre- and post-migration. + */ + public void AdjustForMigration(Chromium.ChromiumTable requestTable) + { + // this: the Session & Streams (with xfer byte counts) after the migration (about to be 'gathered' for charting) + // session2: the Session & Streams (with xfer byte counts) before the migration (not yet 'gathered' for charting) + var session2 = this.sessionPreMigrate; + AssertCritical(session2.fPreMigrate); + AssertCritical(this.timeMigrate.HasValue()); + AssertCritical(this.rgStream.Count >= session2.rgStream.Count); + AssertImportant(this.Type == StreamType.QUIC && session2.Type == StreamType.QUIC); +#if DEBUG + AssertImportant(!this.fGathered); + AssertImportant(!session2.fGathered); +#endif // DEBUG + foreach (var kvp in session2.rgStream) + { + // stream2: the Stream (with xfer byte counts) before the migration + Stream stream2 = kvp.Value; + AssertImportant(stream2 != null); + if (stream2 == null) continue; + + if (this.rgStream.TryGetValue(kvp.Key, out Stream stream)) + { + // stream: the Stream (with xfer byte counts) after the migration + AssertCritical(stream != stream2); + AssertCritical(stream.fIgnore == stream2.fIgnore); + AssertCritical(stream.fAbandoned == stream2.fAbandoned); + + if (stream.fAbandoned || stream.fIgnore) continue; + + // Both the 'before' and 'after' Streams refer to the same Request at this point. + // But we may or may not need two Streams and two Requests, depending on whether activity happened before the migration, after, or both. + AssertCritical(stream.request == stream2.request); + + Request request = stream.request; + if (request == null) continue; + AssertImportant(request.stream == stream); + if (!request.FAttachedToStream) continue; + + AssertImportant(request.Session == this); + AssertImportant(request.Session != session2); + AssertImportant(stream2.request == request); + + stream2.request = null; // stream2.Attach sets this. + + if (this.timeMigrate >= stream.timeLast) + { + // There was no further action on this Stream after the migration. + // Refer the Request back to the earlier copy of the Stream. Invalidate the later copy. + + AssertCritical(!this.ValidStream(stream)); + AssertImportant(session2.ValidStream(stream2)); + + stream.request = null; + stream.fAbandoned = true; + + request.SessionReset(); + stream2.Attach(request); + request.SessionSet = session2; + } + else if (!stream2.HasDataTraffic() && stream.HasDataTraffic()) + { + // The earlier copy of the Stream had no data transfer, but the migrated one did. + // Just leave the Request referencing the migrated Stream. + + AssertImportant(this.ValidStream(stream)); + AssertCritical(!session2.ValidStream(stream2)); + + stream2.fAbandoned = true; + } + else + { + // Both copies of the Stream saw activity, before and after the Migration. + // There will need to be separate Requests to reference each Stream. + // WPA's Chromium Requests table does random access of the Requests array, which needs to reference every active Stream. + + AssertImportant(this.ValidStream(stream)); + AssertImportant(session2.ValidStream(stream2)); +#if DEBUG + AssertImportant(!stream.request.fGathered); // else this is too late! +#endif // DEBUG + // Clone the original Request and adjust some values. + Request request2 = request.Migrate(this.timeMigrate); + requestTable.Add(request2); + + stream2.Attach(request2); + request2.SessionSet = session2; + } + } + else + { + // Every Stream in session2 should have a corresponding Stream in session1. + AssertImportant(false); + } + } // foreach kvp + } // AdjustForMigration + } // Session + + + public class Hash : Dictionary where T : class + { + public Hash(int c) : base(c) { } + public bool HasValue(K key) => base.TryGetValue(key, out T t) && (t != default); + + public void Reset(K key) { base[key] = default; } + + // override the indexer for nothrow + public new T this[K key] + { + set => base[key] = value; + get + { + if (base.TryGetValue(key, out T t)) return t; + + return default; + } + } + } // Hash + + + /* + There is a single Chromium pipeline thread which does all of the work of interest to us. + But several of those threads could show up in one ETW session. + There is one ThreadLocal object for each of them. + */ + public class ThreadLocal + { + public Session sessionRecent; + + public Request reqRecent; + + public Socket sockRecent; + + public UIDVal idRecent; + + public string strTaskRecent; + + public string strJSON; + + public List rgRequestStalled; + + // The ChromiumTable base class is List + + readonly Hash reqFromUID; + + readonly Hash reqFromSrcDep; + + // ChromiumTable.socketTable is List + + public readonly Hash sockFromUID; + + public readonly Hash sockFromSrcDep; + + // ChromiumTable.sessionTable is List + + public readonly Hash sessionFromUID; + + public readonly Hash sessionFromSrcDep; + + // ResolverManager is not stored as a List. + + public readonly Hash managerFromUID; + + public readonly Hash managerFromSrcDep; + + // IsDNS: HashSet of srcdep values of DNS-related Sockets + + public readonly HashSet IsDNS; + + public ThreadLocal(int c) + { + this.rgRequestStalled = new(4); + this.reqFromSrcDep = new(c); + this.reqFromUID = new(c); + this.sockFromSrcDep = new(c / 4); + this.sockFromUID = new(c / 4); + this.managerFromSrcDep = new(c / 8); + this.managerFromUID = new(c / 4); + this.sessionFromSrcDep = new(c / 8); + this.sessionFromUID = new(c / 4); + this.IsDNS = new(c / 8); + } + + public void SetReqUID(Request req, UIDVal uid) + { + this.reqFromUID[uid] = req; +#if DEBUG + req?.AddUID(uid); +#endif + } + + public Request ReqFromUID(UIDVal uid) => this.reqFromUID[uid]; + + public void SetReqSrcDep(Request req, int srcdep) + { + this.reqFromSrcDep[srcdep] = req; +#if DEBUG + req?.AddSrcDep(srcdep); +#endif // DEBUG + } + + public Request ReqFromSrcDep(int srcdep) => this.reqFromSrcDep[srcdep]; + + /* + RECENT: There are certain events which are ALWAYS emitted near/adjacent to each other (for a given pipelining thread). + We can use the RECENT mechanism to pass data across these events, confirming the expected task name. + */ + public Request GetRecent(string strTask) + { + if (!strTask.Equals(this.strTaskRecent)) return null; + + return this.reqRecent; + } + + public void SetRecent(Request req, string strTask) + { + this.sessionRecent = null; + this.reqRecent = req; + this.idRecent = 0; + this.strTaskRecent = strTask; + } + + public void SetRecent(Socket sock, string strTask) + { + this.sessionRecent = null; + this.sockRecent = sock; + this.idRecent = 0; + this.strTaskRecent = strTask; + } + + public void SetRecent(Session session, string strTask) + { + this.sessionRecent = session; + this.idRecent = 0; + this.strTaskRecent = strTask; + } + + public void SetRecentUID(UIDVal uID, string strTask) + { + this.sessionRecent = null; + this.reqRecent = null; + this.sockRecent = null; + this.idRecent = uID; + this.strTaskRecent = strTask; + } + + public UIDVal GetRecentUID(string strTask) + { + if (!strTask.Equals(this.strTaskRecent)) return 0; + + return this.idRecent; + } + + public void ResetRecent() + { + this.sessionRecent = null; + this.reqRecent = null; + this.sockRecent = null; + this.strTaskRecent = null; + this.strJSON = null; + this.idRecent = 0; + } + + +#if DEBUG + // more frequent collection + private int cGC = 64; +#else // DEBUG + // less frequent collection + private int cGC = 1024; +#endif // DEBUG + + /* + Occasionally remove lookups to closed Requests, ResolverManagers, Sessions, and Sockets + */ + public void GarbageCollect() + { + // Test cGC against what's probably the largest data structure. + if (this.reqFromSrcDep.Count < cGC) return; + + this.reqFromSrcDep.DoGC(); + this.reqFromUID.DoGC(); + + this.managerFromSrcDep.DoGC(); + this.managerFromUID.DoGC(); + + this.sessionFromSrcDep.DoGC(); + this.sessionFromUID.DoGC(); + + this.sockFromSrcDep.DoGC(); + this.sockFromUID.DoGC(); + + // Do this again when the count gets larger. + cGC = reqFromSrcDep.Count * 2; + } + + } // ThreadLocal + + + public class ChromiumTable : List + { + readonly AllTables allTables; + + public readonly List socketTable; + + public readonly List sessionTable; + + readonly Hash threadLocal; + + readonly HashSet unhandled; + + public ChromiumTable(int capacity, in AllTables _allTables) : base(capacity) + { + this.allTables = _allTables; + this.socketTable = new(capacity / 4); + this.sessionTable = new(capacity / 8); + this.threadLocal = new(8); // a few active Chromium Network threads + this.unhandled = new(64); + } + + + static uint KeyFromThread(IDVal pid, IDVal tid) => (uint)((pid << 16) ^ pid ^ tid); + + ThreadLocal ThreadLocalFromEvt(in IGenericEvent evt) => this.threadLocal[KeyFromThread(evt.ProcessId, evt.ThreadId)]; + + ThreadLocal EnsureThreadLocal(in IGenericEvent evt) + { + uint kThread = KeyFromThread(evt.ProcessId, evt.ThreadId); + ThreadLocal tl = this.threadLocal[kThread]; // no throw + if (tl == default) + { + tl = new ThreadLocal(8192); // a medium-large data-set + this.threadLocal[kThread] = tl; + } + return tl; + } + + void ResetRecent(IDVal pid, IDVal tid) + { + ThreadLocal tl = this.threadLocal[KeyFromThread(pid, tid)]; + if (tl != default) + tl.ResetRecent(); + } + + void AttachWinsockConnection(in IGenericEvent evt, WinsockAFD.IPPROTO ip) + { + Socket soc = this.SocketFromUID(in evt); + if (soc == null) return; + + AssertImportant(!soc.Closed); + AssertImportant(soc.timeStampConnect.ToNanoseconds != 0); + + // An AfdCreate (Winsock) event occurred between: TCP_CONNECT.Begin & TCP_CONNECT_ATTEMPT.Begin + // That created a new Winsock table entry. + + AssertImportant(this.allTables.wsTable.Count > 0); + + IDVal tid = evt.ThreadId; + WinsockAFD.Connection cxn = this.allTables.wsTable.FindLast(c => c.tidOpen == tid); + + AssertImportant(cxn != null); + if (cxn == null) + return; + + AssertImportant(!cxn.FClosed); + AssertImportant(cxn.timeCreate > soc.timeStampConnect); + + if (cxn.timeCreate < soc.timeStampConnect) + return; + + AssertImportant(cxn.ipProtocol == ip); + AssertImportant(cxn.grbitType == (byte)Protocol.Winsock); + + // Once a Winsock Connection is marked with Chromium, + // its Socket must be 'gathered' via: GatherChromium() + + cxn.grbitType |= (byte)Protocol.Chromium; + + soc.cxn = cxn; + } + + /**** SOCKETS *****/ + + Socket SocketFromUID(in IGenericEvent evt, bool fOpen = true) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + UIDVal id = evt.GetUID(); + Socket sock = tl.sockFromUID[id]; + if (sock == null) return null; + + AssertCritical(sock.pid == evt.ProcessId && sock.tid == evt.ThreadId); + + if (fOpen && sock.Closed) return null; + + tl.SetRecent(sock, evt.TaskName); + + return sock; + } + + Socket SocketFromSrcDep(int srcdep, in IGenericEvent evt) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + AssertCritical(srcdep != jsonIntDefault); + + Socket sock = tl.sockFromSrcDep[srcdep]; + if (sock == null) return null; + + AssertCritical(sock.pid == evt.ProcessId && sock.tid == evt.ThreadId); + + UIDVal id = evt.GetUID(); + tl.sockFromUID[id] = sock; + tl.SetRecent(sock, evt.TaskName); + + return sock; + } + + Socket SocketFromSrcDep(in IGenericEvent evt) => SocketFromSrcDep(evt.GetSourceId(), in evt); + + Socket SocketFromRecent(in IGenericEvent evt, string strTask) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + // Maybe, maybe not... + AssertImportant(strTask.Equals(tl.strTaskRecent)); + + if (!strTask.Equals(tl.strTaskRecent)) return null; + + tl.strTaskRecent = evt.TaskName; + + UIDVal id = evt.GetUID(); + tl.sockFromUID[id] = tl.sockRecent; + + return tl.sockRecent; + } + + void SocketAttachUID(Socket sock, UIDVal uID, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + tl.sockFromUID[uID] = sock; + } + + void SocketAttachSrcDep(Socket sock, int srcdep, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + tl.sockFromSrcDep[srcdep] = sock; + } + + void SocketAttachUID_SrcDep(Socket sock, int srcdep, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); +#if DEBUG + Socket sockT = tl.sockFromSrcDep[srcdep]; + + // sockT must be null or the same or closed (reused srcdep!?) or error (retrying). + AssertImportant(sockT == null || sockT == sock || sockT.Closed || sockT.iError != 0); + AssertCritical(sock != null); + AssertCritical(sock.pid == evt.ProcessId && sock.tid == evt.ThreadId); +#endif // DEBUG + if (srcdep != jsonIntDefault) + tl.sockFromSrcDep[srcdep] = sock; + + UIDVal id = evt.GetUID(); + tl.sockFromUID[id] = sock; + tl.SetRecent(sock, evt.TaskName); + } + + void SocketAttachUID_SrcDep(Socket sock, in IGenericEvent evt) => SocketAttachUID_SrcDep(sock, evt.TryGetSourceId(), in evt); + + /* + RECENT: There are certain events which are ALWAYS emitted near/adjacent to each other (for a given pipelining thread). + We can use the RECENT mechanism to pass data across these events, confirming the expected task name. + */ + UIDVal GetRecentUID(string strTaskName, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + return tl.GetRecentUID(strTaskName); + } + + void SetRecentUID(in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + tl.SetRecentUID(evt.GetUID(), evt.TaskName); + } + + /**** REQUESTS ****/ + + /* + Return the most recent Request which likely matches the given Stream. + */ + Request MatchRequest(IDVal pid, IDVal tid, Session.Stream stream) + { + Request req = this.FindLast(r => r == stream.request || (r.pid == pid && r.tid == tid && r.URLScrub.Equals(stream.strURL))); + AssertImportant(req?.method?.EndsWith(stream.strMethod) ?? true); + return req; + } + + + [Conditional("DEBUG")] + void AssertValidRequest(Request req, in IGenericEvent evt) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + Request reqT = tl?.ReqFromUID(evt.GetUID()); + AssertImportant(reqT == null || reqT == req || reqT.Closed); + AssertCritical(req != null); + AssertCritical(req.pid == evt.ProcessId && req.tid == evt.ThreadId); + } + + void RequestAttachUID(Request req, UIDVal uid, in IGenericEvent evt) + { + this.AssertValidRequest(req, in evt); + + ThreadLocal tl = EnsureThreadLocal(in evt); + tl.SetReqUID(req, uid); + tl.SetRecent(req, evt.TaskName); + } + + void RequestAttachUID(Request req, in IGenericEvent evt) => RequestAttachUID(req, evt.GetUID(), in evt); + + void RequestAttachSrcDep(Request req, int srcdep, in IGenericEvent evt) + { + this.AssertValidRequest(req, in evt); + AssertCritical(srcdep != jsonIntDefault); + + ThreadLocal tl = EnsureThreadLocal(in evt); + tl.SetReqSrcDep(req, srcdep); + tl.SetRecent(req, evt.TaskName); + } + + void RequestAttachSrcDep(Request req, in IGenericEvent evt) => RequestAttachSrcDep(req, evt.GetSourceId(), in evt); + + void RequestAttachUID_SrcDep(Request req, in IGenericEvent evt) + { + this.AssertValidRequest(req, in evt); + + int srcdep = evt.GetSourceId(); + AssertCritical(srcdep != jsonIntDefault); + + ThreadLocal tl = EnsureThreadLocal(in evt); + tl.SetReqSrcDep(req, srcdep); + tl.SetReqUID(req, evt.GetUID()); + tl.SetRecent(req, evt.TaskName); + } + + // Most events should invoke one of these three methods, with: + // ID (in the event), ID + SrcDep, ID + Task Name to correlate + + Request RequestFromUID(UIDVal uID, in IGenericEvent evt, bool fOpen = true) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + Request req = tl.ReqFromUID(uID); + if (req != null) + { + AssertCritical(req.pid == evt.ProcessId && req.tid == evt.ThreadId); + + if (fOpen && req.Closed) + req = null; + } + + tl.SetRecent(req, evt.TaskName); + + return req; + } + + Request RequestFromUID(in IGenericEvent evt, bool fOpen = true) => RequestFromUID(evt.GetUID(), in evt, fOpen); + + Request RequestFromSrcDep(int srcdep, in IGenericEvent evt) + { + AssertCritical(srcdep != jsonIntDefault); + + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + return tl.ReqFromSrcDep(srcdep); + } + + Request RequestFromSrcDep(in IGenericEvent evt) => RequestFromSrcDep(evt.GetSourceId(), in evt); + + Request RequestFromUID_SrcDep(int srcdep, in IGenericEvent evt) + { + AssertCritical(srcdep != jsonIntDefault); + + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + Request req = tl.ReqFromSrcDep(srcdep); + if (req != null) + { + AssertCritical(req.pid == evt.ProcessId && req.tid == evt.ThreadId); + + if (req.Closed) + req = null; + } + + UIDVal uid = evt.GetUID(); + + if (req == null) + { + req = tl.ReqFromUID(uid); + if (req != null) + tl.SetReqSrcDep(req, srcdep); + } + else + { +#if DEBUG + Request reqT = tl.ReqFromUID(uid); + AssertImportant(FImplies(reqT != null, reqT == req)); +#endif // DEBUG + tl.SetReqUID(req, uid); + } + + tl.SetRecent(req, evt.TaskName); + + return req; + } + + Request RequestFromUID_SrcDep(in IGenericEvent evt) => RequestFromUID_SrcDep(evt.GetSourceId(), in evt); + + public void StashStalledRequest(Request req, in IGenericEvent evt, string strTask) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return; + + req.strTaskStash = strTask; + tl.rgRequestStalled.Add(req); + } + + public Request GetStalledRequestGroup(int hidGroup, in IGenericEvent evt, string strTask) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + int iReq = tl.rgRequestStalled.FindIndex(r => r.hidGroup == hidGroup && strTask.Equals(r.strTaskStash)); + if (iReq < 0) return null; + + Request req = tl.rgRequestStalled[iReq]; + req.strTaskStash = null; + tl.rgRequestStalled.RemoveAt(iReq); + return req; + } + + /**** Preconnect Requests ****/ + + /* + Find a Preconnect Request created or chosen by: PreconnectRequest() + Look it up by URL and Session. + (This is the best we've got in the case of Preconnect Requests.) + Also, the Session must match, or pick up a recently created Request with all the right parameters, and the caller gives it this Session. + session may be null. + + The key is a function of the Session (HTTP2 / HTTP3) or of the hashed group_id (HTTP1). + Therefore the caller must set (or check) the Session or the hidGroup of the returned Request. + + strURL may or may not have a trailing '/'. + */ + Request _FindPreconnectRequest(string strURL, int prekey, in IGenericEvent evt) + { + AssertCritical(strURL != null); + + // Typically we will find first a speculative preconnect Request with: .PreKey==0, + // then an active preconnect Request with: .PreKey=something + + // First find the Request where the Session is probably null (which may be our target). + IDVal pid = evt.ProcessId; + IDVal tid = evt.ThreadId; + int iReq = this.FindLastIndex(r => r.IsPreconnect && r.pid == pid && r.tid == tid && r.URL.Equal2(strURL)); + if (iReq < 0) return null; + + Request req1 = this[iReq]; + int prekeyReq = req1.PreKey(); + if (prekeyReq == prekey) return req1; + if (prekeyReq != 0) return null; + if (iReq == 0) return req1; + + // Continue the search + iReq = this.FindLastIndex(iReq-1, iReq/*count*/, r => r.IsPreconnect && r.pid == pid && r.tid == tid && r.URL.Equal2(strURL)); + if (iReq < 0) return req1; // req1.PreKey == 0 + + Request req2 = this[iReq]; + if (req2.PreKey() == prekey) return req2; + + return req1; // req1.PreKey == 0 + } + + Request FindPreconnectRequest(string strURL, int prekey, in IGenericEvent evt) + { + Request req = _FindPreconnectRequest(strURL, prekey, in evt); + + // If there is start time (from an earlier PreconnectRequest) then advance the end time. + if (req?.timeStampBeginJob.HasValue() ?? false) + { + TimestampUI timeStamp = evt.Timestamp.ToGraphable(); + AssertImportant(req.timeStampEndJob.HasMaxValue() || req.timeStampEndJob < timeStamp); + req.timeStampEndJob = timeStamp; + } + + return req; + } + + /* + HTTP_STREAM_JOB_CONTROLLER.Begin : "is_preconnect":true + Find or create a preconnect placeholder Request. + Many such events can refer to the same preconnect Request. + Find it later via: FindPreconnectRequest() + + The Requests found or created here are "unused" because: Request.PreKey==0 + This unused Request gets picked up for use later when the key requested is different. + */ + Request PreconnectRequest(string strURL, in IGenericEvent evt, in Thread.ThreadTable threadTable) + { + AssertCritical(strURL != null); + + // If the (unused) Request is found, it was previously created here. + + Request req = this.FindPreconnectRequest(strURL, 0, in evt); + + if (req == null) + { + req = new Request(strURL, in evt) + { + method = Request.strPreconnect, + priority = Priority.IDLE + }; + + this.Add(req); + } + else + { + req.timeRef = evt.Timestamp; + req.stack = evt.Stack; + } + + // This Request didn't get picked up for use before (via FindPreconnectRequest). + // Speculatively assume that this time it will. + // But later, skip 'gathering' this Request if IsSpeculative (it never got used). + + UIDVal uid = evt.GetUID(); + req.uidRequest = uid; + this.RequestAttachUID(req, uid, in evt); + + if (!req.timeStampBeginJob.HasValue()) + { + req.timeStampBeginJob = evt.Timestamp.ToGraphable(); + req.xlink.ReGetLink(evt.ThreadId, evt.Timestamp.ToGraphable(), in threadTable); + } + + return req; + } + + + /**** Recent / Correlate ****/ + + Request RequestFromUID_Correlate(in IGenericEvent evt, string strTask) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + UIDVal uid = evt.GetUID(); + Request req = tl.ReqFromUID(uid); + if (req == null) + { + if (!strTask.Equals(tl.strTaskRecent)) return null; + + req = tl.reqRecent; + tl.SetReqUID(req, uid); + } + else + { + AssertImportant(tl.reqRecent == null || tl.reqRecent == req || !strTask.Equals(tl.strTaskRecent)); + } + + tl.SetRecent(req, evt.TaskName); + + return req; + } + + Request RequestFromGroupId_Correlate(in IGenericEvent evt, int hidGroup, string strTask, string strTask2 = null) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + Request req = tl.GetRecent(strTask); + if (req == null && strTask2 != null) + req = tl.GetRecent(strTask2); + + if (req == null) + req = this.GetStalledRequestGroup(hidGroup, in evt, strTask); + + if (req != null && req.hidGroup != hidGroup && req.hidGroup != 0) + req = null; + + return req; + } + + + void StashJSON(in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + tl.strJSON = evt.GetParams(); + tl.SetRecent(default(Request), evt.TaskName); + } + + string UnstashJSON(string strTask, in IGenericEvent evt) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + string strJSON = tl.strJSON; + tl.strJSON = null; + + string strTaskRecent = tl.strTaskRecent; + + tl.SetRecent(default(Request), evt.TaskName); + + if (!strTask.Equals(strTaskRecent)) + return null; + + return strJSON; + } + + /**** SESSIONS ****/ + + Session SessionFromUID(UIDVal id, in IGenericEvent evt, bool fOpen = true) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + Session session = tl.sessionFromUID[id]; + if (session == null) return null; + + AssertCritical(session.pid == evt.ProcessId && session.tid == evt.ThreadId); + + if (fOpen && session.Closed) return null; + + return session; + } + + Session SessionFromUID(in IGenericEvent evt, bool fOpen = true) => SessionFromUID(evt.GetUID(), in evt, fOpen); + + Session SessionFromSrcDep(int srcdep, in IGenericEvent evt) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + AssertCritical(srcdep != jsonIntDefault); + + Session session = tl.sessionFromSrcDep[srcdep]; + if (session == null) return null; + + AssertCritical(session.pid == evt.ProcessId && session.tid == evt.ThreadId); + AssertImportant(!session.Closed); + + UIDVal id = evt.GetUID(); + tl.sessionFromUID[id] = session; + + return session; + } + + Session SessionFromSrcDep(in IGenericEvent evt) => SessionFromSrcDep(evt.GetSourceId(), in evt); + + Session SessionFromRecent(string taskRecent, in IGenericEvent evt) + { + ThreadLocal tl = ThreadLocalFromEvt(in evt); + if (tl == default) return null; + + if (tl.strTaskRecent != taskRecent) return null; + + return tl.sessionRecent; + } + + void SessionAddUID(Session session, UIDVal uID, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + tl.sessionFromUID[uID] = session; + } + + void SessionAddUID(Session session, in IGenericEvent evt) => SessionAddUID(session, evt.GetUID(), in evt); + + void SessionAttachUID(Session session, UIDVal uID, in IGenericEvent evt) + { + SessionAddUID(session, uID, in evt); + AssertImportant(session.uidVal == 0 || session.uidVal == uID); + session.uidVal = uID; + } + + void SessionAttachUID(Session session, in IGenericEvent evt) => SessionAttachUID(session, evt.GetUID(), in evt); + + void SessionAttachSrcDep(Session session, int srcdep, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + AssertCritical(srcdep != jsonIntDefault); + AssertImportant(session.srcdep == 0 || session.srcdep == srcdep); + session.srcdep = srcdep; + tl.sessionFromSrcDep[srcdep] = session; + tl.SetRecent(session, evt.TaskName); + } + + void SessionAddSrcDep(Session session, int srcdep, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + AssertCritical(srcdep != jsonIntDefault); + tl.sessionFromSrcDep[srcdep] = session; + } + + void SessionAddSrcDep(Session session, in IGenericEvent evt) => SessionAddSrcDep(session, evt.GetSourceId(), in evt); + + void SessionAddUID_SrcDep(Session session, int srcdep, UIDVal uid, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); +#if DEBUG + Session sessionT = tl.sessionFromSrcDep[srcdep]; + + // sessionT must be null or the same or closed (reused srcdep!?) or error (retrying). + AssertImportant(sessionT == null || sessionT == session || sessionT.Closed || sessionT.iError != 0); + AssertCritical(session != null); + AssertCritical(session.pid == evt.ProcessId && session.tid == evt.ThreadId); + AssertCritical(srcdep != jsonIntDefault); +#endif // DEBUG + tl.sessionFromSrcDep[srcdep] = session; + tl.sessionFromUID[uid] = session; + tl.SetRecent(session, evt.TaskName); + } + + void SessionAddUID_SrcDep(Session session, in IGenericEvent evt) => SessionAddUID_SrcDep(session, evt.GetSourceId(), evt.GetUID(), in evt); + + void SessionAttachUID_SrcDep(Session session, int srcdep, in IGenericEvent evt) + { + UIDVal uID = evt.GetUID(); + SessionAddUID_SrcDep(session, srcdep, uID, in evt); + AssertImportant(session.srcdep == 0 || session.srcdep == srcdep); + session.srcdep = srcdep; + AssertImportant(session.uidVal == 0 || session.uidVal == uID); + session.uidVal = uID; + } + + void SessionAttachUID_SrcDep(Session session, in IGenericEvent evt) => SessionAttachUID_SrcDep(session, evt.GetSourceId(), in evt); + + + /**** ResolverManager ****/ + + void ResolverManagerAttach(ResolverManager resolver, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + UIDVal uid = evt.GetUID(); + tl.managerFromUID[uid] = resolver; + } + + ResolverManager ResolverManagerAttach(UIDVal uid, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + ResolverManager manager = tl.managerFromUID[uid]; + if (manager != null) + { + UIDVal uid2 = evt.GetUID(); + tl.managerFromUID[uid2] = manager; + } + return manager; + } + + ResolverManager ResolverManagerAttachSrcDep(int srcdep, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + UIDVal uid = evt.GetUID(); + ResolverManager manager = tl.managerFromUID[uid]; + if (manager != null) + tl.managerFromSrcDep[srcdep] = manager; + + return manager; + } + + ResolverManager GetResolverManagerFromSrcDep(int srcdep, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + return tl.managerFromSrcDep[srcdep]; + } + + ResolverManager GetResolverManager(UIDVal uid, in IGenericEvent evt) + { + ThreadLocal tl = EnsureThreadLocal(in evt); + return tl.managerFromUID[uid]; + } + + ResolverManager GetResolverManager(in IGenericEvent evt) => GetResolverManager(evt.GetUID(), in evt); + + + /**** DNS ****/ + + // Is the given Source Dependency value related to a DNS Transaction? + + bool IsDNSSrcDep(in IGenericEvent evt, int srcdep) => ThreadLocalFromEvt(in evt)?.IsDNS.Contains(srcdep) ?? false; + + bool AddDNSSrcDep(in IGenericEvent evt, int srcdep) => ThreadLocalFromEvt(in evt)?.IsDNS.Add(srcdep) ?? false; + + /**** GC ****/ + + void GarbageCollect(in IGenericEvent evt) + { + ThreadLocal tl = this.ThreadLocalFromEvt(in evt); + tl?.GarbageCollect(); + } + + /**** STRING ARRAYS FOR ParseSimpleJsonString ****/ + + public static readonly string[] rgstrSourceId = + { + "/source_dependency/id" // "source_dependency":{"id":123} (number) + }; + + public static readonly string[] rgstrUrl = + { + "url" + }; + + static readonly string[] rgstrURL_Method = + { + rgstrUrl[0], // url + "method" + }; + + static readonly string[] rgstrPriority = + { + "priority" + }; + + static readonly string[] rgstrURL_Priority = + { + rgstrUrl[0], // url + rgstrPriority[0] // priority + }; + + static readonly string[] rgstrDestination_SourceId = + { + "destination", // (domain) + rgstrSourceId[0] // "source_dependency":{"id":123} + }; + + static readonly string[] rgstrSourceId_Type_Destination_Quic = + { + rgstrSourceId[0], // "source_dependency":{"id":123} + "type", + rgstrDestination_SourceId[0], // "destination" (domain) + "using_quic" // (bool) + }; + + static readonly string[] rgstrAddress = + { + "address" + }; + + static readonly string[] rgstrAddress_Error = + { + rgstrAddress[0], // "address" + Util.rgstrNetError[0] // "net_error" + }; + + static readonly string[] rgstrSourceId_Address = + { + rgstrSourceId[0], // "source_dependency":{"id":123} + rgstrAddress[0], // "address" + }; + + static readonly string[] rgstrLocal_Remote = + { + "local_address", + "remote_address" + }; + + static readonly string[] rgstrCanon_Endpoint = + { + "/results/canonical_names", + "/results/ip_endpoints" + }; + + static readonly string[] rgstrGroupId = + { + "group_id" + }; + + static readonly string[] rgstrHost = + { + "host" + }; + + static readonly string[] rgstrHost_Key = + { + rgstrHost[0], // host + "network_anonymization_key" + }; + + static readonly string[] rgstrHost_Key_Port = + { + rgstrHost_Key[0], // host + rgstrHost_Key[1], // network_anonymization_key + "port" + }; + + static readonly string[] rgstrHost_Key_Port_SourceId = + { + rgstrHost_Key_Port[0], // "host" + rgstrHost_Key_Port[1], // "network_anonymization_key" + rgstrHost_Key_Port[2], // "port" + rgstrSourceId[0] // "source_dependency":{"id":123} + }; + + static readonly string[] rgstrSize = + { + "size" + }; + + static readonly string[] rgstrPeer_Self = + { + "peer_address", // "remote address" + "self_address" // "local address" + }; + + static readonly string[] rgstrStreamId = + { + "stream_id" + }; + + static readonly string[] rgstrStreamId_Payload = + { + rgstrStreamId[0], // "stream_id" + "payload_length" + }; + + static readonly string[] rgstrStreamId_Size = + { + rgstrStreamId[0], // "stream_id" + rgstrSize[0] // "size" + }; + + public static readonly string[] rgstrStreamId_QStreamId_Headers = + { + rgstrStreamId[0], // "stream_id" + "quic_stream_id", + "headers", + }; + + public static readonly string[] rgstrStreamId_Headers = + { + rgstrStreamId[0], // "stream_id" + "headers" + }; + + public static readonly string[] rgstrProto = + { + "proto" + }; + + public static readonly string[] rgstrProto2 = + { + "next_proto" + }; + + public static readonly string[] rgstrSize_Chunked_Error = + { + "total_size", + "is_chunked", + Util.rgstrNetError[0] // "net_error" + }; + + public static readonly string[] rgstrBytes = + { + "byte_count" + }; + + public static readonly string[] rgstrUrl_Preconnect = + { + rgstrUrl[0], // "url" + "is_preconnect" // boolean + }; + + public static readonly string[] rgstrOpcode_Payload = + { + "opcode", + "payload_length" + }; + +#if DEBUG + IDVal s_tidUnique; + + long nsStartDB = 0_000000000; +#endif // DEBUG + + const int FAILED = -2; + const int QUIC_NETWORK_IDLE_TIMEOUT = 25; + const int ERR_HTTP2_SERVER_REFUSED_STREAM = -351; + + const int keyword_netlog = 0x80; + + public static Guid[] rgGuid = + { + new Guid("{3A5F2396-5C8F-4F1F-9B67-6CCA6C990E61}"), // Microsoft.MSEdgeStable + new Guid("{BD089BAA-4E52-4794-A887-9E96868570D2}"), // Microsoft.MSEdgeBeta + new Guid("{C56B8664-45C5-4E65-B3C7-A8D6BD3F2E67}"), // Microsoft.MSEdgeCanary + new Guid("{D30B5C9F-B58F-4DC9-AFAF-134405D72107}"), // Microsoft.MSEdgeDev + new Guid("{E16EC3D2-BB0F-4E8F-BDB8-DE0BEA82DC3D}"), // Microsoft.MSEdgeWebView2 + new Guid("{d2d578d9-2936-45b6-a09f-30e32715f42d}") // Google.Chrome + }; + + + /* + For the ETW correlation schema table, see: + https://github.com/microsoft/MSO-Scripts/issues/50 + */ + public void PreDispatch(in IGenericEvent evt) + { + if ((evt.Keyword & keyword_netlog) == 0) return; + if (this.unhandled.Contains(evt.TaskName)) return; +#if DEBUG + // Confirm that Dispatch remains single-threaded. + IDVal tidCurrent = Environment.CurrentManagedThreadId; + if (s_tidUnique == 0) + s_tidUnique = tidCurrent; + else + AssertCritical(s_tidUnique == tidCurrent); + + if (evt.Timestamp.Nanoseconds < nsStartDB) return; // process from a certain timestamp +#endif // DEBUG + + ReadOnlySpan task = evt.TaskName.AsSpan(); + + // span is more efficient than string here: + if (task.StartsWith("HOST_")) Dispatch_Host(in evt); + else if (task.StartsWith("HTTP_")) Dispatch_Http(in evt); + else if (task.StartsWith("HTTP2_")) Dispatch_Http2(in evt); + else if (task.StartsWith("HTTP3_")) Dispatch_Http3(in evt); + else if (task.StartsWith("SOCKET_")) Dispatch_Socket(in evt); + else if (task.StartsWith("TCP_")) Dispatch_Tcp(in evt); + else if (task.StartsWith("URL_")) Dispatch_Url(in evt); + else if (task.Contains("QUIC_")) Dispatch_Quic(in evt); + else if (task.Contains("CONNECT_")) Dispatch_Connect(in evt); + else Dispatch_Misc(in evt); + } // PreDispatch + + /* + evt.TaskName must begin with "HOST_" + */ + void Dispatch_Host(in IGenericEvent evt) + { + UIDVal uID; + Request req; + ResolverManager resolver; + JsonElement[] rgje; + + AssertCritical(evt.TaskName.StartsWith("HOST_")); + + switch (evt.TaskName) + { + // ID -> HOST_RESOLVER_MANAGER_CREATE_JOB + // "host":"", "network_anonymization_key":"" + case "HOST_RESOLVER_MANAGER_REQUEST": + if (!evt.IsBeginPhase()) break; + if (!evt.TestResolverSourceType()) break; + + rgje = evt.ParseSimpleJsonString(rgstrHost_Key); + if (rgje == null) break; + + resolver = new ResolverManager + { + host = rgje[0].GetString(), + anon_key = rgje[1].GetString() + }; + + this.ResolverManagerAttach(resolver, in evt); + + break; + + // ID -> HOST_RESOLVER_MANAGER_REQUEST + // NOTE: Relay this ID to: HOST_RESOLVER_MANAGER_JOB + case "HOST_RESOLVER_MANAGER_CREATE_JOB": + AssertImportant(evt.IsInstantPhase()); + + this.SetRecentUID(in evt); + + break; + + // ID -> HOST_RESOLVER_MANAGER_JOB_REQUEST_ATTACH, HOST_RESOLVER_MANAGER_JOB_STARTED + case "HOST_RESOLVER_MANAGER_JOB": + if (!evt.IsBeginPhase()) break; + AssertImportant(evt.CheckSourceType("HOST_RESOLVER_IMPL_JOB")); + + uID = this.GetRecentUID("HOST_RESOLVER_MANAGER_CREATE_JOB", in evt); + AssertImportant(uID != 0); + if (uID == 0) break; + + resolver = this.ResolverManagerAttach(uID, in evt); + + break; + + // SSL_CONNECT_JOB: ID -> CONNECT_JOB, SOCKET_POOL_CONNECT_JOB_CREATED, SSL_CONNECT_JOB_CONNECT, TRANSPORT_CONNECT_JOB_CONNECT, HOST_RESOLVER_MANAGER_REQUEST + // TRANSPORT_CONNECT_JOB: ID -> CONNECT_JOB, CONNECT_JOB_SET_SOCKET + // NETWORK_SERVICE_HOST_RESOLVER: ID -> HOST_RESOLVER_MANAGER_REQUEST + // QUIC_SESSION_POOL_DIRECT_JOB: ID -> QUIC_SESSION_POOL_JOB, QUIC_SESSION_POOL_JOB_BOUND_TO, HOST_RESOLVER_MANAGER_REQUEST + // "results":{"aliases":["",...],"canonical_names":["",...],"ip_endoints":[{"endpoint_address":"###.###.###.###","endpoint_port":0},...]} + // NOTE: This event can connect to a Request via these two events: CONNECT_JOB, HOST_RESOLVER_MANAGER_REQUEST + case "HOST_RESOLVER_MANAGER_CACHE_HIT": + AssertImportant(evt.IsInstantPhase()); + if (!evt.TestResolverSourceType()) break; + + rgje = evt.ParseSimpleJsonString(rgstrCanon_Endpoint); + if (rgje == null) break; + + resolver = this.GetResolverManager(in evt); + AssertImportant(resolver != null); + if (resolver == null) break; + + // There are usually two copies of this event. Ignore the 2nd. + if (resolver.rgstrAddress != null) break; + + resolver.rgstrAddress = rgje[1].MyGetStringArray("endpoint_address"); // exclude "endpoint_port":0 + resolver.rgstrCanon = rgje[0].MyGetStringArray(); + AssertImportant(resolver.rgstrCanon.Length == 1); // else what? + + req = this.RequestFromUID(in evt); + if (req == null) break; + + // There may well be multiple IP Addresses. Take the first. + req.ipAddr = (resolver.rgstrAddress?.Length > 0) ? resolver.rgstrAddress[0] : string.Empty; + + req.Canon = (resolver.rgstrCanon?.Length > 0) ? resolver.rgstrCanon[0] : string.Empty; + + break; + + // ID -> HOST_RESOLVER_MANAGER_JOB, HOST_RESOLVER_DNS_TASK, DNS_TRANSACTION + // "results":[{"domain_name":"","endpoints":["address":"###.###.###.###","port":0},...],"type":"data"}, ...] + // NOTE: Added to the DNS list, not immediately attached to a ResolverManager, etc. + case "HOST_RESOLVER_DNS_TASK_EXTRACTION_RESULTS": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HOST_RESOLVER_IMPL_JOB")); + + DNSInfo.ResolvedDNS rdns = DNSInfo.ParseHostResolveDNS(evt.GetParams()); + + if (rdns != null) + this.allTables.dnsTable.AddServerAndAddress(rdns.Domain, rdns.Alias, rdns.rgAddress); + + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Host + + /* + evt.TaskName must begin with "HTTP_" + */ + void Dispatch_Http(in IGenericEvent evt) + { + int srcdep; + string strJSON; + Request req; + Socket soc; + Session session; + JsonElement[] rgje; + + AssertCritical(evt.TaskName.StartsWith("HTTP_")); + + switch (evt.TaskName) + { + // ID -> HTTP_STREAM_JOB_INIT_CONNECTION, HTTP_STREAM_JOB_BOUND_TO_REQUEST, etc. + // src_dep: HTTP_STREAM_JOB_CONTROLLER_BOUND, and the other HTTP_STREAM_JOB for "use_quic":true/false + // "destination":, "use_quic":true/false + case "HTTP_STREAM_JOB": + if (!evt.IsBeginPhase()) + break; + + AssertImportant(evt.CheckSourceType(evt.TaskName)); + + rgje = evt.ParseSimpleJsonString(rgstrSourceId_Type_Destination_Quic); + if (rgje == null) break; + + srcdep = rgje[0].MyGetNumber(); + AssertCritical(srcdep > 0); + + req = this.RequestFromUID_SrcDep(srcdep, in evt); + AssertImportant(req != null); + if (req == null) break; + + bool fQuic = rgje[3].MyGetBool(); + req.SetStreamUID(evt.GetUID(), fQuic); +#if DEBUG + string strType = rgje[1].MyGetString(); + + if (fQuic) + AssertImportant(strType.Equals("dns_alpn_h3") || strType.Equals("alternative")); // else what? + else + AssertImportant(strType.Equals("main")); + + AssertImportant(req.Type == StreamType.Unknown); +#endif // DEBUG + // Strip "https://" + Uri uri = rgje[2].MyGetString().CreateURI(); + if (uri != null) + { + AssertImportant(req.Domain == null || req.Domain == uri.Host); + req.Domain = uri.Host; + } + + break; + + // ID -> HTTP_STREAM_JOB_CONTROLLER + // is_preconnect:bool, url:string + // NOTE: Preconnect Stream Jobs begin here, in which case we create a placeholder Request. + case "HTTP_STREAM_JOB_CONTROLLER": + if (!evt.IsBeginPhase()) break; + + AssertImportant(evt.CheckSourceType(evt.TaskName)); + + rgje = evt.ParseSimpleJsonString(rgstrUrl_Preconnect); + if (rgje == null) break; + + if (rgje[1].MyGetBool()) // is_preconnect:true + { + // Find or synthesize a placeholder Request for preconnect activity. + req = this.PreconnectRequest(rgje[0].MyGetString(), in evt, in this.allTables.threadTable); + } + else // is_preconnect:false + { + // This event is followed by: HTTP_STREAM_REQUEST and HTTP_STREAM_JOB_CONTROLLER_BOUND/URL_REQUEST + // Stash the JSON params to link with the Request. + this.StashJSON(in evt); + } + + break; + + // ID -> REQUEST_ALIVE, URL_REQUEST_START_JOB, HTTP_STREAM_REQUEST_BOUND_TO_JOB OR HTTP_STREAM_REQUEST_STARTED_JOB, HTTP_STREAM_JOB_BOUND_TO_REQUEST + // source_type: URL_REQUEST or HTTP_STREAM_JOB_CONTROLLER + case "HTTP_STREAM_JOB_CONTROLLER_BOUND": + AssertImportant(evt.IsInstantPhase()); + + if (evt.CheckSourceType("URL_REQUEST")) + { + // This is usually unused, but we have to grab it now. + strJSON = this.UnstashJSON("HTTP_STREAM_JOB_CONTROLLER", in evt); + + // Assign the srcdep given the event's ID. + req = this.RequestFromUID_SrcDep(in evt); + if (req != null) break; // success + + // rare fallback: synthesize a Request + + rgje = ParseSimpleJsonString(strJSON, rgstrUrl); + if (rgje == null) break; + + string strURL = rgje[0].MyGetString(); + + req = new Request(strURL, in evt) + { + method = "restored" + }; + + req.xlink.GetLink(evt.ThreadId, evt.Timestamp.ToGraphable(), in this.allTables.threadTable); + + this.Add(req); + this.RequestAttachUID_SrcDep(req, in evt); + } + else + { + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB_CONTROLLER")); + + // Adjacent tasks with the same name, different source type. + req = this.RequestFromUID_Correlate(in evt, evt.TaskName); + + if (req != null) + this.RequestAttachSrcDep(req, in evt); + } + break; + + // ID -> HTTP_STREAM_JOB_CONTROLLER_BOUND, HTTP_STREAM_JOB_BOUND_TO_REQUEST + // srcdep -> HTTP_STREAM_REQUEST_BOUND_TO_JOB, QUIC_SESSION_POOL_ATTACH_HTTP_STREAM_JOB_TO_EXISTING_SESSION + case "HTTP_STREAM_REQUEST_STARTED_JOB": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB_CONTROLLER")); + + // null when the previous HTTP_STREAM_JOB event, params field, type property is not "main" + req = this.RequestFromUID_Correlate(in evt, "HTTP_STREAM_JOB"); + if (req == null) break; + + this.RequestAttachSrcDep(req, in evt); + + break; + + /* + Adjacent Events: + HTTP_STREAM_REQUEST_PROTO / HTTP_STREAM_JOB (non-QUIC-only, not always available) + HTTP_STREAM_REQUEST_BOUND_TO_JOB / URL_REQUEST / srcdep -> HTTP_STREAM_REQUEST_STARTED_JOB, SOCKET_IN_USE, HTTP2_SESSION_SEND_HEADERS + HTTP_STREAM_JOB_BOUND_TO_REQUEST / HTTP_STREAM_JOB / srcdep -> HTTP_STREAM_JOB_CONTROLLER_BOUND x N + HTTP_STREAM_JOB_BOUND_TO_REQUEST / HTTP_STREAM_JOB_CONTROLLER / " + HTTP_STREAM_JOB_ORPHANED / HTTP_STREAM_JOB (QUIC-only) + A 'losing' QUIC connection can be 'orphaned' and possibly reused. A TCP connection will be canceled rather than orphaned.) + */ + + // ID -> REQUEST_ALIVE + // SrcDep -> HTTP_STREAM_REQUEST_STARTED_JOB, QUIC_SESSION_POOL_ATTACH_HTTP_STREAM_JOB_TO_EXISTING_SESSION, SOCKET_IN_USE + case "HTTP_STREAM_REQUEST_BOUND_TO_JOB": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + // Associate the Request, event ID, source_dependency id + req = this.RequestFromUID_SrcDep(in evt); + if (req == null) break; + + soc = this.SocketFromSrcDep(in evt); + if (soc == null) break; + + // If soc==null then consider a placeholder Session when we know the type in: HTTP_STREAM_JOB_BOUND_TO_REQUEST + // This will attach a placeholder Session (with Socket) only if it appears to be needed. + if (req.FAttachPlaceholderSessionAndStream(soc, in evt)) + { + this.sessionTable.Add(req.Session); + this.SessionAttachUID(req.Session, in evt); // sets: session.uidVal + } + + break; + + // ID -> HTTP_STREAM_REQUEST_STARTED_JOB, HTTP_STREAM_JOB_CONTROLLER_BOUND + // srcdep -> HTTP_STREAM_JOB_CONTROLLER_BOUND + // NOTE: This event (when source_type==HTTP_STREAM_JOB) determines whether QUIC or HTTP2 won the race of the HTTP Stream Jobs. + case "HTTP_STREAM_JOB_BOUND_TO_REQUEST": + AssertImportant(evt.IsInstantPhase()); + + req = this.RequestFromUID_SrcDep(in evt); + AssertImportant(req != null); + if (req == null) break; + + if (evt.CheckSourceType("HTTP_STREAM_JOB")) + { + UIDVal uid = evt.GetUID(); + + // This event links back to HTTP_STREAM_JOB & HTTP_STREAM_JOB_INIT_CONNECTION + // There may have been two HTTP_STREAM_JOB events, one QUIC, the other not. + req.SetStreamType(uid); + + session = this.SessionFromUID(uid, in evt); + if (session != null) + { + AssertImportant(req.Type == session.Type); + + if (req.Session == null) + req.SessionSet = session; + else + AssertImportant(req.Session == session); + + if (req.Type == StreamType.QUIC) + session.AttachQUIC(req); +#if DEBUG + Session sessionT = req.stream?.request?.Session; + AssertImportant(sessionT == null || sessionT == session); +#endif // DEBUG + } + else if (req.Type == StreamType.HTTP1) + { + // The Socket is missing, but fill in with a placeholder Session and Stream. + if (req.FAttachPlaceholderSessionAndStream(StreamType.HTTP1, in evt)) + { + session = req.SessionOther; // HTTP1 Session + this.sessionTable.Add(session); + this.SessionAttachUID(session, uid, in evt); // sets: session.uidVal + } + } + } + // Else this event links back to HTTP_STREAM_JOB_CONTROLLER/_BOUND & HTTP_STREAM_REQUEST_STARTED_JOB + break; + + // ID -> HTTP_STREAM_JOB + // NOTE: Indicates whether the non-QUIC HTTP Stream Job is HTTP/2 (common) or HTTP/1.1 (rare) + // NOTE: Followed by: HTTP_STREAM_REQUEST_BOUND_TO_JOB + case "HTTP_STREAM_REQUEST_PROTO": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + rgje = evt.ParseSimpleJsonString(rgstrProto); + if (rgje == null) break; + + string strProto = rgje[0].MyGetString(); + AssertImportant(strProto.Equals("h2") || strProto.Equals("http/1.1")); + + StreamType type = (strProto.Equals("h2")) ? StreamType.HTTP2 : StreamType.HTTP1; + + req = this.RequestFromUID(in evt); + AssertImportant(req != null); + if (req != null) + { + AssertImportant(req.uidTCP == evt.GetUID()); + req.TypeTCP = type; + } +#if DEBUG + soc = this.SocketFromUID(in evt); + if (soc != null) + AssertImportant(soc.Type == req.TypeTCP); +#endif // DEBUG + + break; + + // ID -> HTTP_STREAM_JOB, etc. + // SrcDep -> HTTP2_SESSION_POOL_IMPORTED_SESSION_FROM_SOCKET, HTTP2_SESSION_POOL_FOUND_EXISTING_SESSION + case "HTTP_STREAM_JOB_HTTP2_SESSION_AVAILABLE": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + req = this.RequestFromUID(in evt); + AssertImportant(req != null); + if (req == null) break; + + AssertImportant(req.TypeTCP == StreamType.HTTP2 || req.TypeTCP == StreamType.Unknown); + req.TypeTCP = StreamType.HTTP2; // provisional + + if (req.SessionHTTP2 != null) break; + + srcdep = evt.GetSourceId(); + session = this.SessionFromSrcDep(srcdep, in evt); + if (session == null) + { + session = req.NewPlaceholderSession(StreamType.HTTP2, in evt); + this.sessionTable.Add(session); + this.SessionAttachSrcDep(session, srcdep, in evt); + } + + req.SessionHTTP2 = session; + + break; + + // ID -> URL_REQUEST_START_JOB, etc. + // NOTE: A Chromium optimization did not work out. Disconnect the Request and abandon the Stream. + // NOTE: The underlying Session/Socket is probably still usable. + case "HTTP_TRANSACTION_RESTART_MISDIRECTED_REQUEST": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID(in evt); + if (req == null) break; + + if (req.FAttachedToStream) + { + req.stream.iError = FAILED; // -2 = generic failure + req.stream.Abandon(true); // Hard abandon the Stream since the Session will retry. + + AssertImportant(!req.FAttachedToStream); + } + + // Other Requests may refer to those Sessions. This one no longer does. + AssertImportant(req.Type != StreamType.HTTP1); + req.SessionReset(); + + break; + + // ID -> URL_REQUEST_START_JOB, etc. + // NOTE: Disconnect and abandon the broken Stream. + // NOTE: The underlying Session/Socket is probably broken as well. + case "HTTP_TRANSACTION_RESTART_AFTER_ERROR": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID(in evt); + if (req == null) break; + + // This Request error will later reset in: Session.Attach + int iError = evt.GetNetError(); // commonly -351 = ERR_HTTP2_SERVER_REFUSED_STREAM + AssertImportant(FImplies(iError == ERR_HTTP2_SERVER_REFUSED_STREAM, req.Type == StreamType.HTTP2)); + + req.iError = iError; + if (req.Session != null) + { + req.Session.iError = iError; + if (req.Session.socket != null) + req.Session.socket.iError = iError; + } +#if DEBUG + if (req.SocketTCP != null) + { + req.SocketTCP.iError = iError; + req.SocketTCP = null; + } +#endif // DEBUG + if (req.FAttachedToStream) + { + req.stream.iError = iError; + req.stream.Abandon(false); // Hard abandon the Stream only if no data transferred. + + AssertImportant(!req.FAttachedToStream); + } + + // Other Requests may refer to those Sessions. This one no longer does. + req.SessionReset(); + + break; + + // ID -> REQUEST_ALIVE, HTTP_STREAM_REQUEST_BOUND_TO_JOB, etc. + // quic_stream_id:#, "headers": + // source_type: URL_REQUEST + // NOTE: Links to a URL_REQUEST, while containing the (redundant) parameters of a Stream. + // NOTE: This UID gets picked up by QUIC_CHROMIUM_CLIENT_STREAM_SEND_REQUEST_HEADERS + case "HTTP_TRANSACTION_QUIC_SEND_REQUEST_HEADERS": // SIMILAR, with "quic_stream_id":# + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + this.SetRecentUID(in evt); + + break; + + // ID -> REQUEST_ALIVE, etc. + // SrcDep -> QUIC_SESSION_CREATED + case "HTTP_STREAM_REQUEST_BOUND_TO_QUIC_SESSION": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID(in evt); + AssertImportant(req != null); + if (req == null) break; + + if (req.SessionQUIC != null) break; + + session = this.SessionFromSrcDep(in evt); + AssertInfo(session != null); + if (session == null) + { + // The QUIC_SESSION event must have happened before tracing started. + session = req.NewPlaceholderSession(StreamType.QUIC, in evt); + this.sessionTable.Add(session); + this.SessionAttachUID_SrcDep(session, in evt); // sets: session.uidVal/.srcdep + } + + session.AttachQUIC(req); // add to the Request Pending list + + break; + + // ID -> REQUEST_ALIVE, etc. + // NOTE: Get the HTTP Status from the headers string. + case "HTTP_TRANSACTION_READ_RESPONSE_HEADERS": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID(in evt); + AssertInfo(req != null); + if (req == null) break; + + AssertImportant(req.stream != null); + if (req.stream == null) break; + + string strHTTPStatus = evt.GetParams().GetStatusJSON(); + AssertImportant(string.IsNullOrEmpty(req.stream.strHTTPStatus) || strHTTPStatus.StartsWith(req.stream.strHTTPStatus)); + req.stream.strHTTPStatus = strHTTPStatus; + + break; + + // ID -> REQUEST_ALIVE, URL_REQUEST_START_JOB, etc. + // NOTE: Reading data (such as an image file) from the filesystem cache rather than via network transactions. + case "HTTP_CACHE_READ_DATA": + if (!evt.IsBeginPhase()) break; + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID(in evt, false); // Request might be closed + AssertImportant(req != null); + if (req == null) break; + + // If it has gotten this far with StreamType.Unknown and no Session + // then it has done no network transaction, and is of type CACHE. + + if (!(req.Type == StreamType.CACHE || req.Type == StreamType.Unknown)) break; + + req.Type = StreamType.CACHE; + + if (req.FAttachPlaceholderSessionAndStream(StreamType.CACHE, in evt)) + { + session = req.SessionOther; // Session with CACHE type + this.sessionTable.Add(session); + this.SessionAttachUID(session, in evt); // sets: session.uidVal + } + + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Http + + /* + evt.TaskName must begin with "HTTP2_" + */ + void Dispatch_Http2(in IGenericEvent evt) + { + uint cb; + int srcdep; + Request req; + Socket soc; + Session session; + Session.Stream stream; + ResolverManager resolver; + JsonElement[] rgje; + + AssertCritical(evt.TaskName.StartsWith("HTTP2_")); + + switch (evt.TaskName) + { + // ID -> first; HTTP2_SESSION_INITIALIZED, HTTP2_SESSION_SEND/RECV_HEADERS, HTTP2_SESSION_SEND/RECV_DATA, HTTP2_SESSION_CLOSE, HTTP2_SESSION_POOL_REMOVE_SESSION + // "host":":" + // source_type: HTTP2_SESSION + case "HTTP2_SESSION": + if (evt.IsEndPhase()) + { + session = this.SessionFromUID(in evt, false); + AssertInfo(session != null); + session?.Shutdown(); + + break; + } + + AssertImportant(evt.CheckSourceType(evt.TaskName)); + + rgje = evt.ParseSimpleJsonString(rgstrHost); + if (rgje == null) break; + + string strHost = rgje[0].ToString().GetHostAndPort(out ushort port); + if (strHost == null) break; + + session = new Session(StreamType.HTTP2, in evt) + { + domain = strHost, + port = port + }; + + this.sessionTable.Add(session); + this.SessionAttachUID(session, in evt); // sets: session.uidVal + + break; + + // ID -> HTTP2_SESSION, etc. + // SrcDep -> TRANSPORT_CONNECT_JOB_CONNECT_ATTEMPT, CONNECT_JOB_SET_SOCKET, SOCKET_POOL_BOUND_TO_SOCKET + // source_type: HTTP2_SESSION + // NOTE: Links to the TCP socket. + case "HTTP2_SESSION_INITIALIZED": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP2_SESSION")); + + session = this.SessionFromUID(in evt); + AssertImportant(session != null); + if (session == null) break; + + srcdep = evt.GetSourceId(); + + soc = this.SocketFromSrcDep(srcdep, in evt); + AssertImportant(soc != null); + if (soc != null) + { + session.Attach(soc); + + // HTTP2_SESSION_POOL_IMPORTED_SESSION_FROM_SOCKET will attach via this event Id. + AssertImportant(soc.uidBound != 0); + if (soc.uidBound != 0) + this.SessionAddUID(session, soc.uidBound, in evt); + } + + resolver = this.GetResolverManagerFromSrcDep(srcdep, in evt); + session.resolver = resolver; + + // Set session.srcdep and recent Session for: HTTP2_SESSION_POOL_IMPORTED_SESSION_FROM_SOCKET + this.SessionAttachSrcDep(session, srcdep, in evt); + + break; + + // ID -> HTTP2_SESSION/_INITIALIZED + // source_type: HTTP2_SESSION + // "net_error":# + case "HTTP2_SESSION_CLOSE": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP2_SESSION")); + + session = this.SessionFromUID(in evt); + if (session == null) break; + + if (session.iError == 0) + session.iError = evt.GetNetError(); + + AssertImportant(!session.FQuic); + session.Closed = true; + + break; + + // ID -> HTTP_STREAM_JOB, TCP_CLIENT_SOCKET_POOL_REQUESTED_SOCKET, SOCKET_POOL_BOUND_TO_SOCKET + // srcdep -> HTTP_STREAM_JOB_HTTP2_SESSION_AVAILABLE, HTTP2_SESSION_POOL_FOUND_EXISTING_SESSION/_FROM_IP_POOL + // NOTE: A new HTTP2 Session has recently been created. So attach the Session to the Request. + // NOTE: Also add the source_dependency id to the Session so that other Requests can also attach. + case "HTTP2_SESSION_POOL_IMPORTED_SESSION_FROM_SOCKET": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + // This event was preceeded by: HTTP2_SESSION_INITIALIZED + session = this.SessionFromRecent("HTTP2_SESSION_INITIALIZED", in evt); + + // This event Id was attached to the recently created Session via HTTP_STREAM_REQUEST_PROTO(proto:h2) + if (session == null) + session = this.SessionFromUID(in evt); + else + AssertImportant(session == this.SessionFromUID(in evt) || null == this.SessionFromUID(in evt)); + + if (session == null) + { + session = new Session(StreamType.HTTP2, in evt) + { + fRecovered = true + }; + this.sessionTable.Add(session); + this.SessionAttachUID(session, in evt); // sets session.uidVal + } + else + { + AssertImportant(session.socket != null); + } + + // Other events reference this Session using the source_dependency id: + // HTTP_STREAM_JOB_HTTP2_SESSION_AVAILABLE, HTTP2_SESSION_POOL_FOUND_EXISTING_SESSION/_FROM_IP_POOL + AssertImportant(session.Type == StreamType.HTTP2); + this.SessionAddSrcDep(session, in evt); + + req = this.RequestFromUID(in evt); + AssertImportant(req != null); + if (req == null) break; + + AssertImportant(req.Type == StreamType.Unknown); + AssertImportant(req.TypeTCP == StreamType.HTTP2); + AssertImportant(req.port != 0); // else req.port = (soc?.addrRemote).PortGraphable() + AssertImportant(req.Domain != null); // else look it up via allTables.dnsTable ? + + if (req.SessionHTTP2 == null) + req.SessionHTTP2 = session; + else + AssertImportant(req.SessionHTTP2 == session); + + break; + + /* + Adjacent: + HTTP_STREAM_JOB_INIT_CONNECTION.Begin + HTTP2_SESSION_POOL_FOUND_EXISTING_SESSION + ... + HTTP_STREAM_JOB_INIT_CONNECTION.End + */ + + // ID -> HTTP_STREAM_JOB/_INIT_CONNECTION, HTTP_STREAM_JOB_BOUND_TO_REQUEST + // srcdep -> HTTP2_SESSION_POOL_IMPORTED_SESSION_FROM_SOCKET, HTTP_STREAM_JOB_HTTP2_SESSION_AVAILABLE, HTTP2_SESSION_POOL_FOUND_EXISTING_SESSION_FROM_IP_POOL + // NOTE: These occur before HTTP_STREAM_JOB_BOUND_TO_REQUEST and mark the TCP (non-QUIC) Stream Job as HTTP2 rather than HTTP1. + case "HTTP2_SESSION_POOL_FOUND_EXISTING_SESSION": + case "HTTP2_SESSION_POOL_FOUND_EXISTING_SESSION_FROM_IP_POOL": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + req = this.RequestFromUID(in evt); + if (req != null) + { + AssertImportant(req.TypeTCP == StreamType.HTTP2 || req.TypeTCP == StreamType.Unknown); + req.TypeTCP = StreamType.HTTP2; // provisional + + // This Request _might_ get picked up (below) for processing a Preconnect Request. + this.RequestAttachSrcDep(req, in evt); + + if (req.SessionHTTP2 != null) break; + + srcdep = evt.GetSourceId(); + session = this.SessionFromSrcDep(in evt); + if (session == null) + { + session = req.NewPlaceholderSession(StreamType.HTTP2, in evt); + this.sessionTable.Add(session); + this.SessionAttachSrcDep(session, srcdep, in evt); + } + + req.SessionHTTP2 = session; + + break; + } + + // Get the HTTP2 Session to attach to a Preconnect Request. + + req = this.RequestFromSrcDep(in evt); + session = this.SessionFromSrcDep(in evt); + if (session != null) + { + AssertCritical(session.Type == StreamType.HTTP2); + AssertImportant(req == null || req.SessionHTTP2 == session); + } + else + { + if (req == null) break; + session = req.SessionHTTP2; + AssertImportant(req.TypeTCP == StreamType.HTTP2); + if (session == null) break; + } + + // A preconnect HTTP Stream Job has a placeholder Request that connects in a different way. + // See: HTTP_STREAM_JOB_CONTROLLER + // cf. QUIC_SESSION_POOL_ATTACH_HTTP_STREAM_JOB_TO_EXISTING_SESSION + + AssertImportant(!session.Closed); + AssertImportant(session.Type == StreamType.HTTP2); + + Uri uri = req?.URL.CreateURI(); + if (uri == null) break; + + req = this.FindPreconnectRequest(uri.GetLeftPart(UriPartial.Authority), session.PreKey, in evt); + if (req == null) break; + + AssertCritical(req.IsPreconnect); + + // A Preconnect Request has no Stream, so it simply refers to a Session, and not vice-versa. + req.SessionSet = session; + + break; + + // ID -> HTTP2_SESSION, etc. + // SrcDep -> HTTP_STREAM_REQUEST_STARTED_JOB & SOCKET_IN_USE & HTTP_STREAM_REQUEST_BOUND_TO_JOB + // "fin":true/false, "stream_id":#, "headers": + // source_type: HTTP2_SESSION + // NOTE: This data is for a specific Stream within a specific Session. + // NOTE: When both Send and Recv have "fin":true from either _DATA or _HEADERS events then the stream is closed. + // NOTE: HTTP2_SESSION_RECV_HEADERS does not have the SourceId/SrcDep + case "HTTP2_SESSION_SEND_HEADERS": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP2_SESSION")); + + req = this.RequestFromSrcDep(in evt); + if (req == null) break; + + session = this.SessionFromUID(evt); + AssertInfo(session != null); + if (session == null) + session = req.SessionHTTP2; + else + AssertImportant(session == req.SessionHTTP2 || req.SessionHTTP2 == null); + + if (session == null) + { + session = req.NewPlaceholderSession(StreamType.HTTP2, in evt); + this.sessionTable.Add(session); + this.SessionAttachUID(session, in evt); + } + + AssertCritical(session.Type == StreamType.HTTP2); + + stream = session.PopulateStreamFromHeader(in evt); + AssertCritical(stream != null); + if (stream == null) break; + + AssertImportant(!stream.HasDataTraffic()); + stream.cbUpload = req.cbUpload; + stream.cbDownload = req.cbDownload; + + AssertImportant(req.method?.EndsWith(stream.strMethod) ?? false); + AssertImportant(req.Domain?.Equals(stream.strDomain) ?? false); + + // If there's a Request error then its Stream should have been cleared but not its Session. See: HTTP_TRANSACTION_RESTART_AFTER_ERROR + AssertImportant(FImplies(req.SessionHTTP2 != null, req.SessionHTTP2 == session || req.iError < 0)); + AssertImportant(FImplies(req.Session != null, req.Session == session || req.iError < 0)); + + // Attach the Request to the Stream and to the Session. + stream.Attach(req); // Request <-> Stream + session.Finalize(req); // Request -> Session + + AssertCritical(stream.request.Session == session); + + this.SessionAddUID(session, req.uidTCP, in evt); + + break; + + // ID -> HTTP2_SESSION, etc. + // "fin":true/false, "size":##, "stream_id":# + // NOTE: "fin" = finished (No more data will be sent/received on this stream.) + case "HTTP2_SESSION_SEND_DATA": + case "HTTP2_SESSION_RECV_DATA": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP2_SESSION")); + + session = this.SessionFromUID(evt); + AssertInfo(session != null); + if (session == null) + { + session = new Session(StreamType.QUIC, in evt); + this.sessionTable.Add(session); + this.SessionAttachUID(session, in evt); + } + + rgje = evt.ParseSimpleJsonString(rgstrStreamId_Size); + if (rgje == null) break; + + cb = rgje[1].MyGetUNumber(); + AssertCritical((int)cb >= 0); + if (cb == 0) break; + + int iStream = rgje[0].MyGetNumber(-1); + AssertCritical(iStream >= 0); + if (iStream < 0) break; + + stream = session.EnsureStream(iStream, evt.Timestamp.ToGraphable()); + + if (evt.TaskName.Equals("HTTP2_SESSION_SEND_DATA")) + stream.cbSend += cb; + else + stream.cbRecv += cb; + + break; + + // ID -> HTTP2_SESSION + // NOTE: Get the HTTP Status from the Header text + case "HTTP2_SESSION_RECV_HEADERS": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP2_SESSION")); + + session = this.SessionFromUID(evt); + AssertInfo(session != null); + if (session == null) + { + session = new Session(StreamType.HTTP2, in evt); + this.sessionTable.Add(session); + this.SessionAttachUID(session, in evt); + } + + AssertCritical(session.Type == StreamType.HTTP2); + + session.SetHTTPStatus(in evt); + + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Http2 + + /* + evt.TaskName must begin with "HTTP3_" + */ + void Dispatch_Http3(in IGenericEvent evt) + { + uint cb; + int iStream; + Request req; + Socket soc; + Session session; + Session.Stream stream; + JsonElement[] rgje; + + AssertCritical(evt.TaskName.StartsWith("HTTP3_")); + + switch (evt.TaskName) + { + // ID -> QUIC_SESSION, etc. + // "stream_id":#, "headers": + // source_type: QUIC_SESSION + // NOTE: This data is for a specific Stream within a specific Session. + // NOTE: cf. QUIC_CHROMIUM_CLIENT_STREAM_SEND_REQUEST_HEADERS (same event, different layer) + case "HTTP3_HEADERS_SENT": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + session = this.SessionFromUID(evt); + AssertImportant(session != null); + if (session == null) break; + + AssertCritical(session.Type == StreamType.QUIC); + + // A Socket belongs to the Session. A Request belongs to a Stream. + + soc = this.SocketFromUID(in evt); + if (soc != null) + session.Attach(soc); + + stream = session.PopulateStreamFromHeader(in evt); + AssertCritical(stream != null); + if (stream == null) break; + + // Get a matching pending (or recent) Request. + req = session.MatchRequest(stream); + AssertImportant(req != null); // normally shouldn't use the fallback option + if (req == null) + req = this.MatchRequest(session.pid, session.tid, stream); + + AssertImportant(req != null); + if (req == null) break; + + // Attach the Request to the Stream, or confirm. + stream.Attach(req); + // Not needed here: session.Finalize(req) + // But there could be another pending Request queued up with the same URL. + AssertInfo(session.LookupPendingRequestByURL(stream.strURL) < 0); + + AssertCritical(req.Session == session); + req.SessionSet = session; // just in case + + AssertCritical(stream.request?.Session == session); + + break; + + // ID -> QUIC_SESSION + // "payload_length":#, "stream_id":# + case "HTTP3_DATA_FRAME_RECEIVED": + case "HTTP3_DATA_SENT": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + session = this.SessionFromUID(evt); + AssertInfo(session != null); + if (session == null) + { + session = new Session(StreamType.QUIC, in evt); + this.sessionTable.Add(session); + this.SessionAttachUID(session, in evt); + } + + AssertCritical(session.Type == StreamType.QUIC); + + rgje = evt.ParseSimpleJsonString(ChromiumTable.rgstrStreamId_Payload); + if (rgje == null) break; + + cb = rgje[1].MyGetUNumber(); + AssertCritical((int)cb >= 0); + if (cb == 0) break; + + iStream = rgje[0].MyGetNumber(-1); + AssertCritical(iStream >= 0); + if (iStream < 0) break; + + stream = session.EnsureStream(iStream, evt.Timestamp.ToGraphable()); + + if (evt.TaskName.Equals("HTTP3_DATA_SENT")) + stream.cbSend = cb; + else + stream.cbRecv = cb; + + AssertCritical(FImplies(stream.request != null, stream.request?.Session == session)); + + break; + + // ID -> QUIC_SESSION + // NOTE: Get the HTTP Status from the Header text + case "HTTP3_HEADERS_DECODED": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + session = this.SessionFromUID(evt); + AssertInfo(session != null); + if (session == null) break; + + AssertCritical(session.Type == StreamType.QUIC); + + session.SetHTTPStatus(in evt); + + break; + + // ID -> QUIC_SESSION + // NOTE: Mark these streams as ignorable overhead. + case "HTTP3_LOCAL_CONTROL_STREAM_CREATED": + case "HTTP3_LOCAL_QPACK_DECODER_STREAM_CREATED": + case "HTTP3_LOCAL_QPACK_ENCODER_STREAM_CREATED": + case "HTTP3_PEER_CONTROL_STREAM_CREATED": + case "HTTP3_PEER_QPACK_DECODER_STREAM_CREATED": + case "HTTP3_PEER_QPACK_ENCODER_STREAM_CREATED": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + session = this.SessionFromUID(evt); + AssertImportant(session != null); + if (session == null) break; + + AssertCritical(session.Type == StreamType.QUIC); + + rgje = evt.ParseSimpleJsonString(ChromiumTable.rgstrStreamId); + if (rgje == null) break; + + iStream = rgje[0].MyGetNumber(-1); + AssertCritical(iStream >= 0); + if (iStream < 0) break; + + stream = session.EnsureStream(iStream, evt.Timestamp.ToGraphable()); + stream.fIgnore = true; + + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Http3 + + /* + evt.TaskName must contain "QUIC_" + */ + void Dispatch_Quic(in IGenericEvent evt) + { + ushort port; + int srcdep; + int err; + string strURL; + UIDVal uID; + Request req; + Socket soc; + Session session; + Session.Stream stream; + ResolverManager resolver; + IPAddress ipAddress; + JsonElement[] rgje; + + // QUIC + // QUIC = Quick UDP Internet Connections, a UDP-based alternative to TCP + // Chromium may launch both QUIC and TCP connections, and use the one which responds first. + + AssertCritical(evt.TaskName.Contains("QUIC_")); + + switch (evt.TaskName) + { + // ID -> QUIC_SESSION_*, HTTP3_LOCAL_CONTROL_STREAM_CREATED, UDP_BYTES_SENT/RECEIVED, etc. + // SrcDep -> BOUND_TO_QUIC_SESSION_POOL_JOB, SOCKET_ALIVE + // "host":"", "network_anonymization_key":"", "port":# + case "QUIC_SESSION": + if (evt.IsEndPhase()) + { + session = this.SessionFromUID(in evt, false); + AssertImportant(session != null); + session?.Shutdown(); + + break; + } + + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + rgje = evt.ParseSimpleJsonString(rgstrHost_Key_Port_SourceId); + if (rgje == null) break; + + session = new Session(StreamType.QUIC, in evt) + { + domain = rgje[0].MyGetString(), // host server name (no http://) + anon_key = rgje[1].MyGetString(), + port = (ushort)rgje[2].MyGetUNumber() + }; + + this.sessionTable.Add(session); + + srcdep = rgje[3].MyGetNumber(); + soc = this.SocketFromSrcDep(srcdep, in evt); +#if DEBUG + AssertImportant(soc != null && !soc.fTCP); +#endif // DEBUG + session.Attach(soc); + + this.SessionAttachUID_SrcDep(session, srcdep, in evt); // sets session.uidVal and .srcdep + + break; + + // ID -> QUIC_SESSION, etc. + // SrcDep -> QUIC_SESSION_POOL_JOB_RESULT, HTTP_STREAM_REQUEST_BOUND_TO_QUIC_SESSION + case "QUIC_SESSION_CREATED": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION_POOL_DIRECT_JOB")); + + // Get the Session of the previous event. + session = this.SessionFromRecent("QUIC_SESSION", in evt); + AssertImportant(session != null); + if (session == null) break; + + AssertCritical(session.Type == StreamType.QUIC); + + resolver = this.GetResolverManager(in evt); + session.resolver = resolver; + + this.SessionAddUID_SrcDep(session, in evt); // sets: session.srcdep + + // Also assign to this Session the UID (event ID) of: HTTP_STREAM_JOB(use_quic:true) + + // QUIC_SESSION_POOL_JOB_BOUND_TO assigned this UID to a Request. + req = this.RequestFromUID(in evt); + if (req == null) break; + if (req.uidQUIC == 0) break; + + this.SessionAddUID(session, req.uidQUIC, in evt); + + break; + + /* + ID -> QUIC_SESSION_POOL_JOB, QUIC_SESSION_POOL_JOB_BOUND_TO + 1. ID1 QUIC_SESSION_POOL_JOB host, key, port, ... + 2. ID1 QUIC_SESSION_POOL_JOB_BOUND_TO srcdep-C + 3. ID2 BOUND_TO_QUIC_SESSION_POOL_JOB srcdep-D + Events 1,2,3 usually occur together. + When Event 1 does not precede Event 2, then Events 2,3 likely refer to a previous Request. + */ + case "QUIC_SESSION_POOL_JOB": + // This event usually can't be directly associated with a Request. + // Stash its JSON and pick it up with the next event. + if (!evt.IsBeginPhase()) break; + AssertImportant(evt.CheckSourceType("QUIC_SESSION_POOL_DIRECT_JOB")); + + this.StashJSON(in evt); + + break; + + // ID -> QUIC_SESSION_POOL_JOB + // SrcDep -> HTTP_STREAM_REQUEST_STARTED_JOB, HTTP_STREAM_REQUEST_BOUND_TO_JOB + // NOTE: Adjacent to QUIC_SESSION_POOL_JOB & BOUND_TO_QUIC_SESSION_POOL_JOB + case "QUIC_SESSION_POOL_JOB_BOUND_TO": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION_POOL_DIRECT_JOB")); + + string strJSON = this.UnstashJSON("QUIC_SESSION_POOL_JOB", in evt); + if (strJSON == null) + { + // Disable a subsequent: BOUND_TO_QUIC_SESSION_POOL_JOB + this.ResetRecent(evt.ProcessId, evt.ThreadId); + break; + } + + // Look up the Request from the srcdep and assign the UID. + // If null then this is either near the start of the trace, or it's a Preconnect QUIC Session. + req = this.RequestFromUID_SrcDep(in evt); + if (req == null) break; + + rgje = ParseSimpleJsonString(strJSON, rgstrHost_Key_Port); + if (rgje == null) break; + + port = (ushort)rgje[2].MyGetUNumber(); + string strAnonKey = rgje[1].MyGetString(); // network_anonymization_key + string strHost = rgje[0].MyGetString(); // host server name (no http://) + + AssertImportant(req.Domain == null || req.Domain.Equals(strHost)); + req.Domain = strHost; + + AssertImportant(req.anon_key == null || req.anon_key.Equals(strAnonKey)); + req.anon_key = strAnonKey; + + AssertImportant(req.port == 0 || req.port == port); + req.port = port; + + break; + + // ID -> HTTP_STREAM_JOB + // srcdep -> QUIC_SESSION, SOCKET_ALIVE + case "BOUND_TO_QUIC_SESSION_POOL_JOB": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + // Look up the Request and assign the srcdep. + req = this.RequestFromSrcDep(in evt); + if (req == null) + req = this.RequestFromUID_Correlate(in evt, "QUIC_SESSION_POOL_JOB_BOUND_TO"); + + AssertImportant(req != null || this.RequestFromUID(in evt) == null); // else what did we miss? + AssertImportant(req == null || req.uidQUIC == 0 || req.uidQUIC == evt.GetUID()); + + if (req != null && req.uidQUIC == 0) + req.uidQUIC = evt.GetUID(); + + session = this.SessionFromSrcDep(in evt); + if (session == null) + { + session = req?.SessionQUIC; + if (session == null) break; // common + } + + AssertCritical(session.Type == StreamType.QUIC); + + // Associate the HTTP_STREAM_JOB with the Session. + this.SessionAddUID(session, in evt); + + break; + + // ID -> QUIC_SESSION, etc. + // "quic_error":# + case "QUIC_SESSION_CLOSED": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + session = this.SessionFromUID(in evt, false); + if (session == null) break; + + // Timeout (25) is a default reason to close. + err = evt.GetQuicError(); + if (err == QUIC_NETWORK_IDLE_TIMEOUT/*25*/) + err = 0; + + if (session.iError == 0) + session.iError = err; + + AssertImportant(session.FQuic); + session.Closed = true; + + // Error: close the Socket immediately + if (err != 0) + session.socket?.Close(evt.Timestamp.ToGraphable()); + + break; + + // ID -> QUIC_SESSION, etc. + // "net_error":# + case "QUIC_SESSION_CLOSE_ON_ERROR": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + session = this.SessionFromUID(in evt, false); + if (session == null) break; + AssertImportant(session.FQuic); + + err = evt.GetNetError(); + + if (session.iError == 0) + session.iError = err; + + // Error: close the Socket immediately + if (err != 0) + session.socket?.Close(evt.Timestamp.ToGraphable()); + + session.Closed = true; + + break; + + // ID -> QUIC_SESSION + // source_type: QUIC_SESSION + // NOTE: The Session's Socket has 'degraded' and a new one is being spun up (which may or may not succeed). + // NOTE: The .End event is preceeded by UDP_LOCAL_ADDRESS. + case "QUIC_PORT_MIGRATION_TRIGGERED": + AssertImportant(!evt.IsInstantPhase()); + if (evt.IsBeginPhase()) + { + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + break; + } + + soc = this.SocketFromRecent(in evt, "UDP_LOCAL_ADDRESS"); + AssertImportant(soc != null); + if (soc == null) break; + + session = this.SessionFromUID(in evt); + AssertImportant(session != null); + if (session == null) break; + + AssertCritical(session.Type == StreamType.QUIC); + + session.socketPreMigrate = soc; + + break; + + // ID -> QUIC_SESSION + // source_type: QUIC_SESSION + // NOTE: The QUIC Session has successfully done a connectivity probe with a challenge frame, etc. on the new Socket. + case "QUIC_PORT_MIGRATION_SUCCESS": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + session = this.SessionFromUID(in evt); + AssertImportant(session != null); + if (session == null) break; + + AssertCritical(session.Type == StreamType.QUIC); + + AssertImportant(session.socketPreMigrate != null); + if (session.socketPreMigrate == null) break; + /* + Since NetBlame is very Socket-based, we'll clone/migrate the current Session: + Create a copy of the Session with the original Socket and all of its original Streams. + The original Session gets the new Socket (session.socketPreMigrate). + All the Streams of this original Session get reset with cbSend/Recv = 0. + All of the Requests linked to these original Streams remain linked, + as these Session/Streams are now using the new Socket. + + NOTE: In this case TWO sets of Streams will refer to ONE set of Requests, + while that ONE set of Requests refers to just ONE set of Streams. + This is all handled in the "gather" phase via: Session.AdjustForMigration + */ + this.sessionTable.Add(session.Migrate(evt.Timestamp.ToGraphable())); + + break; + + case "QUIC_CONNECTION_MIGRATION_TRIGGERED": + AssertImportant(false); // Test This + goto case "QUIC_PORT_MIGRATION_TRIGGERED"; + + case "QUIC_CONNECTION_MIGRATION_SUCCESS": + AssertImportant(false); // Test This + goto case "QUIC_PORT_MIGRATION_SUCCESS"; + + // ID -> QUIC_SESSION, etc. + // quic_stream_id:#, "headers": + // source_type: QUIC_SESSION + // NOTE: This data is for a specific Stream within a Session. + // NOTE: The previous event provides the Request connection: HTTP_TRANSACTION_QUIC_SEND_REQUEST_HEADERS + // NOTE: cf. HTTP3_HEADERS_SENT (same event, different layer) + case "QUIC_CHROMIUM_CLIENT_STREAM_SEND_REQUEST_HEADERS": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + uID = this.GetRecentUID("HTTP_TRANSACTION_QUIC_SEND_REQUEST_HEADERS", in evt); + AssertImportant(uID != 0); + + req = this.RequestFromUID(uID, in evt); + AssertImportant(req != null); + if (req == null) break; + + session = this.SessionFromUID(evt); + AssertImportant(session != null); + if (session == null) + { + session = req.SessionQUIC; + if (session == null) break; + } + + AssertCritical(session.Type == StreamType.QUIC); + + // A Socket belongs to the Session. A Request belongs to a Stream. + + soc = this.SocketFromUID(in evt); + if (soc != null) + session.Attach(soc); + + stream = session.PopulateStreamFromHeader(in evt); + AssertCritical(stream != null); + if (stream == null) break; + + // If there's a Request error then its Stream should have been cleared but not its Session. See: HTTP_TRANSACTION_RESTART_AFTER_ERROR + AssertImportant(FImplies(req.SessionQUIC != null, req.SessionQUIC == session || req.iError < 0)); + AssertImportant(FImplies(req.Session != null, req.Session == session || req.iError < 0)); + + // Attach the Request to the Stream and to the Session. + stream.Attach(req); // Request <-> Stream + session.Finalize(req); // Request -> Session & remove the lookup + + AssertCritical(stream.request?.Session == session); + + break; + + // ID -> same events + // SrcDep -> QUIC_SESSION_CREATED, HTTP_STREAM_REQUEST_BOUND_TO_QUIC_SESSION, QUIC_SESSION_POOL_USE_EXISTING_SESSION + case "QUIC_SESSION_POOL_MATCHING_IP_SESSION_FOUND": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION_POOL")); + + session = this.SessionFromSrcDep(in evt); + if (session != null) + this.SessionAddUID(session, in evt); + + break; + + /* + Adjacent: + HTTP_STREAM_JOB_INIT_CONNECTION.Begin + QUIC_SESSION_POOL_USE_EXISTING_SESSION : SrcDep -> QUIC_SESSION_CREATED + QUIC_SESSION_POOL_ATTACH_HTTP_STREAM_JOB_TO_EXISTING_SESSION : SrcDep -> HTTP_STREAM_REQUEST_STARTED_JOB + ... + HTTP_STREAM_JOB_INIT_CONNECTION.End + */ + + // ID -> HTTP_STREAM_JOB_INIT_CONNECTION + // SrcDep -> QUIC_SESSION_CREATED, QUIC_SESSION_POOL_JOB_RESULT, HTTP_STREAM_REQUEST_BOUND_TO_QUIC_SESSION + // "destination":"", "source_dependency":... + case "QUIC_SESSION_POOL_USE_EXISTING_SESSION": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + // Pass the JSON on to the next event: QUIC_SESSION_POOL_ATTACH_HTTP_STREAM_JOB_TO_EXISTING_SESSION + this.StashJSON(in evt); + + break; + + // ID -> QUIC_SESSION, etc. + // SrcDep -> HTTP_STREAM_REQUEST_STARTED_JOB, HTTP_STREAM_REQUEST_BOUND_TO_JOB + case "QUIC_SESSION_POOL_ATTACH_HTTP_STREAM_JOB_TO_EXISTING_SESSION": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + strJSON = this.UnstashJSON("QUIC_SESSION_POOL_USE_EXISTING_SESSION", in evt); + rgje = ParseSimpleJsonString(strJSON, rgstrDestination_SourceId); + + req = this.RequestFromSrcDep(in evt); + if (req != null) + { + AssertImportant(req.Type == StreamType.Unknown); // not yet determined + + if (req.SessionQUIC != null) break; + + session = this.SessionFromUID(in evt); + AssertInfo(session != null); + if (session == null) + { + // The QUIC_SESSION event must have happened before tracing started. + session = req.NewPlaceholderSession(StreamType.QUIC, in evt); + this.sessionTable.Add(session); + this.SessionAttachUID_SrcDep(session, in evt); // sets: session.uidVal/.srcdep + } + + // Also associate the SrcDep of the paired event: QUIC_SESSION_POOL_USE_EXISTING_SESSION + if (rgje != null) + { + srcdep = rgje[1].MyGetNumber(); + this.SessionAddSrcDep(session, srcdep, in evt); + } + + req.SessionQUIC = session; + + break; + } + + // This is probably a "preconnect" Stream Job. + // Reference a preconnect Request and Session, if available. + + if (rgje == null) break; + + srcdep = rgje[1].MyGetNumber(); + session = this.SessionFromSrcDep(srcdep, in evt); + if (session == null) + session = this.SessionFromUID(in evt); + else + AssertImportant(session == this.SessionFromUID(in evt)); + + AssertImportant(session != null); + if (session == null) + { + session = new Session(StreamType.QUIC, in evt); + this.sessionTable.Add(session); + this.SessionAttachUID_SrcDep(session, in evt); + } + + strURL = rgje[0].MyGetString(); + req = this.FindPreconnectRequest(strURL, session.PreKey, in evt); + if (req == null) break; + + // A preconnect HTTP Stream Job has a placeholder Request that connects in a different way. + // See: HTTP_STREAM_JOB_CONTROLLER + // cf. HTTP2_SESSION_POOL_FOUND_EXISTING_SESSION + + AssertCritical(req.IsPreconnect); + AssertCritical(req.URL.StartsWith(strURL)); // "destination" string + + // A Preconnect Request has no Stream, so it simply refers to a Session, and not vice-versa. + AssertImportant(req.Session == session || req.Session == null); + req.SessionSet = session; + + AssertImportant(req.Type == StreamType.QUIC); + + if (!(session.socket?.addrRemote).Empty() && req.port == 0) + { + req.ipAddr = session.socket.addrRemote.Address.ToString(); + req.port = (ushort)session.socket.addrRemote.PortGraphable(); + } + + break; + + // ID -> QUIC_SESSION + // peer_address:##.##.##.##, self_address:##.##.##.##, size:##7215 + case "QUIC_SESSION_PACKET_RECEIVED": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("QUIC_SESSION")); + + session = this.SessionFromUID(in evt); + AssertInfo(session != null); + if (session == null) break; + + soc = session.socket; + AssertInfo(soc != null); + if (soc == null) + { + // Reconstruct the missing Socket. + + AssertImportant(session.socketPreMigrate == null); // else how!? + + rgje = evt.ParseSimpleJsonString(rgstrPeer_Self); + if (rgje == null) break; + + if (!DNSClient.DNSTable.TryParseWithPort(rgje[0].MyGetString(), out ipAddress, out port)) break; + + soc = new Socket(StreamType.QUIC, in evt); + + soc.addrRemote = new IPEndPoint(ipAddress, port); + AssertImportant(session.port == 0 || session.port == port); + session.port = port; + + if (DNSClient.DNSTable.TryParseWithPort(rgje[1].MyGetString(), out ipAddress, out port)) + soc.addrLocal = new IPEndPoint(ipAddress, port); + + session.Attach(soc); + + break; + } +#if DEBUG + // Confirm the addresses. + + rgje = evt.ParseSimpleJsonString(rgstrPeer_Self); + if (rgje == null) break; + + if (DNSClient.DNSTable.TryParseWithPort(rgje[0].MyGetString(), out ipAddress, out port)) + { + IPEndPoint ipep = new IPEndPoint(ipAddress, port); + AssertImportant(ipep.Equals(soc.addrRemote)); + AssertImportant(session.port == port); + } + + if (DNSClient.DNSTable.TryParseWithPort(rgje[1].MyGetString(), out ipAddress, out port)) + { + IPEndPoint ipep = new IPEndPoint(ipAddress, port); + AssertImportant(ipep.Equals(soc.addrLocal) || ipep.Equals(session.socketPreMigrate?.addrLocal)); + } +#endif // DEBUG + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Quic + + /* + evt.TaskName must begin with "SOCKET_" + */ + void Dispatch_Socket(in IGenericEvent evt) + { + int srcdep; + int err; + UIDVal uID; + Request req; + Socket soc; + JsonElement[] rgje; + + AssertCritical(evt.TaskName.StartsWith("SOCKET_")); + + switch (evt.TaskName) + { + // ID -> SSL_CONNECT_JOB_CONNECT, TRANSPORT_CONNECT_JOB_CONNECT_ATTEMPT, CONNECT_JOB_SET_SOCKET, HOST_RESOLVER_MANAGER_CACHE_HIT + // source_type: SSL_CONNECT_JOB, TRANSPORT_CONNECT_JOB + case "SOCKET_POOL_CONNECT_JOB_CREATED": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("SSL_CONNECT_JOB") || evt.CheckSourceType("TRANSPORT_CONNECT_JOB")); + + rgje = evt.ParseSimpleJsonString(rgstrGroupId); + if (rgje == null) break; + + srcdep = rgje[0].MyGetString().GetHashCode(); // pseudo-srcdep: hash of group_id string + + // Common sequence: + // TCP_CLIENT_SOCKET_POOL_REQUESTED_SOCKET/S + // SOCKET_POOL*.Begin // ignored? + // SOCKET_POOL_STALLED_MAX_SOCKETS_PER_GROUP // occasionally + // CONNECT_JOB.Begin // ignored + // SOCKET_POOL_CONNECT_JOB_CREATED // << this event + + req = this.RequestFromGroupId_Correlate(in evt, srcdep, "TCP_CLIENT_SOCKET_POOL_REQUESTED_SOCKET", "TCP_CLIENT_SOCKET_POOL_REQUESTED_SOCKETS"); + if (req == null) + { + req = this.RequestFromSrcDep(srcdep, in evt); + if (req == null) break; + } + + AssertImportant(this.RequestFromUID(in evt) == null); + this.RequestAttachUID(req, in evt); + + break; + + // ID -> SOCKET_ALIVE + case "SOCKET_CLOSED": + case "SOCKET_POOL_CLOSING_SOCKET": + AssertImportant(evt.IsInstantPhase()); + + if (!evt.CheckSourceType("SOCKET")) break; + + soc = this.SocketFromUID(in evt, false); + AssertInfo(soc != null); + if (soc == null) break; + + soc.Close(evt.Timestamp.ToGraphable()); + + break; + + // Socket / WinSock-related Events + // These four events are adjacent within the trace for a given thread (except when Phase = End). + + // source_type: + // SOCKET: ID -> SSL_CONNECT, TCP_CONNECT, TCP_CONNECT_ATTEMPT, SOCKET_BYTES_SENT/RECEIVED, SOCKET_IN_USE, SOCKET_CLOSED + // UDP_SOCKET: ID -> UDP_CONNECT, UDP_LOCAL_ADDRESS, UDP_BYTES_SENT/RECEIVED + // UDP_CLIENT_SOCKET: ID -> SOCKET_OPEN, SOCKET_CONNECT + case "SOCKET_ALIVE": + if (!evt.IsBeginPhase()) + break; + + switch (evt.GetSourceType()) + { + /* + TCP: Six adjacent Chromium events: + - SOCKET_ALIVE / SOCKET: Create a new Socket. (The next event is correlated/adjacent.) + - TRANSPORT_CONNECT_JOB_CONNECT_ATTEMPT: Get the IP Address and attach the Socket to a Request. (The next event is correlated/adjacent.) + - TCP_CONNECT / SOCKET: Set the time span for capturing the Winsock Connection. + - TCP_CONNECT_ATTEMPT / SOCKET: Capture the intervening Winsock Connection creation. + - TCP_CONNECT_ATTEMPT.End: No operation. + - TCP_CONNECT.End: Get the Local and Remote IP Addresses. + */ + case "SOCKET": + soc = new Socket(StreamType.TCP, in evt); + + this.socketTable.Add(soc); + + // Also sets the recent Socket for: TRANSPORT_CONNECT_JOB_CONNECT_ATTEMPT + this.SocketAttachUID_SrcDep(soc, in evt); + + break; + + /* + UDP: Three UDP_CLIENT_SOCKET events with interspersed UDP_SOCKET events: + x SOCKET_ALIVE / UDP_SOCKET: (redundant?) + - SOCKET_ALIVE / UDP_CLIENT_SOCKET: Determine that it's a QUIC Socket, create a new Socket and attach the UID to the Request. + - SOCKET_OPEN / UDP_CLIENT_SOCKET: Capture the intervening Winsock Connection creation. + x UDP_CONNECT / UDP_SOCKET: IP Address (redundant? UDP_CONNECT.End carries "net_error") + - SOCKET_CONNECT/UDP_CLIENT_SOCKET: Get the IP Address and confirm (from the port) that it's not DNS. + x UDP_LOCAL_ADDRESS / UDP_SOCKET: LocalAddress:Socket + x UDP_BYTES_SENT/RECEIVED / UDP_SOCKET + The UDP_SOCKET events are seemingly redundant, but the ID captures later SENT/RECEIVED events. + */ + case "UDP_SOCKET": + this.SetRecentUID(in evt); // This event's uID will be linked to the associated Socket. + break; + + case "UDP_CLIENT_SOCKET": + srcdep = evt.GetSourceId(); + + // Ignore events associated with a DNS_TRANSACTION. + if (this.IsDNSSrcDep(in evt, srcdep)) break; + + soc = new Socket(StreamType.QUIC, in evt); + + this.socketTable.Add(soc); + + // Attach the UID from the SOCKET_ALIVE/UDP_SOCKET event to this Request & Socket. + uID = this.GetRecentUID(evt.TaskName, in evt); + if (uID != 0) + this.SocketAttachUID(soc, uID, in evt); + + this.SocketAttachUID_SrcDep(soc, in evt); + + // This source dependency id should also match: BOUND_TO_QUIC_SESSION_POOL_JOB & QUIC_SESSION + // Otherwise this is not the UDP Socket that we want. (It's probably for DNS.) + req = this.RequestFromSrcDep(in evt); + if (req == null) break; // common + + if (uID != 0) + req.AddUID(uID); + + req.AddUID(evt.GetUID()); + + break; + + default: + AssertImportant(false); // else what? + break; + } + break; + + // ID -> SOCKET_ALIVE + case "SOCKET_IN_USE": + AssertImportant(!evt.IsInstantPhase()); + if (!evt.IsBeginPhase()) break; + + AssertImportant(evt.CheckSourceType("SOCKET")); + + soc = this.SocketFromUID(in evt); + if (soc == null) break; + + this.SocketAttachSrcDep(soc, evt.GetSourceId(), in evt); + + break; + + // ID -> SOCKET_ALIVE/UDP_CLIENT_SOCKET + case "SOCKET_OPEN": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("UDP_CLIENT_SOCKET")); // Must be a UDP Socket. + + this.AttachWinsockConnection(in evt, WinsockAFD.IPPROTO.UDP); + + break; + + // ID -> SOCKET_ALIVE/UDP_CLIENT_SOCKET + // "address":"#.#.#.#:#", "net_error":# + // source_type: UDP_CLIENT_SOCKET + case "SOCKET_CONNECT": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("UDP_CLIENT_SOCKET")); // Must be a UDP Socket. + + soc = this.SocketFromUID(in evt); + if (soc == null) break; + + AssertImportant(!soc.Closed); + + rgje = evt.ParseSimpleJsonString(rgstrAddress_Error); + if (rgje == null) break; + + soc.SetAddrLocalRemote(null, rgje[0].MyGetString()); + + err = rgje[1].MyGetNumber(); + if (err != jsonIntDefault) + soc.iError = err; + + break; + + // ID -> HTTP_STREAM_JOB, etc. + // SrcDep -> TRANSPORT_CONNECT_JOB_CONNECT_ATTEMPT, CONNECT_JOB_SET_SOCKET, HTTP2_SESSION_INITIALIZED + // NOTE: Link the TCP Stream Job to its TCP Socket and set the type. + case "SOCKET_POOL_BOUND_TO_SOCKET": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + req = this.RequestFromUID(in evt); + AssertImportant(req != null); + if (req == null) break; + + soc = this.SocketFromSrcDep(in evt); + AssertImportant(soc != null); + if (soc == null) break; + + soc.uidBound = evt.GetUID(); + + if (req.FAttachPlaceholderSessionAndStream(soc, in evt)) + { + this.sessionTable.Add(req.Session); + this.SessionAttachUID(req.Session, in evt); // sets: session.uidVal + } + + break; + + case "SOCKET_POOL_STALLED_MAX_SOCKETS_PER_GROUP": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + const string strTaskLink = "TCP_CLIENT_SOCKET_POOL_REQUESTED_SOCKET"; + + req = this.RequestFromUID_Correlate(in evt, strTaskLink); + + if (req != null) + this.StashStalledRequest(req, in evt, strTaskLink); + + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Socket + + /* + evt.TaskName must begin with "TCP_" + */ + void Dispatch_Tcp(in IGenericEvent evt) + { + int srcdep, hidGroup; + Request req; + Socket soc; + TimestampUI timeStamp; + JsonElement[] rgje; + + AssertCritical(evt.TaskName.StartsWith("TCP_")); + + switch (evt.TaskName) + { + // ID -> HTTP_STREAM_JOB, HTTP_STREAM_JOB_BOUND_TO_REQUEST, CANCELLED + // srcdep -> (hashed "group_id") SOCKET_POOL_CONNECT_JOB_CREATED + case "TCP_CLIENT_SOCKET_POOL_REQUESTED_SOCKET": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + rgje = evt.ParseSimpleJsonString(rgstrGroupId); + if (rgje == null) break; + + // Look up the Request and attach the hash of the group_id string (hidGroup) as a srcdep. + + hidGroup = srcdep = rgje[0].MyGetString().GetHashCode(); + + req = this.RequestFromUID(in evt); + if (req == null) break; + + if (req.hidGroup == 0) + req.hidGroup = hidGroup; + else + AssertImportant(req.hidGroup == hidGroup); + + this.RequestAttachSrcDep(req, srcdep, in evt); // reattach/replace + + break; + + // ID -> HTTP_STREAM_JOB, HTTP_STREAM_JOB_BOUND_TO_REQUEST, CANCELLED + // srcdep -> (hashed "group_id") SOCKET_POOL_CONNECT_JOB_CREATED + // NOTE: This refers to PRECONNECT sockets. cf. HTTP_STREAM_JOB_CONTROLLER is_preconnect:true + case "TCP_CLIENT_SOCKET_POOL_REQUESTED_SOCKETS": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("HTTP_STREAM_JOB")); + + rgje = evt.ParseSimpleJsonString(rgstrGroupId); + if (rgje == null) break; + + string strGroupId = rgje[0].MyGetString(); + if (string.IsNullOrWhiteSpace(strGroupId)) break; + + // Look up the Request and attach the hash of the group_id string (hidGroup) as a srcdep. + + hidGroup = srcdep = strGroupId.GetHashCode(); + + AssertImportant(this.RequestFromUID(in evt) == null); + + req = this.RequestFromSrcDep(srcdep, in evt); + + if (req == null || !req.IsPreconnect) + { + string strURLBase = strGroupId.BaseURLFromGroupId(); + + req = this.FindPreconnectRequest(strURLBase, hidGroup, in evt); + + if (req == null) break; + } + + // req.FAttachPlaceholderSessionAndStream will be done when a Socket becomes available in: CONNECT_JOB_SET_SOCKET + + AssertCritical(req.TypeTCP != StreamType.QUIC); + if (req.TypeTCP == StreamType.Unknown) + req.TypeTCP = StreamType.TCP; + + if (req.hidGroup == 0) + req.hidGroup = hidGroup; + else + AssertImportant(req.hidGroup == hidGroup); + + this.RequestAttachSrcDep(req, srcdep, in evt); // reattach/replace + + break; + + // "address_list":["#.#.#.#:#", ...] + // .End: "local_address":"#.#.#.#:#", "remote_addess":"#.#.#.#:#", "net_error":# + case "TCP_CONNECT": + soc = this.SocketFromUID(in evt); + if (soc == null) break; + + AssertImportant(!soc.Closed); + + if (!evt.IsEndPhase()) + { + AssertImportant(evt.CheckSourceType("SOCKET")); + + timeStamp = evt.Timestamp.ToGraphable(); + soc.timeStampConnect = timeStamp; + + break; + } + + rgje = evt.ParseSimpleJsonString(rgstrLocal_Remote); + if (rgje == null) break; + + soc.SetAddrLocalRemote(rgje[0].MyGetString(), rgje[1].MyGetString()); + + break; + + // "address":"#.#.#.#:#" + // .End: "os_error":# + case "TCP_CONNECT_ATTEMPT": + if (evt.IsEndPhase()) + break; + + AssertImportant(evt.CheckSourceType("SOCKET")); + + this.AttachWinsockConnection(in evt, WinsockAFD.IPPROTO.TCP); + + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Tcp + + /* + evt.TaskName must begin with "URL_" + */ + void Dispatch_Url(in IGenericEvent evt) + { + uint cb; + Request req; + TimestampUI timeStamp; + JsonElement[] rgje; + + AssertCritical(evt.TaskName.StartsWith("URL_")); + + switch (evt.TaskName) + { + // ID -> REQUEST_ALIVE; HTTP_STREAM_REQUEST, HTTP_STREAM_JOB_CONTROLLER_BOUND, HTTP_STREAM_REQUEST_BOUND_TO_JOB, HTTP_STREAM_REQUEST_BOUND_TO_QUIC_SESSION + // "initiator":"", "method","", "network_isolation_key":" ", "url","" + // .End: "net_error":# + // NOTE: A Redirect (301 or 302) will create multiple URL_REQUEST_START_JOB for a single REQUEST_ALIVE. + case "URL_REQUEST_START_JOB": + timeStamp = evt.Timestamp.ToGraphable(); + + req = this.RequestFromUID(in evt, false); + + if (!evt.IsBeginPhase()) + { + if (req == null) break; + + req.SetNetError(in evt); + + AssertImportant(!req.Closed); + req.Close(in timeStamp); + + break; + } + + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + Priority priRedir = Priority.Unknown; + bool fRedirect = false; + if (req != null && req.Closed) + { + priRedir = req.priority; + fRedirect = req.fRedirect; + req = null; + } + + if (req == null) + { + req = new Request(in evt) + { + priority = priRedir + }; + + req.xlink.GetLink(evt.ThreadId, timeStamp, in this.allTables.threadTable); + + this.Add(req); + this.RequestAttachUID(req, in evt); + } + + AssertImportant(!req.fRedirect); + AssertImportant(req.method == null); + + req.uidRequest = evt.GetUID(); + req.timeStampBeginJob = timeStamp; // Overwrite the time from: REQUEST_ALIVE + + rgje = evt.ParseSimpleJsonString(rgstrURL_Method); + if (rgje == null) break; + + req.method = rgje[1].MyGetString(); + + if (fRedirect) + req.method = "REDIRECT/" + req.method; + + if (req.URL == null) + { + req.URL = rgje[0].MyGetString(); + } + else + { + AssertImportant(req.URL.Equals(rgje[0].MyGetString())); + AssertImportant(req.URLScrub.Equals(rgje[0].MyGetString().Split('#', 2)[0])); + } + + break; + + // ID -> REQUEST_ALIVE, URL_REQUEST_START_JOB, etc. + // "byte_count":# + // NOTE: This event exists even when the data wasn't compressed ("filtered"). + // NOTE: cf. UPLOAD_DATA_STREAM_INIT + case "URL_REQUEST_JOB_FILTERED_BYTES_READ": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID(in evt, false); + AssertInfo(req != null); + if (req == null) break; + + rgje = evt.ParseSimpleJsonString(rgstrBytes); + if (rgje == null) break; + + cb = rgje[0].MyGetUNumber(0); + AssertCritical((int)cb >= 0); + + req.cbDownload += cb; + + if (req.stream == null) break; + + AssertCritical(req.Session != null); + AssertImportant(req.Session?.Type == req.Type); + + req.stream.cbDownload += cb; + + req.stream.SetLastTime(evt.Timestamp.ToGraphable()); + + break; + + // ID -> REQUEST_ALIVE, URL_REQUEST_START_JOB x2, etc. + // "location":"" + case "URL_REQUEST_REDIRECTED": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID(in evt, false); + AssertImportant(req != null); + if (req != null) + req.fRedirect = true; + + break; + + // ID -> REQUEST_ALIVE, URL_REQUEST_START_JOB, etc. + // "priority":"" + case "URL_REQUEST_SET_PRIORITY": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID(in evt); + if (req == null) break; + + rgje = evt.ParseSimpleJsonString(rgstrPriority); + if (rgje == null) break; + + req.priority = rgje[0].MyGetString().GetPriority(); + + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Url + + /* + evt.TaskName must contain "CONNECT_" + */ + void Dispatch_Connect(in IGenericEvent evt) + { + ushort port; + int srcdep; + Request req; + Socket soc; + IPAddress ipAddress; + JsonElement[] rgje; + + AssertCritical(evt.TaskName.Contains("CONNECT_")); + + switch (evt.TaskName) + { + // ID -> SOCKET_POOL_CONNECT_JOB_CREATED + // srcdep -> TRANSPORT_CONNECT_JOB_CONNECT_ATTEMPT, SOCKET_POOL_BOUND_TO_SOCKET + // source_type: SSL_CONNECT_JOB, TRANSPORT_CONNECT_JOB + case "CONNECT_JOB_SET_SOCKET": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("SSL_CONNECT_JOB") || evt.CheckSourceType("TRANSPORT_CONNECT_JOB")); + + srcdep = evt.GetSourceId(); + + soc = this.SocketFromUID(in evt); + if (soc == null) + soc = this.SocketFromSrcDep(srcdep, in evt); + + if (soc == null) break; + + this.SocketAttachUID_SrcDep(soc, srcdep, in evt); + + this.ResolverManagerAttachSrcDep(srcdep, in evt); + + req = this.RequestFromUID(in evt); + if (req == null) break; + + if (req.port == 0) + req.port = (ushort)soc.addrRemote.PortGraphable(); + + // If this is a Preconnect Request then this is our last opportunity to assign a Socket/Session. + // Otherwise wait for a binding event: SOCKET_POOL_BOUND_TO_SOCKET + + if (!req.IsPreconnect) break; + + if (req.FAttachPlaceholderSessionAndStream(soc, in evt)) + { + this.sessionTable.Add(req.Session); + this.SessionAttachUID(req.Session, in evt); // sets: session.uidVal + } + + break; + + // ID -> CONNECT_JOB, etc. + // .End: "net_error":# + // source_type: TRANSPORT/SSL_CONNECT_JOB + case "SSL_CONNECT_JOB_CONNECT": + case "TRANSPORT_CONNECT_JOB_CONNECT": + AssertCritical(!evt.IsInstantPhase()); + + req = this.RequestFromUID(in evt); + if (req == null) break; + + if (evt.TaskName.StartsWith("SSL")) + req.fSSL = true; + + if (evt.IsBeginPhase()) + { + AssertImportant(evt.CheckSourceType("SSL_CONNECT_JOB") || evt.CheckSourceType("TRANSPORT_CONNECT_JOB")); + break; + } + + req.SetNetError(in evt); + + break; + + // ID -> SOCKET_POOL_CONNECT_JOB_CREATED + case "TRANSPORT_CONNECT_JOB_CONNECT_ATTEMPT": + AssertImportant(evt.IsInstantPhase()); + + soc = this.SocketFromRecent(in evt, "SOCKET_ALIVE"); + + this.ResetRecent(evt.ProcessId, evt.ThreadId); // No adjacent event to track, yet. + + AssertImportant(soc != null); + if (soc == null) break; + + rgje = evt.ParseSimpleJsonString(rgstrSourceId_Address); + if (rgje == null) break; + + srcdep = rgje[0].MyGetNumber(); + this.SocketAttachUID_SrcDep(soc, srcdep, evt); + + if (DNSClient.DNSTable.TryParseWithPort(rgje[1].MyGetString(), out ipAddress, out port)) + soc.addrRemote = new IPEndPoint(ipAddress, port); + + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Connect + + /* + evt.TaskName must NOT match any of the other standard in/prefix strings. + */ + void Dispatch_Misc(in IGenericEvent evt) + { + int srcdep; + int err; + uint cb; + Request req; + Socket soc; + TimestampUI timeStamp; + JsonElement[] rgje; + + switch (evt.TaskName) + { + // ID -> REQUEST_ALIVE, URL_REQUEST_START_JOB, etc. + // NOTE: This is (the first and) the last time that we see any activity on this Request. + case "CORS_REQUEST": + if (evt.IsEndPhase()) + { + req = this.RequestFromUID(in evt, false); + AssertImportant(req != null); + if (req == null) break; + + req.Gone = true; + this.GarbageCollect(in evt); + + break; + } + + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + // REQUEST_ALIVE: Sometimes this UID is different from that one. + this.SetRecentUID(in evt); + + break; + + // ID -> WEBSOCKET_STATE_CHANGED + // NOTE: This event effectively takes the place of CORS_REQUEST for a WebSocket Channel. + case "WEBSOCKET_ALIVE": + if (evt.IsEndPhase()) + { + req = this.RequestFromUID(in evt, false); + AssertImportant(req != null); + if (req == null) break; + if (!req.Closed) break; + + req.Gone = true; + this.GarbageCollect(in evt); + + break; + } + + AssertImportant(evt.CheckSourceType("WEBSOCKET_CHANNEL")); + AssertImportant(evt.ParseSimpleJsonString(new[]{"state"})?[0].MyGetString() == "FRESHLY_CONSTRUCTED"); // else see the event: WEBSOCKET_STATE_CHANGED + + // for: REQUEST_ALIVE + this.SetRecentUID(in evt); + + break; + + // ID -> CORS_REQUEST, URL_REQUEST_START_JOB, HTTP_STREAM_REQUEST, HTTP_STREAM_JOB_CONTROLLER_BOUND, HTTP_STREAM_REQUEST_BOUND_TO_JOB, HTTP_STREAM_REQUEST_BOUND_TO_QUIC_SESSION + // "priority":"", "url":"" + // This is the first event of interest in a Request, and the event for which we capture a StackWalk. + case "REQUEST_ALIVE": + timeStamp = evt.Timestamp.ToGraphable(); + + if (!evt.IsBeginPhase()) + { + req = this.RequestFromUID(in evt, false); + AssertImportant(req != null); + if (req == null) break; + + req.SetNetError(in evt); + if (req.timeStampEndJob.HasMaxValue()) + req.timeStampEndJob = timeStamp; // just in case + + break; + } + + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + // If the previous event was: WEBSOCKET_ALIVE + UIDVal uid = this.GetRecentUID("CORS_REQUEST", in evt); + + bool _fWebSocket = false; + + if (uid == 0) + { + uid = this.GetRecentUID("WEBSOCKET_ALIVE", in evt); + if (uid != 0) + _fWebSocket = true; + } + + AssertImportant(this.RequestFromUID(in evt) == null); + + rgje = evt.ParseSimpleJsonString(rgstrURL_Priority); + if (rgje == null) break; + + string strURL = rgje[0].MyGetString(); + + req = new Request(strURL, in evt) + { + fWebSocket = _fWebSocket, + priority = rgje[1].MyGetString().GetPriority(), + timeStampBeginJob = timeStamp // to be overwritten by: URL_REQUEST_START_JOB + }; + + req.xlink.GetLink(evt.ThreadId, timeStamp, in this.allTables.threadTable); + + this.Add(req); + this.RequestAttachUID(req, in evt); + + if (uid != 0 && uid != evt.GetUID()) + this.RequestAttachUID(req, uid, in evt); + + break; + + // ID -> REQUEST_ALIVE, etc. + // SrcDep -> HTTP_STREAM_JOB_CONTROLLER_BOUND, HTTP_STREAM_JOB_BOUND_TO_REQUEST + // source_type: URL_REQUEST + // NOTE: Add the SrcDep to the Request identified by the ID for future correlation. + case "CREATED_BY": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID_SrcDep(in evt); + + break; + + // ID -> the thing being canceled + // source_type: SOCKET, URL_REQUEST, HTTP_STREAM_JOB, CERT_VERIFIER_JOB + // One of several things is canceled, depending on the source_type. + case "CANCELLED": + AssertImportant(evt.IsInstantPhase()); + + switch (evt.GetSourceType()) + { + case "SOCKET": + soc = this.SocketFromUID(in evt, false); + if (soc == null) break; + + // This event is followed by: SOCKET_CLOSED + soc.fCanceled = true; + + break; + + case "URL_REQUEST": + // The Request may have already been closed by: URL_REQUEST_START_JOB.End + req = this.RequestFromUID(in evt, false); + AssertImportant(req != null); + if (req == null) break; + + req.SetNetError(in evt); + req.fCanceled = true; + + break; +#if DEBUG + case "HTTP_STREAM_JOB": + break; + + case "HOST_RESOLVER_IMPL_JOB": + break; + + case "CERT_VERIFIER_JOB": + case "PAC_FILE_DECIDER": + break; + + default: + AssertImportant(false); // Else what? + break; +#endif // DEBUG + } + break; + + // 'Instant' version: + // ID -> HOST_RESOLVER_MANAGER_JOB, etc. + // SrcDep -> SOCKET_ALIVE + // source_type: HOST_RESOLVER_IMPL_JOB + // NOTE: If the SrcDep of a SOCKET_ALIVE/UDP_CLIENT_SOCKET matches this one then it is a DNS Socket. + case "DNS_TRANSACTION": + if (!evt.IsInstantPhase()) + break; + + AssertImportant(evt.CheckSourceType("HOST_RESOLVER_IMPL_JOB")); // else what? + + srcdep = evt.GetSourceId(); + this.AddDNSSrcDep(in evt, srcdep); + + break; + + // ID -> SOCKET_ALIVE/UDP_SOCKET + // source_type: UDP_SOCKET + case "UDP_LOCAL_ADDRESS": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("UDP_SOCKET")); + + soc = this.SocketFromUID(in evt); + if (soc == null) break; // common + + if (!soc.addrLocal.Empty()) break; + + rgje = evt.ParseSimpleJsonString(rgstrAddress); + if (rgje == null) break; + + soc.SetAddrLocalRemote(rgje[0].MyGetString(), null); + + break; + + // ID -> REQUEST_ALIVE, URL_REQUEST_START_JOB + // "total_size":#, "is_chunked":bool, "net_error":# + // NOTE: For POST Requests, get the size of the object being uploaded. + // NOTE: cf. URL_REQUEST_JOB_FILTERED_BYTES_READ + case "UPLOAD_DATA_STREAM_INIT": + if (!evt.IsEndPhase()) + { + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + break; + } + + req = this.RequestFromUID(in evt); + AssertInfo(req != null); + if (req == null) break; + + rgje = evt.ParseSimpleJsonString(rgstrSize_Chunked_Error); + if (rgje == null) break; + + err = rgje[2].MyGetNumber(int.MinValue); + + // This event should have: net_error & is_chunked & total_size, or net_error alone, or nothing. + // If there is no net_error then the event contains nothing of interest. + if (err == int.MinValue) break; + + cb = rgje[0].MyGetUNumber(0); + AssertCritical((int)cb >= 0); + + if (err != 0) + req.iError = err; + else if (rgje[1].MyGetBool()) // is_chunked + req.fChunkedUpload = true; + else + req.cbUpload += cb; + + AssertImportant(!req.fChunkedUpload); // else invalid cbSend? + + // HTTP1: A placeholder Session & Stream were already created. + // HTTP2/3: The Session was already created. The Stream will be created later. + if (req.stream == null) + { + AssertImportant(req.Type == StreamType.HTTP2 || req.Type == StreamType.QUIC); + break; + } + + AssertImportant(req.Type == StreamType.HTTP1); + AssertImportant(req.Session != null && req.Session.Type == StreamType.HTTP1); + + if (err != 0) + req.stream.iError = err; + else if (rgje[1].MyGetBool()) // is_chunked + req.stream.fChunkedUpload = true; + else + req.stream.cbUpload += cb; + + AssertImportant(!req.stream.fChunkedUpload); // else invalid dbSend? + + req.stream.SetLastTime(evt.Timestamp.ToGraphable()); + + break; + + case "WEBSOCKET_SENT_FRAME_HEADER": + case "WEBSOCKET_RECV_FRAME_HEADER": + AssertImportant(evt.IsInstantPhase()); + AssertImportant(evt.CheckSourceType("URL_REQUEST")); + + req = this.RequestFromUID(in evt, false); + AssertImportant(req != null); + if (req == null) break; + + AssertImportant(!req.Gone); + AssertImportant(req.fWebSocket || req.URLScrub.StartsWith("wss:")); + req.fWebSocket = true; + + rgje = evt.ParseSimpleJsonString(rgstrOpcode_Payload); + if (rgje == null) break; + + // opcode >= 8 are control frames + if (rgje[0].MyGetUNumber() >= 8) break; + + cb = (uint)rgje[1].MyGetDecimal(); + if (cb == 0) break; + + if (evt.TaskName.Equals("WEBSOCKET_SENT_FRAME_HEADER")) + { + req.cbUpload += cb; + if (req.stream != null) + req.stream.cbUpload += cb; + } + else + { + req.cbDownload += cb; + if (req.stream != null) + req.stream.cbDownload += cb; + } + + break; + + case "SSL_CONNECT": + if (!evt.IsEndPhase()) + { + AssertImportant(evt.CheckSourceType("SOCKET")); + break; + } + + soc = this.SocketFromUID(in evt); + if (soc == null) break; // Canceled + + soc.fSSL = true; + + rgje = evt.ParseSimpleJsonString(rgstrProto2); + if (rgje == null) break; + + string proto = rgje[0].MyGetString(); + if (string.IsNullOrWhiteSpace(proto)) + { + err = evt.GetNetError(); + soc.iError = err; + break; + } + + AssertImportant(proto.Equals("h2") || proto.Equals("http/1.1") || proto.Equals("unknown")); + AssertImportant(soc.Type == StreamType.TCP); + + soc.Type = (proto.Equals("h2") ? StreamType.HTTP2 : StreamType.HTTP1); + + break; + + default: + // Remember this event as unhandled. + this.unhandled.Add(evt.TaskName); + break; + } // switch evt.TaskName + } // Dispatch_Misc + + + /* + DEBUG-only: Emit a text file of that can be pasted into a WPA View Profile: .wpaProfile + The text file goes to: %TEMP%\.Annotations.txt + It contains a WPA annotation () for each Request, which will attach in WPA to the ETL events related to that Request. + The name of each annotation will be the timestamp of the Request's event: URL_REQUEST_START_JOB + + Paste the generated XML for within the "Annotation" Column's XML in the .wpaProfile: + + + + + + + + + + + + WARNING: WPA can be VERY slow with more than a few dozen Annotation Queries. + */ + [Conditional("DEBUG")] + public void _EmitViewProfileAnnotationQueriesDB() + { + System.Text.StringBuilder sb = new(512); + + string pathETL = this.allTables.traceMetadata.TracePath; + sb.AppendFormat("{0}{1}.Annotations.txt", Path.GetTempPath(), Path.GetFileNameWithoutExtension(pathETL)); + + using (StreamWriter writer = new StreamWriter(sb.ToString())) // could throw + { + string strTab = " "; // 18 + + // Prolog to the tags: + writer.WriteLine(" "); + writer.Write(strTab); + writer.WriteLine(""); + writer.Write(strTab); + writer.WriteLine(" "); + + // Write an for each Request, where the name is the opening timestamp: + + foreach (Request req in this) + { + sb.Clear(); + sb.Append(" string.Format("[Field 3]:={0:#,#}", uID))); // clever but not too efficient + sb.Append("\" />"); + + writer.Write(strTab); + writer.WriteLine(sb.ToString()); + } + + // Epilog to the tags: + writer.Write(strTab); + writer.WriteLine(" "); + writer.Write(strTab); + writer.WriteLine(""); + writer.WriteLine(" "); + } // close writer + } // _EmitViewProfileAnnotationQueriesDB + + [Conditional("DEBUG")] + public void EmitViewProfileAnnotationQueriesDB() + { + try + { + _EmitViewProfileAnnotationQueriesDB(); + } + catch + { + // too bad, so sad + } + } + + } // class TraceTable +} // namespace NetBlameCustomDataSource.Chromium \ No newline at end of file diff --git a/src/NetBlame/Providers/DNSClient.cs b/src/NetBlame/Providers/DNSClient.cs index fc44cdc..9a5fd0b 100644 --- a/src/NetBlame/Providers/DNSClient.cs +++ b/src/NetBlame/Providers/DNSClient.cs @@ -219,13 +219,13 @@ string DNSNameAndAlt(uint iDNS, ref string strDNSAlt, int iAddr /*0-based*/) return strDNSName; } - string DNSNameAndAlt(IPAddress ipAddr, ref string strDNSAlt) + public string DNSNameAndAlt(IPAddress ipAddr, ref string strDNSAlt) { uint iDNS = IFindDNSEntryByIPAddress0(ipAddr, out int iAddr); return DNSNameAndAlt(iDNS, ref strDNSAlt, iAddr /*0-based*/); } - string GetServerNameAndAlt(string strURL /*opt*/, string strServer /*opt*/, string strServer2 /*opt*/, string strServer3 /*opt*/, ref string strServerAlt) + static string GetServerNameAndAlt(string strURL /*opt*/, string strServer /*opt*/, string strServer2 /*opt*/, string strServer3 /*opt*/, ref string strServerAlt) { AssertImportant(strServer != null); // Should be at least String.Empty or strNA @@ -251,8 +251,8 @@ string GetServerNameAndAlt(string strURL /*opt*/, string strServer /*opt*/, stri if (strServer3 != null) strServerAlt = strServer3; - AssertInfo(!strServer.IsNA()); - AssertInfo(!strServerAlt.IsNA()); + if (strServerAlt.IsNA()) + strServerAlt = string.Empty; if (string.IsNullOrEmpty(strServer)) strServer = Util.strNA; diff --git a/src/NetBlame/Providers/TcpIp.cs b/src/NetBlame/Providers/TcpIp.cs index cece454..4789644 100644 --- a/src/NetBlame/Providers/TcpIp.cs +++ b/src/NetBlame/Providers/TcpIp.cs @@ -356,7 +356,7 @@ public void Reset() tcbRCache = tcbr; } - new void Remove(TcbRecord tcbr) + static new void Remove(TcbRecord tcbr) { throw(new Exception("Remove not allowed. Elements are reference by index.")); } @@ -475,7 +475,7 @@ TcbRecord RestoreTcbRecord(QWord tcb, TcbRecord.MyStatus status, in TimestampETW /* - Return the 1-based index if the most recent, UDP receive event's TcbRecord with the given IP Address, cb, etc. + Return the 1-based index of the most recent, UDP receive event's TcbRecord with the given IP Address, cb, etc. Mark it as correlated with Winsock. */ public uint CorrelateUDPRecvEvent(IDVal pid, IDVal tid, uint cb, ushort socket, IPEndPoint ipAddr) @@ -917,7 +917,7 @@ public void Dispatch(in IGenericEvent evt) } // Here the actual ThreadId matches the ThreadId of WebIO.AFD.AcceptExWithAddress - cxn = this.allTables.wsTable.CorrelateListener(tcbr, pid, (pid==evt.ProcessId) ? evt.ThreadId : tidUnknown); + cxn = this.allTables.wsTable.CorrelateListener(tcbr, pid, (evt.ProcessId > TcbRecord.pidSystem) ? evt.ThreadId : tidUnknown); break; case TCP.CloseTcbRequest: diff --git a/src/NetBlame/Providers/WinsockAFD.cs b/src/NetBlame/Providers/WinsockAFD.cs index 52eb327..59f30be 100644 --- a/src/NetBlame/Providers/WinsockAFD.cs +++ b/src/NetBlame/Providers/WinsockAFD.cs @@ -116,6 +116,7 @@ public class Connection : IGraphableEntry public bool FClosed => !this.timeClose.HasMaxValue(); + public Protocol Protocol => Prominent((Protocol)this.grbitType); public Connection(AddrVal qwEndpoint, IDVal pid, IDVal tid, in TimestampETW timeStamp) { @@ -188,8 +189,8 @@ public class WinsockTable : List // This list/table can shrink (CloseConnection). Do not hold indices. - public Connection CxnFromI(uint iCxn) => null; - public uint IFromCxn(Connection cxn) => 0; + public static Connection CxnFromI(uint iCxn) => null; + public static uint IFromCxn(Connection cxn) => 0; /* @@ -398,7 +399,7 @@ public Connection CorrelateListener(TcpIp.TcbRecord tcbR, IDVal pid, IDVal tid) { cxn = this.FindLast(c => c.tidConnect == tid && - c.socket == tcbR.socket && + (c.socket == 0 || c.socket == tcbR.socket) && (c.addrRemote.Empty() || c.addrRemote.Equals(tcbR.addrRemote)) ); } @@ -585,7 +586,7 @@ void CloseConnection(in IGenericEvent evt) Nested AFD events can double-count. Empirically filter out the double-counters according to their location code. */ - bool FKnownDoubleCountEvent(uint location) + static bool FKnownDoubleCountEvent(uint location) { switch (location) { @@ -668,7 +669,7 @@ bool FKnownDoubleCountEvent(uint location) /*Exit*/ 7022, // AfdSend - /*Enter*/ 3003, 3047, 3058, 3100, + /*Enter*/ 3003, 3047, 3057, 3058, 3100, /*Exit*/ 3014, 3051, 3201, // AfdSendMessageWithAddress @@ -676,7 +677,7 @@ bool FKnownDoubleCountEvent(uint location) /*Exit*/ 3200, 3045, // AfdReceive - /*Enter*/ 4106, 4115, 4117, 4200, + /*Enter*/ 4001, 4106, 4115, 4117, 4200, /*Exit*/ 4109, 4110, 4111, 4116, 4118, 4122, // AfdReceiveMessageWithAddress @@ -1038,7 +1039,7 @@ ThreadX AFD.ReceiveMessage/FromWithAddress 1 ushort socket = cxn?.socket ?? 0; #if DEBUG pid = GetProcessId(hProc, evt); - AssertImportant(cxn?.pid == pid); + AssertImportant(FImplies(cxn != null, cxn?.pid == pid)); SocketAddress addrLocal = cxn?.addrLocal; #endif // DEBUG @@ -1252,6 +1253,10 @@ Thread0 AFD.BindWithAddress Enter AFD-Endpoint // CorrelateUDPRecvEvent has side effects to clear caches for found records. uint iTCB = allTables.tcpTable.CorrelateUDPRecvEvent(pid, tid, cb, cxn.socket, ipAddr); + // Try again, same process, different/async thread. + if (iTCB == 0 && tid != tidUnknown) + iTCB = allTables.tcpTable.CorrelateUDPRecvEvent(pid, tidUnknown, cb, cxn.socket, ipAddr); + if (cxn.iTCB == 0) { cxn.iTCB = iTCB; diff --git a/src/NetBlame/Tables/NetBlameTable.URL.cs b/src/NetBlame/Tables/NetBlameTable.URL.cs index 76634bc..0c67f96 100644 --- a/src/NetBlame/Tables/NetBlameTable.URL.cs +++ b/src/NetBlame/Tables/NetBlameTable.URL.cs @@ -1,4 +1,7 @@ -using System; +// Copyright(c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; using Microsoft.Performance.SDK.Processing; @@ -58,8 +61,8 @@ static class Columns DeclareColumn ( "Protocol", - "WinHTTP, WinINet, WinSock, LDAP", - width: 70, + "Chromium, WinHTTP, WinINet, WinSock, LDAP", + width: 76, visible: true ); @@ -67,7 +70,7 @@ static class Columns DeclareColumn ( "Method", - "HTTP/WinINet Method or WinSock IPProtocol", + "HTTP Method or WinSock IPProtocol", width: 62, visible: true ); @@ -85,7 +88,7 @@ static class Columns DeclareColumn ( "Status", - "Last non-zero status of the transaction", + "HTTP status of the transaction", width: 150, visible: false ); @@ -146,8 +149,8 @@ static class Columns DeclareColumn ( "Request ID", - "WebIO or WinINet Request ID, or WinSock Endpoint (reusable)", - width: 126, + "WebIO or WinINet or Chromium Request ID (reusable)", + width: 130, format: ColumnFormats.HexFormat, visible: false ); @@ -156,7 +159,7 @@ static class Columns DeclareColumn ( "Connection ID", - "WebIO or WinINet Connection ID", + "WebIO or WinINet Connection ID, or WinSock Endpoint (reusable)", width: 126, format: ColumnFormats.HexFormat, visible: false @@ -333,7 +336,7 @@ public static string GeoLoc(URL urlObj, GeoCache geoCache) { // Implement a simple caching optimization to reduce network requests. System.Net.IPAddress addrCur = urlObj.ipAddrPort?.Address; - if (addrCur == null) return String.Empty; + if (addrCur.Empty()) return String.Empty; return geoCache.Cache(addrCur); } diff --git a/src/NetBlame/Tables/NetBlameTable.WebIO.Request.cs b/src/NetBlame/Tables/NetBlameTable.WebIO.Request.cs index 5855e04..1d3638b 100644 --- a/src/NetBlame/Tables/NetBlameTable.WebIO.Request.cs +++ b/src/NetBlame/Tables/NetBlameTable.WebIO.Request.cs @@ -1,4 +1,7 @@ -#if AUX_TABLES +// Copyright(c) Microsoft Corporation. +// Licensed under the MIT License. + +#if AUX_TABLES using System; diff --git a/src/NetBlame/Tables/NetBlameTable.Winsock.cs b/src/NetBlame/Tables/NetBlameTable.Winsock.cs index e8918b8..e00f4a4 100644 --- a/src/NetBlame/Tables/NetBlameTable.Winsock.cs +++ b/src/NetBlame/Tables/NetBlameTable.Winsock.cs @@ -182,7 +182,7 @@ static class Generators public static string Server(Connection cxnObj, DNSClient.DNSTable dnsTable) => dnsTable.DNSEntryFromI(cxnObj.iDNS)?.strServer ?? strNA; - public static string Protocol(Connection cxnObj) => Prominent((Protocol)cxnObj.grbitType).ToString(); // pinned enum names + public static string Protocol(Connection cxnObj) => cxnObj.Protocol.ToString(); // pinned enum names public static string IPProto(Connection cxnObj) => cxnObj.ipProtocol.ToString(); // pinned enum names diff --git a/src/NetBlame/Tables/TableBase.cs b/src/NetBlame/Tables/TableBase.cs index e81d51a..d4bcc54 100644 --- a/src/NetBlame/Tables/TableBase.cs +++ b/src/NetBlame/Tables/TableBase.cs @@ -28,7 +28,7 @@ static class TID { public const IDVal Unknown = -1; } // TID.Unknown public abstract class NetBlameTableBase { - public static string StringFromInt(IDVal val) => (val >= 0) ? val.ToString("N0") : Util.strNA; + public static string StringFromInt(IDVal val) => (val >= 0) ? val.ToString("N0") : strNA; protected PendingSources Sources { get; } diff --git a/src/TraceNetwork.ps1 b/src/TraceNetwork.ps1 index 383f28a..4279e07 100644 --- a/src/TraceNetwork.ps1 +++ b/src/TraceNetwork.ps1 @@ -131,7 +131,7 @@ Param( @{ # The configuration files define the data tabs in the WPA viewer. # https://learn.microsoft.com/en-us/windows-hardware/test/wpt/view-profiles - ViewerConfig = ".\WPAP\BasicInfo.wpaProfile", ".\WPAP\EdgeRegions.wpaProfile", ".\WPAP\Network.wpaProfile" + ViewerConfig = ".\WPAP\BasicInfo.wpaProfile", ".\WPAP\ChromiumRegions.wpaProfile", ".\WPAP\Network.wpaProfile" # The trace file name is: .etl TraceName = $TraceParams.InstanceName diff --git a/src/WPRP/EdgeChrome.15002.wprp b/src/WPRP/EdgeChrome.15002.wprp index 31f9dce..a9c29be 100644 --- a/src/WPRP/EdgeChrome.15002.wprp +++ b/src/WPRP/EdgeChrome.15002.wprp @@ -51,7 +51,7 @@ CHROME: Chrome {D2D578D9-2936-45B6-A09F-30E32715F42D} - From: https://source.chromium.org/chromium/chromium/src/+/main:base/trace_event/trace_event_etw_export_win.cc + https://source.chromium.org/chromium/chromium/src/+/main:base/trace_event/etw_interceptor_win.cc KEYWORDS: benchmark // 0x00001 blink // 0x00002 @@ -109,11 +109,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - @@ -136,18 +178,14 @@ - - - - @@ -170,25 +208,33 @@ + + + + + + - + + + @@ -196,11 +242,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - @@ -249,7 +337,7 @@ - + @@ -270,7 +358,7 @@ - + @@ -296,6 +384,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/WPRP/EdgeChrome.wprp b/src/WPRP/EdgeChrome.wprp index b4e342c..8e76f71 100644 --- a/src/WPRP/EdgeChrome.wprp +++ b/src/WPRP/EdgeChrome.wprp @@ -49,7 +49,7 @@ CHROME: Chrome {D2D578D9-2936-45B6-A09F-30E32715F42D} - From: https://source.chromium.org/chromium/chromium/src/+/main:base/trace_event/trace_event_etw_export_win.cc + https://source.chromium.org/chromium/chromium/src/+/main:base/trace_event/etw_interceptor_win.cc KEYWORDS: benchmark // 0x00001 blink // 0x00002 @@ -107,7 +107,14 @@ + + + + + + + @@ -128,25 +135,33 @@ + + + + + + - + + + @@ -154,7 +169,14 @@ + + + + + + + @@ -171,7 +193,7 @@ - + @@ -192,7 +214,7 @@ - + @@ -218,6 +240,30 @@ + + + + + + + + + + + + + + + + + + + + + + + +