fix(webapp): stop api inheriting inbound sampled traceparents so trace sampling applies

The internal APM tracer's ParentBasedSampler left remoteParentSampled at its
AlwaysOn default, so any request arriving with a sampled traceparent (SDK task-run
traces propagated in from running tasks) was recorded in full, ignoring
INTERNAL_OTEL_TRACE_SAMPLING_RATE. On api that is ~99.6% of spans, so the divisor
was effectively inert.

Register a non-inheriting propagator: inject still delegates to W3C trace+baggage
so outbound propagation is unchanged, but extract drops the parent span, so every
inbound request roots its own trace and the ratio sampler applies uniformly. This
also stops api stitching onto (and inflating) the SDK's task-run traces, which is
where the untrimmable multi-thousand-span chains came from. Also set
remoteParentSampled to the ratio sampler as a fallback.

Author: ericallam

.server-changes/reduce-api-tracing-overhead.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Reduced internal overhead on the API under high load.

apps/webapp/app/v3/tracer.server.ts

@@ -14,7 +14,15 @@ import {
   trace,
   metrics,
   type Meter,
+  type TextMapPropagator,
+  type TextMapGetter,
+  type TextMapSetter,
 } from "@opentelemetry/api";
+import {
+  CompositePropagator,
+  W3CBaggagePropagator,
+  W3CTraceContextPropagator,
+} from "@opentelemetry/core";
 import sentryRemix from "@sentry/remix";
 import { logs, SeverityNumber } from "@opentelemetry/api-logs";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
@@ -125,6 +133,24 @@ class CustomWebappSampler implements Sampler {
   }
 }
 
+class NonInheritingTraceContextPropagator implements TextMapPropagator {
+  private readonly _delegate = new CompositePropagator({
+    propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
+  });
+
+  inject(context: Context, carrier: unknown, setter: TextMapSetter): void {
+    this._delegate.inject(context, carrier, setter);
+  }
+
+  extract(context: Context, carrier: unknown, getter: TextMapGetter): Context {
+    return trace.deleteSpan(this._delegate.extract(context, carrier, getter));
+  }
+
+  fields(): string[] {
+    return this._delegate.fields();
+  }
+}
+
 export const {
   tracer,
   logger: otelLogger,
@@ -281,11 +307,14 @@ function setupTelemetry() {
     }
   }
 
+  const ratioSampler = new TraceIdRatioBasedSampler(samplingRate);
+
   const provider = new NodeTracerProvider({
     forceFlushTimeoutMillis: 15_000,
     resource: getResource(),
     sampler: new ParentBasedSampler({
-      root: new CustomWebappSampler(new TraceIdRatioBasedSampler(samplingRate)),
+      root: new CustomWebappSampler(ratioSampler),
+      remoteParentSampled: ratioSampler,
     }),
     spanLimits: {
       attributeCountLimit: 1024,
@@ -324,7 +353,10 @@ function setupTelemetry() {
     );
   }
 
-  provider.register({ contextManager: createContextManager() });
+  provider.register({
+    contextManager: createContextManager(),
+    propagator: new NonInheritingTraceContextPropagator(),
+  });
 
   let instrumentations: Instrumentation[] = [
     new AwsSdkInstrumentation({