diff --git a/.env.sample b/.env.sample index 0130c64..e747ca8 100644 --- a/.env.sample +++ b/.env.sample @@ -10,9 +10,6 @@ NETRA_DISABLE_BATCH= # Debug Mode NETRA_DEBUG= -# Root Span Configuration -NETRA_ENABLE_ROOT_SPAN= - # Scrubbing Configuration NETRA_ENABLE_SCRUBBING= diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ed104e..afd3fdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.9.0] - Unreleased + +### Added + +- **Time to First Token (TTFT) & Relative TTFT**: All LLM generation spans now record `gen_ai.performance.time_to_first_token` (seconds from call start to first content chunk), `gen_ai.performance.relative_time_to_first_token` (seconds from trace root span start to first content chunk), and `gen_ai.performance.time_to_first_token.timestamp` (absolute ISO 8601 UTC time of first token). Supported across OpenAI, Anthropic, Groq, Mistral, Google GenAI, and Google Generative AI for both streaming and non-streaming calls. RTTFT is silently skipped when no root span exists. + +- **Opt-in prompt caching** — `Netra.prompts.getPrompt()` accepts `useCache` and `cacheTtl`. When `useCache` is true, responses are served from an in-memory TTL cache (default TTL: `PROMPT_CACHE_TTL_SECONDS` = 60). Caching is off by default. +- **Models API** — `Netra.models.getModelPricing()` fetches model pricing (optional `name` filter) with the same opt-in cache pattern (`useCache`, `cacheTtl`; default TTL: `MODEL_PRICING_CACHE_TTL_SECONDS` = 300). +- **Cache lifecycle** — `Netra.shutdown()` clears prompts and models in-memory caches. `clearCache()` is also available on each client. +- **Exported cache constants** — `PROMPT_CACHE_TTL_SECONDS` and `MODEL_PRICING_CACHE_TTL_SECONDS` are public exports. + +### Changed + +- **Prompt cache TTL** — Default TTL is the module constant `PROMPT_CACHE_TTL_SECONDS` (60). Override per call with `cacheTtl`. Removed unused `cacheTtlSeconds` init config and `NETRA_CACHE_TTL_SECONDS` env var. + ## [1.8.0] - 2026-08-03 ### Added @@ -124,7 +139,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **Context Propagation Helpers**: Exported `netraExpressMiddleware` and `runWithExtractedContext` for distributed tracing. These utilities extract incoming W3C Trace Context from HTTP headers and run code within that context, covering cases where auto-instrumentation is unavailable (ESM load-order issues, missing peer dependencies, or non-Express frameworks). +- **Context Propagation Helpers**: Exported `netraExpressMiddleware` and `runWithExtractedContext` for distributed tracing. These utilities extract incoming W3C Trace Context from HTTP headers and run code within that context, covering cases where auto-instrumentation is unavailable (ESM load-order issues, missing peer dependencies, or non-Express frameworks). ### Fixed diff --git a/README.md b/README.md index c99b256..8a2b4a3 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ - 🔧 **Multi-Provider Support**: Works with OpenAI, Google GenAI, Mistral, Anthropic, and more - 📈 **Session Management**: Track user sessions and custom attributes - 🌐 **Automatic Instrumentation**: Zero-code instrumentation for popular frameworks and libraries +- ⚡ **Opt-in Read Caching**: In-memory TTL caching for read-heavy SDK calls (`getPrompt`, `getModelPricing`) ## 📦 Installation @@ -184,6 +185,73 @@ async function generateContent(prompt: string) { } ``` +## 📝 Prompts API + +Fetch prompt versions from Prompt Studio via `Netra.prompts.getPrompt()`. Caching is **opt-in per call** — omit `useCache` (or set it to `false`) to always hit the API. + +Default TTL is **60 seconds** (`PROMPT_CACHE_TTL_SECONDS`). Override TTL for a single call with `cacheTtl`. + +```typescript +import { Netra } from "netra-sdk"; + +await Netra.init({ + appName: "my-ai-app", +}); + +// Always fetches from the API (default) +const prompt = await Netra.prompts.getPrompt({ name: "my-prompt" }); + +// Cached for 60s (default TTL) +const cached = await Netra.prompts.getPrompt({ + name: "my-prompt", + useCache: true, +}); + +// Cached for 30s for this call only +const shortLived = await Netra.prompts.getPrompt({ + name: "my-prompt", + label: "production", // default label when omitted + useCache: true, + cacheTtl: 30, +}); +``` + +> **Note**: Cached prompts may be stale for up to the TTL after dashboard edits. Use `useCache: false` when you need the latest version immediately. `Netra.shutdown()` clears in-memory caches. + +## 💰 Models API + +Fetch model pricing via `Netra.models.getModelPricing()`. Caching is **opt-in per call** — omit `useCache` (or set it to `false`) to always hit the API. + +Default TTL is **300 seconds** (`MODEL_PRICING_CACHE_TTL_SECONDS`). Override TTL for a single call with `cacheTtl`. + +```typescript +import { Netra } from "netra-sdk"; + +await Netra.init({ + appName: "my-ai-app", +}); + +// Always fetches from the API (default) +const pricing = await Netra.models.getModelPricing(); + +// Optional name filter +const gptPricing = await Netra.models.getModelPricing({ name: "gpt-4o" }); + +// Cached for 300s (default TTL) +const cached = await Netra.models.getModelPricing({ + useCache: true, +}); + +// Cached for 60s for this call only +const shortLived = await Netra.models.getModelPricing({ + name: "gpt-4o", + useCache: true, + cacheTtl: 60, +}); +``` + +> **Note**: Cached pricing may be stale for up to the TTL after dashboard edits. Use `useCache: false` when you need the latest values immediately. `Netra.shutdown()` clears in-memory caches. + ## 🔧 Environment Variables You can configure the SDK using environment variables: diff --git a/package-lock.json b/package-lock.json index 9e92f2a..804b9ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "netra-sdk", - "version": "1.8.0", + "version": "1.9.0-dev.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "netra-sdk", - "version": "1.8.0", + "version": "1.9.0-dev.0", "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.9.0", @@ -28,7 +28,8 @@ "@types/shimmer": "^1.2.0", "ts-node": "^10.9.2", "tsup": "^8.5.1", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^3.2.4" }, "engines": { "node": ">=18.0.0" @@ -3536,6 +3537,24 @@ "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", "license": "MIT" }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -3600,6 +3619,121 @@ "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", "license": "MIT" }, + "node_modules/@vitest/expect": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.7.tgz", + "integrity": "sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.7.tgz", + "integrity": "sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.7", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.7.tgz", + "integrity": "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.7.tgz", + "integrity": "sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.7", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.7.tgz", + "integrity": "sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.7.tgz", + "integrity": "sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.7.tgz", + "integrity": "sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/a-sync-waterfall": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz", @@ -3702,6 +3836,16 @@ "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", "license": "MIT" }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -3842,6 +3986,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -3885,6 +4046,16 @@ "node": ">=8" } }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, "node_modules/cjs-module-lexer": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", @@ -4012,6 +4183,16 @@ "node": ">=0.10.0" } }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -4099,6 +4280,13 @@ "node": ">= 0.4" } }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, "node_modules/es-object-atoms": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", @@ -4177,6 +4365,16 @@ "node": ">=6" } }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -4213,6 +4411,16 @@ "node": ">=18.0.0" } }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -4710,6 +4918,13 @@ "base64-js": "^1.5.1" } }, + "node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -4832,6 +5047,13 @@ "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", "license": "Apache-2.0" }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -4929,6 +5151,25 @@ "thenify-all": "^1.0.0" } }, + "node_modules/nanoid": { + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -5099,6 +5340,16 @@ "dev": true, "license": "MIT" }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -5107,9 +5358,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", "engines": { @@ -5141,6 +5392,35 @@ "pathe": "^2.0.1" } }, + "node_modules/postcss": { + "version": "8.5.16", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", + "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, "node_modules/postcss-load-config": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", @@ -5399,6 +5679,13 @@ "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==", "license": "BSD-2-Clause" }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/simple-wcswidth": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", @@ -5415,6 +5702,30 @@ "node": ">= 12" } }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, "node_modules/stream-events": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", @@ -5465,6 +5776,19 @@ "node": ">=8" } }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, "node_modules/stubs": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", @@ -5605,6 +5929,13 @@ "node": ">=0.8" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyexec": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", @@ -5613,14 +5944,14 @@ "license": "MIT" }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -5629,6 +5960,36 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", + "integrity": "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -5838,6 +6199,177 @@ "dev": true, "license": "MIT" }, + "node_modules/vite": { + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", + "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0 || ^0.28.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vitest": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.7.tgz", + "integrity": "sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.7", + "@vitest/mocker": "3.2.7", + "@vitest/pretty-format": "^3.2.7", + "@vitest/runner": "3.2.7", + "@vitest/snapshot": "3.2.7", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.7", + "@vitest/ui": "3.2.7", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -5861,6 +6393,23 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index f0ed327..454e74b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "netra-sdk", - "version": "1.8.0", + "version": "1.9.0-dev.0", "description": "A comprehensive TypeScript/JavaScript SDK for AI application observability built on top of OpenTelemetry and Traceloop", "type": "module", "main": "./dist/index.cjs", @@ -20,7 +20,7 @@ "build": "tsup", "start:dev": "tsup --watch", "prepack": "npm run build", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "vitest run" }, "keywords": [ "netra", @@ -79,40 +79,46 @@ "openai": "^4.0.0 || ^5.0.0 || ^6.0.0" }, "peerDependenciesMeta": { - "@opentelemetry/instrumentation": { + "@anthropic-ai/sdk": { "optional": true }, - "@opentelemetry/instrumentation-http": { + "@google/genai": { "optional": true }, - "@opentelemetry/instrumentation-express": { + "@google/generative-ai": { "optional": true }, - "@opentelemetry/instrumentation-undici": { + "@langchain/langgraph": { "optional": true }, - "@prisma/instrumentation": { + "@langchain/ollama": { "optional": true }, - "openai": { + "@mistralai/mistralai": { "optional": true }, - "groq-sdk": { + "@openai/agents": { "optional": true }, - "@mistralai/mistralai": { + "@opentelemetry/instrumentation": { "optional": true }, - "@google/generative-ai": { + "@opentelemetry/instrumentation-express": { "optional": true }, - "@google/genai": { + "@opentelemetry/instrumentation-http": { "optional": true }, - "@anthropic-ai/sdk": { + "@opentelemetry/instrumentation-undici": { "optional": true }, - "@openai/agents": { + "@prisma/instrumentation": { + "optional": true + }, + "groq-sdk": { + "optional": true + }, + "openai": { "optional": true } }, @@ -121,6 +127,7 @@ "@types/shimmer": "^1.2.0", "ts-node": "^10.9.2", "tsup": "^8.5.1", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^3.2.4" } } diff --git a/src/api/dashboard/api.ts b/src/api/dashboard/api.ts index 1c2fea8..b684be9 100644 --- a/src/api/dashboard/api.ts +++ b/src/api/dashboard/api.ts @@ -14,6 +14,7 @@ import { QueryDataParams, QueryResponse, Scope, + SessionDetailsResponse, SessionFilter, SessionFilterConfig, SessionStatsData, @@ -196,6 +197,23 @@ async getSessionStats( return result.data ?? {}; } + async getSessionDetails(sessionId: string): Promise { + if (!sessionId) { + Logger.error( + "netra.dashboard: session_id is required to fetch session details", + ); + return null; + } + + const result = await this.client.getSessionDetails(sessionId); + + if (!result) { + return null; + } + + return result.data ?? result; + } + private isValidScope(scope: any): scope is Scope { return Object.values(Scope).includes(scope); } diff --git a/src/api/dashboard/client.ts b/src/api/dashboard/client.ts index 026c33f..fc4bb7e 100644 --- a/src/api/dashboard/client.ts +++ b/src/api/dashboard/client.ts @@ -244,4 +244,46 @@ export class DashboardHttpClient extends NetraHttpClient { return null; } } + + /** + * Get detailed information for a specific session including all its traces. + * + * Args: + * sessionId: The session ID to retrieve details for. + * + * Returns: + * The session details response data or null on error. + */ + async getSessionDetails(sessionId: string): Promise { + if (!this.isInitialized()) { + Logger.error( + "netra.dashboard: Dashboard client is not initialized; cannot get session details", + ); + return null; + } + + try { + const url = `/public/dashboard/session/${encodeURIComponent(sessionId)}`; + + const response = await this.get(url); + + if (!response.ok) { + const errorMessage = response.data?.error?.message ?? "Unknown error"; + Logger.error( + `netra.dashboard: Failed to fetch session details: ${errorMessage}`, + ); + return null; + } + + return response.data; + } catch (err: any) { + const message = err?.response?.data?.error?.message ?? ""; + + Logger.error( + "netra.dashboard: Failed to fetch session details:", + message, + ); + return null; + } + } } diff --git a/src/api/dashboard/index.ts b/src/api/dashboard/index.ts index dfde5e8..c98f100 100644 --- a/src/api/dashboard/index.ts +++ b/src/api/dashboard/index.ts @@ -35,6 +35,9 @@ export type { NumberResponse, QueryDataParams, QueryResponse, + SessionDetailsResponse, + SessionDetailsToolCall, + SessionDetailsTrace, TimeRange, TimeSeriesDataPoint, TimeSeriesResponse, diff --git a/src/api/dashboard/models.ts b/src/api/dashboard/models.ts index 5a8aa0d..7674715 100644 --- a/src/api/dashboard/models.ts +++ b/src/api/dashboard/models.ts @@ -228,3 +228,40 @@ export interface SessionStatsData { session_duration: string; cursor: string; } + +// Session Detail +export interface SessionDetailsToolCall { + toolName: string; + toolCallCount: number; +} + +export interface SessionDetailsTrace { + traceId: string; + traceName: string; + startTime: string; + endTime: string; + latencyMs: number; + input: string | null; + output: string | null; + tokens: { + promptTokens: number; + completionTokens: number; + cachedTokens: number; + cacheCreationTokens: number; + totalTokens: number; + }; + cost: { + promptTokensCost: number; + completionTokensCost: number; + cachedTokensCost: number; + cacheCreationTokensCost: number; + totalCost: number; + }; + models: string[]; + toolCalls: SessionDetailsToolCall[]; +} + +export interface SessionDetailsResponse { + sessionId: string; + traces: SessionDetailsTrace[]; +} diff --git a/src/api/evaluation/api.ts b/src/api/evaluation/api.ts index dae512a..3edda31 100644 --- a/src/api/evaluation/api.ts +++ b/src/api/evaluation/api.ts @@ -368,7 +368,11 @@ export class Evaluation { ): Promise<{ status: string }> { const spanName = `TestRun.${runName}`; - const span = new SpanWrapper(spanName, {}, "netra.evaluation"); + const span = new SpanWrapper( + spanName, + { [Config.TRACE_ORIGIN_KEY]: Config.TRACE_ORIGIN_EVALUATION }, + "netra.evaluation", + ); span.start(); try { diff --git a/src/api/index.ts b/src/api/index.ts index 6dc1451..520f136 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -59,6 +59,9 @@ export type { NumberResponse, QueryDataParams, QueryResponse, + SessionDetailsResponse, + SessionDetailsToolCall, + SessionDetailsTrace, TimeRange, TimeSeriesDataPoint, TimeSeriesResponse, @@ -66,5 +69,13 @@ export type { } from "./dashboard"; // Prompts API -export { Prompts } from "./prompts"; +export { Prompts, PROMPT_CACHE_TTL_SECONDS } from "./prompts"; export type { GetPromptParams, PromptResponse } from "./prompts"; + +// Models API +export { Models, MODEL_PRICING_CACHE_TTL_SECONDS } from "./models"; +export type { + GetModelPricingParams, + ModelPrice, + ModelPricing, +} from "./models"; diff --git a/src/api/models/api.test.ts b/src/api/models/api.test.ts new file mode 100644 index 0000000..3acf8ff --- /dev/null +++ b/src/api/models/api.test.ts @@ -0,0 +1,136 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Config } from "../../config"; +import { Models } from "./api"; +import { ModelsHttpClient } from "./client"; +import { ModelPricing } from "./models"; + +const samplePricing: ModelPricing[] = [ + { + name: "gpt-4", + projectId: null, + matchPattern: "gpt-4*", + prices: [ + { + usageType: "input", + minUnits: 0, + maxUnits: 1000, + price: 0.03, + unitValue: 1000, + }, + ], + }, +]; + +describe("Models.getModelPricing caching", () => { + let models: Models; + let getModelPricing: ReturnType; + + beforeEach(() => { + const config = new Config(); + models = new Models(config); + getModelPricing = vi.fn(); + (models as unknown as { client: ModelsHttpClient }).client = { + getModelPricing, + } as unknown as ModelsHttpClient; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("calls HTTP on every request when useCache is omitted", async () => { + getModelPricing.mockResolvedValue(samplePricing); + + await models.getModelPricing(); + await models.getModelPricing(); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); + + it("serves from cache on second call with same name when useCache is true", async () => { + getModelPricing.mockResolvedValue(samplePricing); + + const first = await models.getModelPricing({ + name: "gpt-4", + useCache: true, + }); + const second = await models.getModelPricing({ + name: "gpt-4", + useCache: true, + }); + + expect(getModelPricing).toHaveBeenCalledTimes(1); + expect(getModelPricing).toHaveBeenCalledWith("gpt-4"); + expect(first).toEqual(samplePricing); + expect(second).toEqual(samplePricing); + }); + + it("keeps separate cache entries for different name and all", async () => { + const allPricing: ModelPricing[] = []; + getModelPricing + .mockResolvedValueOnce(samplePricing) + .mockResolvedValueOnce(allPricing); + + const named = await models.getModelPricing({ + name: "gpt-4", + useCache: true, + }); + const all = await models.getModelPricing({ useCache: true }); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + expect(getModelPricing).toHaveBeenNthCalledWith(1, "gpt-4"); + expect(getModelPricing).toHaveBeenNthCalledWith(2, undefined); + expect(named).toEqual(samplePricing); + expect(all).toEqual(allPricing); + }); + + it("does not cache null API responses", async () => { + getModelPricing.mockResolvedValue(null); + + await models.getModelPricing({ useCache: true }); + await models.getModelPricing({ useCache: true }); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); + + it("caches empty arrays as successful responses", async () => { + getModelPricing.mockResolvedValue([]); + + await models.getModelPricing({ useCache: true }); + await models.getModelPricing({ useCache: true }); + + expect(getModelPricing).toHaveBeenCalledTimes(1); + }); + + it("ignores cache when useCache is false even if cacheTtl is set", async () => { + getModelPricing.mockResolvedValue(samplePricing); + + await models.getModelPricing({ useCache: false, cacheTtl: 30 }); + await models.getModelPricing({ useCache: false, cacheTtl: 30 }); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); + + it("expires per-call cacheTtl before the models default TTL", async () => { + vi.useFakeTimers(); + getModelPricing.mockResolvedValue(samplePricing); + + await models.getModelPricing({ useCache: true, cacheTtl: 1 }); + expect(getModelPricing).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(1001); + + await models.getModelPricing({ useCache: true, cacheTtl: 1 }); + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); + + it("hits HTTP again after clearCache when useCache is true", async () => { + getModelPricing.mockResolvedValue(samplePricing); + + await models.getModelPricing({ useCache: true }); + models.clearCache(); + await models.getModelPricing({ useCache: true }); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/api/models/api.ts b/src/api/models/api.ts new file mode 100644 index 0000000..f7ee692 --- /dev/null +++ b/src/api/models/api.ts @@ -0,0 +1,54 @@ +import { TTLCache } from "../../cache"; +import { Config } from "../../config"; +import { ModelsHttpClient } from "./client"; +import { + GetModelPricingParams, + MODEL_PRICING_CACHE_TTL_SECONDS, + ModelPricing, +} from "./models"; + +export class Models { + private config: Config; + private client: ModelsHttpClient; + private cache: TTLCache; + + constructor(config: Config) { + this.config = config; + this.client = new ModelsHttpClient(config); + this.cache = new TTLCache(MODEL_PRICING_CACHE_TTL_SECONDS); + } + + /** Clear all cached model pricing entries. */ + clearCache(): void { + this.cache.clear(); + } + + /** + * Fetch model pricing from the backend. + * + * @param params.name - Optional model name filter + * @param params.useCache - When true, read/write the in-memory cache (default: false) + * @param params.cacheTtl - Per-call cache TTL in seconds (default: MODEL_PRICING_CACHE_TTL_SECONDS) + */ + async getModelPricing( + params: GetModelPricingParams = {}, + ): Promise { + const useCache = params.useCache === true; + const cacheKey = `model:pricing:${params.name ?? "all"}`; + + if (useCache) { + const cached = this.cache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + } + + const data = await this.client.getModelPricing(params.name); + + if (data !== null && useCache) { + this.cache.set(cacheKey, data, params.cacheTtl); + } + + return data; + } +} diff --git a/src/api/models/client.test.ts b/src/api/models/client.test.ts new file mode 100644 index 0000000..3fe917d --- /dev/null +++ b/src/api/models/client.test.ts @@ -0,0 +1,169 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Config } from "../../config"; +import { Logger } from "../../logger"; +import { ModelsHttpClient } from "./client"; +import { ModelPricing } from "./models"; + +const samplePricing: ModelPricing[] = [ + { + name: "gpt-4", + projectId: null, + matchPattern: "gpt-4*", + prices: [ + { + usageType: "input", + minUnits: 0, + maxUnits: 1000, + price: 0.03, + unitValue: 1000, + }, + ], + }, +]; + +describe("ModelsHttpClient.getModelPricing", () => { + let client: ModelsHttpClient; + let get: ReturnType; + let isInitialized: ReturnType; + let logError: ReturnType; + + beforeEach(() => { + client = new ModelsHttpClient(new Config()); + get = vi.spyOn(client, "get"); + isInitialized = vi.spyOn(client, "isInitialized").mockReturnValue(true); + logError = vi.spyOn(Logger, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe("when the request succeeds", () => { + it("returns the pricing array from an enveloped API response", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: samplePricing }, + }); + + const result = await client.getModelPricing(); + + expect(result).toEqual(samplePricing); + expect(get).toHaveBeenCalledWith("/sdk/models", undefined); + }); + + it("calls GET /sdk/models with the name query param when name is provided", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: samplePricing }, + }); + + await client.getModelPricing("gpt-4"); + + expect(get).toHaveBeenCalledWith("/sdk/models", { name: "gpt-4" }); + }); + + it("calls GET /sdk/models with no query params when name is omitted", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: samplePricing }, + }); + + await client.getModelPricing(); + + expect(get).toHaveBeenCalledWith("/sdk/models", undefined); + }); + + it("returns an empty array when the API returns an empty list", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: [] }, + }); + + const result = await client.getModelPricing(); + + expect(result).toEqual([]); + expect(logError).not.toHaveBeenCalled(); + }); + }); + + describe("when the client is not initialized", () => { + it("returns null without calling GET", async () => { + isInitialized.mockReturnValue(false); + + const result = await client.getModelPricing("gpt-4"); + + expect(result).toBeNull(); + expect(get).not.toHaveBeenCalled(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Models client is not initialized; cannot fetch model pricing", + ); + }); + }); + + describe("when the HTTP response fails", () => { + it("returns null and logs the API error message when response is not ok", async () => { + get.mockResolvedValue({ + ok: false, + status: 500, + data: { error: { message: "backend unavailable" } }, + }); + + const result = await client.getModelPricing(); + + expect(result).toBeNull(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Failed to fetch model pricing: backend unavailable", + ); + }); + + it("returns null and logs a generic message when the error payload has no message", async () => { + get.mockResolvedValue({ + ok: false, + status: 502, + data: {}, + }); + + const result = await client.getModelPricing(); + + expect(result).toBeNull(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Failed to fetch model pricing: Unknown error", + ); + }); + }); + + describe("when the response payload is unexpected", () => { + it("returns null and logs when the unwrapped payload is not an array", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: { name: "not-an-array" } }, + }); + + const result = await client.getModelPricing(); + + expect(result).toBeNull(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Unexpected response format; expected an array", + ); + }); + }); + + describe("when GET throws", () => { + it("returns null and logs the thrown error message", async () => { + get.mockRejectedValue(new Error("network down")); + + const result = await client.getModelPricing(); + + expect(result).toBeNull(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Failed to fetch model pricing:", + "network down", + ); + }); + }); +}); diff --git a/src/api/models/client.ts b/src/api/models/client.ts new file mode 100644 index 0000000..9dfa4d3 --- /dev/null +++ b/src/api/models/client.ts @@ -0,0 +1,49 @@ +/** + * Internal HTTP client for Models APIs + */ + +import { Config } from "../../config"; +import { Logger } from "../../logger"; +import { NetraHttpClient } from "../http-client"; +import { ModelPricing } from "./models"; + +export class ModelsHttpClient extends NetraHttpClient { + constructor(config: Config) { + super(config, "NETRA_MODELS_TIMEOUT", 10.0); + } + + async getModelPricing(name?: string): Promise { + if (!this.isInitialized()) { + Logger.error( + "netra.models: Models client is not initialized; cannot fetch model pricing", + ); + return null; + } + + try { + const params = name ? { name } : undefined; + const response = await this.get("/sdk/models", params); + + if (!response.ok) { + const errorMessage = response.data?.error?.message ?? "Unknown error"; + Logger.error( + `netra.models: Failed to fetch model pricing: ${errorMessage}`, + ); + return null; + } + + const items = this.extractData(response, null); + if (items !== null && !Array.isArray(items)) { + Logger.error( + "netra.models: Unexpected response format; expected an array", + ); + return null; + } + return items; + } catch (err: any) { + const message = err?.response?.data?.error?.message ?? err?.message ?? ""; + Logger.error("netra.models: Failed to fetch model pricing:", message); + return null; + } + } +} diff --git a/src/api/models/index.ts b/src/api/models/index.ts new file mode 100644 index 0000000..138c4b9 --- /dev/null +++ b/src/api/models/index.ts @@ -0,0 +1,13 @@ +/** + * Models API exports + */ + +export { Models } from "./api"; + +export type { + GetModelPricingParams, + ModelPrice, + ModelPricing, +} from "./models"; + +export { MODEL_PRICING_CACHE_TTL_SECONDS } from "./models"; diff --git a/src/api/models/models.ts b/src/api/models/models.ts new file mode 100644 index 0000000..4ccf3a8 --- /dev/null +++ b/src/api/models/models.ts @@ -0,0 +1,32 @@ +/** + * Models API Models + */ + +/** Default TTL (seconds) for model pricing cache when useCache is true and cacheTtl is omitted. */ +export const MODEL_PRICING_CACHE_TTL_SECONDS = 300; + +export interface ModelPrice { + usageType: string; + minUnits: number; + maxUnits: number; + price: number; + unitValue: number; +} + +export interface ModelPricing { + name: string; + projectId: string | null; + matchPattern: string; + prices: ModelPrice[]; +} + +export interface GetModelPricingParams { + name?: string; + /** When true, read/write in-memory cache (default: false). */ + useCache?: boolean; + /** + * Per-call TTL in seconds. + * When omitted with useCache: true, uses MODEL_PRICING_CACHE_TTL_SECONDS (300). + */ + cacheTtl?: number; +} diff --git a/src/api/prompts/api.test.ts b/src/api/prompts/api.test.ts new file mode 100644 index 0000000..75294de --- /dev/null +++ b/src/api/prompts/api.test.ts @@ -0,0 +1,137 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Config } from "../../config"; +import { Prompts } from "./api"; +import { PromptsHttpClient } from "./client"; +import { PROMPT_CACHE_TTL_SECONDS } from "./models"; + +describe("Prompts.getPrompt caching", () => { + let prompts: Prompts; + let getPromptVersion: ReturnType; + + beforeEach(() => { + const config = new Config(); + prompts = new Prompts(config); + getPromptVersion = vi.fn(); + (prompts as unknown as { client: PromptsHttpClient }).client = { + getPromptVersion, + } as unknown as PromptsHttpClient; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("calls HTTP on every request when useCache is omitted", async () => { + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ name: "my-prompt" }); + await prompts.getPrompt({ name: "my-prompt" }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("serves from cache on second call with same name and label when useCache is true", async () => { + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + const first = await prompts.getPrompt({ name: "my-prompt", useCache: true }); + const second = await prompts.getPrompt({ name: "my-prompt", useCache: true }); + + expect(getPromptVersion).toHaveBeenCalledTimes(1); + expect(first).toEqual({ template: "v1" }); + expect(second).toEqual({ template: "v1" }); + }); + + it("keeps separate cache entries per label when useCache is true", async () => { + getPromptVersion + .mockResolvedValueOnce({ data: { template: "prod" } }) + .mockResolvedValueOnce({ data: { template: "staging" } }); + + const prod = await prompts.getPrompt({ + name: "my-prompt", + label: "production", + useCache: true, + }); + const staging = await prompts.getPrompt({ + name: "my-prompt", + label: "staging", + useCache: true, + }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + expect(prod).toEqual({ template: "prod" }); + expect(staging).toEqual({ template: "staging" }); + }); + + it("does not cache null API responses", async () => { + getPromptVersion.mockResolvedValue(null); + + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("ignores cache when useCache is false even if cacheTtl is set", async () => { + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ + name: "my-prompt", + useCache: false, + cacheTtl: 30, + }); + await prompts.getPrompt({ + name: "my-prompt", + useCache: false, + cacheTtl: 30, + }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("expires per-call cacheTtl before the module default TTL", async () => { + vi.useFakeTimers(); + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ + name: "my-prompt", + useCache: true, + cacheTtl: 1, + }); + expect(getPromptVersion).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(1001); + + await prompts.getPrompt({ + name: "my-prompt", + useCache: true, + cacheTtl: 1, + }); + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("expires cached prompts after PROMPT_CACHE_TTL_SECONDS when cacheTtl is omitted", async () => { + vi.useFakeTimers(); + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(PROMPT_CACHE_TTL_SECONDS * 1000 - 1); + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(2); + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("hits HTTP again after clearCache when useCache is true", async () => { + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + prompts.clearCache(); + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/api/prompts/api.ts b/src/api/prompts/api.ts index 7709cf2..3d8ca6d 100644 --- a/src/api/prompts/api.ts +++ b/src/api/prompts/api.ts @@ -1,22 +1,36 @@ +import { TTLCache } from "../../cache"; import { Config } from "../../config"; import { Logger } from "../../logger"; import { PromptsHttpClient } from "./client"; -import { GetPromptParams, PromptResponse } from "./models"; +import { + GetPromptParams, + PROMPT_CACHE_TTL_SECONDS, + PromptResponse, +} from "./models"; export class Prompts { private config: Config; private client: PromptsHttpClient; + private cache: TTLCache; constructor(config: Config) { this.config = config; this.client = new PromptsHttpClient(config); + this.cache = new TTLCache(PROMPT_CACHE_TTL_SECONDS); + } + + /** Clear all cached prompt entries. */ + clearCache(): void { + this.cache.clear(); } /** * Fetch prompt version by name and label. * - * @param params.name - Name of the prompt (required) - * @param params.label - Label of the prompt version (default: "production") + * @param params.name - Name of the prompt (required) + * @param params.label - Label of the prompt version (default: "production") + * @param params.useCache - When true, read/write the in-memory cache (default: false) + * @param params.cacheTtl - Per-call cache TTL in seconds (default: PROMPT_CACHE_TTL_SECONDS) */ async getPrompt(params: GetPromptParams): Promise { if (!params || typeof params.name !== "string" || !params.name) { @@ -25,12 +39,28 @@ export class Prompts { } const label = params.label ?? "production"; + const useCache = params.useCache === true; + const cacheKey = `prompt:${params.name}:${label}`; + + if (useCache) { + const cached = this.cache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + } + const result = await this.client.getPromptVersion(params.name, label); if (!result) { return null; } - return result.data ?? null; + const data = result.data ?? null; + + if (data !== null && useCache) { + this.cache.set(cacheKey, data, params.cacheTtl); + } + + return data; } } diff --git a/src/api/prompts/index.ts b/src/api/prompts/index.ts index bedb08f..0847ecf 100644 --- a/src/api/prompts/index.ts +++ b/src/api/prompts/index.ts @@ -3,5 +3,5 @@ */ export { Prompts } from "./api"; - +export { PROMPT_CACHE_TTL_SECONDS } from "./models"; export type { GetPromptParams, PromptResponse } from "./models"; diff --git a/src/api/prompts/models.ts b/src/api/prompts/models.ts index 1ec6cb3..50ed7cc 100644 --- a/src/api/prompts/models.ts +++ b/src/api/prompts/models.ts @@ -2,9 +2,19 @@ * Prompts API Models */ +/** Default TTL (seconds) for prompt cache when useCache is true and cacheTtl is omitted. */ +export const PROMPT_CACHE_TTL_SECONDS = 60; + export interface GetPromptParams { name: string; label?: string; + /** When true, serve from in-memory cache when available (default: false). */ + useCache?: boolean; + /** + * Per-call TTL in seconds. + * When omitted with useCache: true, uses PROMPT_CACHE_TTL_SECONDS (60). + */ + cacheTtl?: number; } /** diff --git a/src/cache.test.ts b/src/cache.test.ts new file mode 100644 index 0000000..af71f5e --- /dev/null +++ b/src/cache.test.ts @@ -0,0 +1,55 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { TTLCache } from "./cache"; + +describe("TTLCache", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("get() returns undefined for missing key", () => { + const cache = new TTLCache(); + expect(cache.get("missing")).toBeUndefined(); + }); + + it("set() + get() returns stored value before TTL expires", () => { + const cache = new TTLCache(60); + cache.set("key", "value"); + expect(cache.get("key")).toBe("value"); + }); + + it("get() returns undefined after TTL expires", () => { + vi.useFakeTimers(); + const cache = new TTLCache(1); + cache.set("key", "value"); + vi.advanceTimersByTime(1001); + expect(cache.get("key")).toBeUndefined(); + }); + + it("per-entry ttl override expires independently of default", () => { + vi.useFakeTimers(); + const cache = new TTLCache(60); + cache.set("short", "a", 1); + cache.set("long", "b", 60); + vi.advanceTimersByTime(1001); + expect(cache.get("short")).toBeUndefined(); + expect(cache.get("long")).toBe("b"); + }); + + it("clear() removes all entries", () => { + const cache = new TTLCache(); + cache.set("a", "1"); + cache.set("b", "2"); + cache.clear(); + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("b")).toBeUndefined(); + }); + + it("invalidate(key) removes single entry", () => { + const cache = new TTLCache(); + cache.set("a", "1"); + cache.set("b", "2"); + cache.invalidate("a"); + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("b")).toBe("2"); + }); +}); diff --git a/src/cache.ts b/src/cache.ts new file mode 100644 index 0000000..0bbf765 --- /dev/null +++ b/src/cache.ts @@ -0,0 +1,35 @@ +/** + * In-memory TTL cache for SDK read API responses. + */ + +export class TTLCache { + private store = new Map(); + private defaultTtl: number; + + constructor(defaultTtl = 60) { + this.defaultTtl = defaultTtl; + } + + get(key: string): T | undefined { + const entry = this.store.get(key); + if (!entry) return undefined; + if (performance.now() > entry.expiresAt) { + this.store.delete(key); + return undefined; + } + return entry.value; + } + + set(key: string, value: T, ttl?: number): void { + const ttlMs = (ttl ?? this.defaultTtl) * 1000; + this.store.set(key, { value, expiresAt: performance.now() + ttlMs }); + } + + invalidate(key: string): void { + this.store.delete(key); + } + + clear(): void { + this.store.clear(); + } +} diff --git a/src/config.ts b/src/config.ts index f297a7e..c4ca815 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,7 +11,6 @@ export interface NetraConfig { disableBatch?: boolean; traceContent?: boolean; debugMode?: boolean; - enableRootSpan?: boolean; resourceAttributes?: Record; environment?: string; enableScrubbing?: boolean; @@ -33,10 +32,8 @@ export interface NetraConfig { * link is remote (cross-process) is kept as a root so an upstream distributed * trace is not severed. * - * Note: when `enableRootSpan` is true, Netra attaches its own root span and - * every auto-instrumentation span becomes its child, so no reparenting - * occurs. Pass a set containing `NetraInstruments.ALL` to allow all - * instrumentations to produce root spans (legacy behaviour). + * Pass a set containing `NetraInstruments.ALL` to allow all instrumentations + * to produce root spans (legacy behaviour). */ rootInstruments?: Set; } @@ -127,6 +124,11 @@ export class Config { static readonly LIBRARY_VERSION = SDK_VERSION; static readonly TRIAL_BLOCK_DURATION_SECONDS = 900; // 15 minutes + // Root-span attribute marking traces produced by evaluation/simulation runs + // so the FE/BE can distinguish them from normal workflow invocations. + static readonly TRACE_ORIGIN_KEY = "netra.trace.origin"; + static readonly TRACE_ORIGIN_EVALUATION = "evaluation"; + private static _spanAttributeMaxSize: number | undefined; /** @@ -150,7 +152,6 @@ export class Config { disableBatch: boolean; traceContent: boolean; debugMode: boolean; - enableRootSpan: boolean; enableScrubbing: boolean; environment: string; resourceAttributes: Record; @@ -176,11 +177,6 @@ export class Config { "NETRA_DEBUG", false, ); - this.enableRootSpan = this._getBoolConfig( - config.enableRootSpan, - "NETRA_ENABLE_ROOT_SPAN", - false, - ); this.enableScrubbing = this._getBoolConfig( config.enableScrubbing, "NETRA_ENABLE_SCRUBBING", diff --git a/src/index.shutdown-cache.test.ts b/src/index.shutdown-cache.test.ts new file mode 100644 index 0000000..ed9076e --- /dev/null +++ b/src/index.shutdown-cache.test.ts @@ -0,0 +1,75 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ModelsHttpClient } from "./api/models/client"; +import { PromptsHttpClient } from "./api/prompts/client"; + +vi.mock("./instrumentation", () => ({ + initInstrumentations: vi.fn(() => ({})), + instrumentationsReady: Promise.resolve(), + uninstrumentAll: vi.fn(() => Promise.resolve()), +})); + +import { Netra } from "./index"; + +describe("Netra.shutdown cache clearing", () => { + afterEach(async () => { + if (Netra.isInitialized()) { + await Netra.shutdown(); + } + }); + + it("clears prompts cache on shutdown so the next cached getPrompt hits HTTP", async () => { + await Netra.init({}); + + const getPromptVersion = vi + .fn() + .mockResolvedValue({ data: { template: "v1" } }); + (Netra.prompts as unknown as { client: PromptsHttpClient }).client = { + getPromptVersion, + } as unknown as PromptsHttpClient; + + await Netra.prompts.getPrompt({ name: "my-prompt", useCache: true }); + await Netra.prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(1); + + await Netra.shutdown(); + + await Netra.init({}); + (Netra.prompts as unknown as { client: PromptsHttpClient }).client = { + getPromptVersion, + } as unknown as PromptsHttpClient; + + await Netra.prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("clears models cache on shutdown so the next cached getModelPricing hits HTTP", async () => { + await Netra.init({}); + + const pricing = [ + { + name: "gpt-4", + projectId: null, + matchPattern: "gpt-4*", + prices: [], + }, + ]; + const getModelPricing = vi.fn().mockResolvedValue(pricing); + (Netra.models as unknown as { client: ModelsHttpClient }).client = { + getModelPricing, + } as unknown as ModelsHttpClient; + + await Netra.models.getModelPricing({ useCache: true }); + await Netra.models.getModelPricing({ useCache: true }); + expect(getModelPricing).toHaveBeenCalledTimes(1); + + await Netra.shutdown(); + + await Netra.init({}); + (Netra.models as unknown as { client: ModelsHttpClient }).client = { + getModelPricing, + } as unknown as ModelsHttpClient; + + await Netra.models.getModelPricing({ useCache: true }); + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/index.ts b/src/index.ts index d9393f1..2cfe47e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,9 +4,9 @@ * Built on top of OpenTelemetry and Traceloop */ -import { context, Span, SpanKind, trace } from "@opentelemetry/api"; +import { trace } from "@opentelemetry/api"; import { createRequire } from "module"; -import { Prompts, Dashboard, Evaluation, Usage } from "./api"; +import { Prompts, Dashboard, Evaluation, Usage, Models } from "./api"; import { Config, NetraConfig } from "./config"; import { initInstrumentations, instrumentationsReady, uninstrumentAll } from "./instrumentation"; import { Logger } from "./logger"; @@ -68,6 +68,10 @@ export { Usage, // Prompts API Prompts, + PROMPT_CACHE_TTL_SECONDS, + // Models API + Models, + MODEL_PRICING_CACHE_TTL_SECONDS, } from "./api"; export type { @@ -93,6 +97,9 @@ export type { QueryDataParams, QueryResponse, Run, + SessionDetailsResponse, + SessionDetailsToolCall, + SessionDetailsTrace, SessionUsageData, SpansPage, TaskFunction, @@ -107,6 +114,10 @@ export type { TraceSummary, GetPromptParams, PromptResponse, + // Models API + GetModelPricingParams, + ModelPrice, + ModelPricing, } from "./api"; // Export simulation types and classes @@ -140,7 +151,6 @@ export class Netra { private static _initialized = false; private static _config: Config | undefined; private static _tracer: any; - private static _rootSpan: Span | undefined; private static _metricsEnabled = false; static usage: Usage; @@ -148,6 +158,7 @@ export class Netra { static dashboard: Dashboard; static simulation: Simulation; static prompts: Prompts; + static models: Models; static getConfig(): Config { if (!this._config) { @@ -160,6 +171,9 @@ export class Netra { return this._initialized; } + /** + * Initialize the Netra SDK. + */ static async init(config: NetraConfig = {}): Promise { if (this._initialized) { Logger.warn("Netra.init() called more than once; ignoring subsequent calls."); @@ -214,25 +228,14 @@ export class Netra { Logger.warn("Netra: failed to initialize prompts client:", e); } - this._initialized = true; - Logger.info("Netra successfully initialized."); - - { - let pkgVersion = Config.LIBRARY_VERSION; - let pkgPath = "unknown"; - try { - const req = createRequire(import.meta.url); - pkgPath = req.resolve("../package.json"); - const pkg = req("../package.json"); - pkgVersion = pkg?.version || pkgVersion; - } catch { - // keep defaults - } - Logger.debug( - `SDK version=${pkgVersion} libraryVersion=${Config.LIBRARY_VERSION} build=langgraph-parenting-v3 packageJson=${pkgPath}`, - ); + try { + this.models = new Models(cfg); + } catch (e) { + Logger.warn("Netra: failed to initialize models client:", e); } + this._initialized = true; + // Graceful shutdown logic const handleSignal = async (signal: string) => { Logger.log(`\nReceived ${signal}. Shutting down Netra SDK...`); @@ -265,35 +268,9 @@ export class Netra { // Handle crashes process.once("uncaughtException", handleUncaughtException); - // Create root span if enabled - if (cfg.enableRootSpan) { - // Use the effective tracer if available - const tracer = this._tracer || trace.getTracer("netra.root.span"); - const rootName = `${Config.LIBRARY_NAME}.root.span`; - - // Create the root span - this._rootSpan = tracer.startSpan(rootName, { - kind: SpanKind.INTERNAL, - }); - - if (this._rootSpan) { - if (cfg.appName) { - this._rootSpan.setAttribute("service.name", cfg.appName); - } - this._rootSpan.setAttribute("netra.environment", cfg.environment); - this._rootSpan.setAttribute( - "netra.library.version", - Config.LIBRARY_VERSION, - ); - - Logger.info( - "Netra root span created. Use Netra.runWithRootSpan() to parent spans under it.", - ); - } - } - // Wait for all async instrumentations to be ready await instrumentationsReady; + Logger.info("Netra successfully initialized."); } static async shutdown(): Promise { @@ -308,15 +285,6 @@ export class Netra { Logger.error("Error during uninstrumentAll:", e); } - if (this._rootSpan) { - try { - this._rootSpan.end(); - } catch (e) { - } finally { - this._rootSpan = undefined; - } - } - const FLUSH_TIMEOUT_MS = 5000; try { @@ -344,6 +312,13 @@ export class Netra { Logger.error("Error during Netra trace shutdown:", e); } + try { + this.prompts?.clearCache(); + this.models?.clearCache(); + } catch (e) { + Logger.error("Error clearing SDK API caches:", e); + } + this._initialized = false; this._tracer = undefined; } @@ -368,21 +343,6 @@ export class Netra { SessionManager.setRootOutput(value); } - /** - * Run a function with the root span as the active parent context. - * All spans created within this function will be children of the root span. - * Note: required in JS because OTel JS has no persistent context.attach() - */ - static runWithRootSpan(fn: () => T): T { - if (!this._rootSpan) { - Logger.warn( - "runWithRootSpan: No root span available. Running function without parent context.", - ); - return fn(); - } - return context.with(trace.setSpan(context.active(), this._rootSpan), fn); - } - static setSessionId(sessionId: string): void { if (typeof sessionId !== "string") { Logger.error(`setSessionId: sessionId must be a string, got ${typeof sessionId}`); diff --git a/src/instrumentation/anthropic/utils.ts b/src/instrumentation/anthropic/utils.ts index b378226..0ff521f 100644 --- a/src/instrumentation/anthropic/utils.ts +++ b/src/instrumentation/anthropic/utils.ts @@ -1,5 +1,6 @@ import { Span, SpanStatusCode } from "@opentelemetry/api"; import { Logger } from "../../logger"; +import type { FirstTokenTracker } from "../../utils/span-timing"; import { setRequestAttributes as setBaseRequestAttributes, setResponseAttributes as setBaseResponseAttributes, @@ -13,6 +14,7 @@ export function processStreamChunk( completeResponse: Record, chunk: any, span: Span, + tokenTracker?: FirstTokenTracker, ): void { try { switch (chunk.type) { @@ -67,6 +69,7 @@ export function processStreamChunk( targetBlock.input += chunk.delta.partial_json ?? ""; } else if (chunk.delta?.text) { targetBlock.text += chunk.delta.text; + tokenTracker?.markFirstToken(); } break; } diff --git a/src/instrumentation/anthropic/wrappers.ts b/src/instrumentation/anthropic/wrappers.ts index 73267b0..41642aa 100644 --- a/src/instrumentation/anthropic/wrappers.ts +++ b/src/instrumentation/anthropic/wrappers.ts @@ -10,6 +10,10 @@ import { import { Logger } from "../../logger"; import { wrapResponse } from "../../utils/response-handler"; import { safeStringify } from "../../utils/serialization"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { SpanAttributes } from "../span-attributes"; import { defineHidden, @@ -38,6 +42,8 @@ const WRAPPER_OWN_PROPS = new Set([ "spanFinalized", "completionPending", "listenerMap", + "tokenTracker", + "ttftListener", ]); const EVENT_EMITTER_METHODS = new Set([ @@ -152,6 +158,8 @@ class MessageStreamWrapper { private spanFinalized = false; private completionPending = false; private listenerMap = new WeakMap>(); + private tokenTracker!: FirstTokenTracker; + private ttftListener!: (data: any) => void; constructor( span: Span, @@ -165,6 +173,7 @@ class MessageStreamWrapper { defineHidden(this, "messageStream", messageStream); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); defineHidden( this, "spanContext", @@ -238,7 +247,13 @@ class MessageStreamWrapper { if (prop === "removeAllListeners") { return function (event?: string) { target.listenerMap = new WeakMap(); - return method.call(target.messageStream, event); + const result = method.call(target.messageStream, event); + if (!event) { + target.attachSafetyNetListeners(); + } else if (event === "text") { + target.messageStream.on("text", target.ttftListener); + } + return result; }; } return method.bind(target.messageStream); @@ -254,12 +269,20 @@ class MessageStreamWrapper { const result = await method.call(target.messageStream, ...args); if (prop === "finalMessage" || prop === "done") { + if (result) { + const hasText = Array.isArray(result.content) && + result.content.some((b: any) => b.type === "text" && b.text); + if (hasText) { + target.tokenTracker.markFirstToken(); + } + } target.finalizeSpanFromMessage(result); } else if (prop === "finalText") { if (typeof result === "string" && result.length > 0) { target.completeResponse.content = [ { type: "text", text: result }, ]; + target.tokenTracker.markFirstToken(); } else { target.flushCurrentText(); } @@ -282,27 +305,35 @@ class MessageStreamWrapper { }); } + private attachSafetyNetListeners(): void { + this.ttftListener = (data: any) => { + if (data) this.tokenTracker.markFirstToken(); + }; + this.messageStream.on("text", this.ttftListener); + + this.messageStream.on("end", () => { + if (!this.completionPending) { + this.finalizeSpanOnce(SpanStatusCode.OK); + } + }); + this.messageStream.on("error", (err: any) => { + if (err && !this.spanFinalized) { + this.span.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : String(err), + }); + this.span.recordException( + err instanceof Error ? err : new Error(String(err)), + ); + } + this.finalizeSpanOnce(SpanStatusCode.ERROR); + }); + } + private registerSafetyNetListeners(): void { try { if (typeof this.messageStream?.on !== "function") return; - - this.messageStream.on("end", () => { - if (!this.completionPending) { - this.finalizeSpanOnce(SpanStatusCode.OK); - } - }); - this.messageStream.on("error", (err: any) => { - if (err && !this.spanFinalized) { - this.span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); - this.span.recordException( - err instanceof Error ? err : new Error(String(err)), - ); - } - this.finalizeSpanOnce(SpanStatusCode.ERROR); - }); + this.attachSafetyNetListeners(); } catch (e) { Logger.error( "netra.instrumentation.anthropic: safety net listener registration failed", @@ -319,7 +350,12 @@ class MessageStreamWrapper { let errorOccurred = false; try { for await (const chunk of this.messageStream) { - processStreamChunk(this.completeResponse, chunk, this.span); + processStreamChunk( + this.completeResponse, + chunk, + this.span, + this.tokenTracker, + ); yield chunk; } } catch (err) { @@ -350,6 +386,7 @@ class MessageStreamWrapper { this.completeResponse.currentText = ""; } this.completeResponse.currentText += data; + if (data) this.tokenTracker.markFirstToken(); break; case "contentBlock": @@ -444,13 +481,19 @@ function anthropicWrapper( model: "", usage: {}, }; + const tokenTracker = new FirstTokenTracker(span, startTime); return wrapResponse( response, { withContext: (fn) => context.with(spanContext, fn), onChunk: (chunk) => - processStreamChunk(completeResponse, chunk, span), + processStreamChunk( + completeResponse, + chunk, + span, + tokenTracker, + ), onError: (error) => { Logger.error("netra.instrumentation.anthropic:", error); span.setStatus({ @@ -468,6 +511,9 @@ function anthropicWrapper( "llm.response.duration", (endTime - startTime) / 1000, ); + if (requestType !== "batches") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } }, finalize: (status) => { const hasStreamData = diff --git a/src/instrumentation/google-genai/utils.ts b/src/instrumentation/google-genai/utils.ts index e2308ab..91840db 100644 --- a/src/instrumentation/google-genai/utils.ts +++ b/src/instrumentation/google-genai/utils.ts @@ -14,6 +14,7 @@ import { Span } from "@opentelemetry/api"; import { Logger } from "../../logger"; import { safeStringify } from "../../utils/serialization"; +import type { FirstTokenTracker } from "../../utils/span-timing"; import { SpanAttributes } from "../span-attributes"; import { TracedMessage, @@ -488,6 +489,7 @@ export function processStreamChunk( chunk: any, span: Span, startTime: number, + tokenTracker?: FirstTokenTracker, ): void { try { if (chunk.modelVersion) { @@ -502,10 +504,7 @@ export function processStreamChunk( if (chunkText && chunkText.length > 0) { if (!accumulated._text) { accumulated._text = chunkText; - span.setAttribute( - "gen_ai.performance.time_to_first_token", - (Date.now() - startTime) / 1000, - ); + tokenTracker?.markFirstToken(); } else { accumulated._text += chunkText; } diff --git a/src/instrumentation/google-genai/wrappers.ts b/src/instrumentation/google-genai/wrappers.ts index 724fc2e..3a5fd96 100644 --- a/src/instrumentation/google-genai/wrappers.ts +++ b/src/instrumentation/google-genai/wrappers.ts @@ -21,6 +21,10 @@ import { } from "@opentelemetry/api"; import { Logger } from "../../logger"; import { wrapResponse } from "../../utils/response-handler"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { createSuppressedContext, modelAsDict, @@ -127,6 +131,13 @@ function genericWrapperFactory( SpanAttributes.LLM_RESPONSE_DURATION, (endTime - startTime) / 1000, ); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes( + span, + startTime, + endTime, + ); + } }, onError: (error) => { span.setStatus({ @@ -194,6 +205,7 @@ function streamWrapperFactory( (span: Span) => { const startTime = Date.now(); const accumulated: Record = {}; + const tokenTracker = new FirstTokenTracker(span, startTime); try { setRequestAttributes(span, params, requestType); @@ -209,7 +221,13 @@ function streamWrapperFactory( { withContext: (fn) => context.with(spanContext, fn), onChunk: (chunk) => { - processStreamChunk(accumulated, chunk, span, startTime); + processStreamChunk( + accumulated, + chunk, + span, + startTime, + tokenTracker, + ); }, onError: (error) => { span.setStatus({ diff --git a/src/instrumentation/google-generative-ai/wrappers.ts b/src/instrumentation/google-generative-ai/wrappers.ts index e60a8a3..7eb0811 100644 --- a/src/instrumentation/google-generative-ai/wrappers.ts +++ b/src/instrumentation/google-generative-ai/wrappers.ts @@ -6,6 +6,10 @@ import { context, } from "@opentelemetry/api"; import { Logger } from "../../logger"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { isPromise, modelAsDict, @@ -99,12 +103,15 @@ function googleGenerativeAIWrapper( const endTime = Date.now(); const responseDict = modelAsDict(value); setResponseAttributes(span, responseDict); - const duration = (endTime - startTime) / 1000; - span.setAttribute("llm.response.duration", duration); + span.setAttribute( + "llm.response.duration", + (endTime - startTime) / 1000, + ); if (requestType !== "embedding") { - span.setAttribute( - "gen_ai.performance.time_to_first_token", - duration, + recordNonStreamingTimingAttributes( + span, + startTime, + endTime, ); } } catch (e) { @@ -133,6 +140,9 @@ function googleGenerativeAIWrapper( "llm.response.duration", (endTime - startTime) / 1000, ); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } } catch (e) { Logger.error(`${LOG_PREFIX}:`, e); } @@ -196,6 +206,8 @@ function googleGenerativeAIStreamWrapper( return response; } + const tokenTracker = new FirstTokenTracker(span, startTime); + return (async () => { try { const streamResult: any = await response; @@ -210,11 +222,14 @@ function googleGenerativeAIStreamWrapper( const endTime = Date.now(); const responseDict = modelAsDict(streamResult); setResponseAttributes(span, responseDict); - const duration = (endTime - startTime) / 1000; - span.setAttribute("llm.response.duration", duration); span.setAttribute( - "gen_ai.performance.time_to_first_token", - duration, + "llm.response.duration", + (endTime - startTime) / 1000, + ); + recordNonStreamingTimingAttributes( + span, + startTime, + endTime, ); } catch (e) { Logger.error(`${LOG_PREFIX}:`, e); @@ -224,8 +239,6 @@ function googleGenerativeAIStreamWrapper( return streamResult; } - let firstTokenRecorded = false; - const wrappedStream: AsyncIterable = { [Symbol.asyncIterator]() { const iterator = originalStream[Symbol.asyncIterator](); @@ -256,10 +269,9 @@ function googleGenerativeAIStreamWrapper( setResponseAttributes(span, responseDict); } - const duration = (endTime - startTime) / 1000; span.setAttribute( "llm.response.duration", - duration, + (endTime - startTime) / 1000, ); } catch (e) { Logger.error(`${LOG_PREFIX}:`, e); @@ -276,14 +288,8 @@ function googleGenerativeAIStreamWrapper( typeof chunk?.text === "function" ? chunk.text() : chunk?.text; - if (typeof t === "string") { - if (t && !firstTokenRecorded) { - span.setAttribute( - "gen_ai.performance.time_to_first_token", - (Date.now() - startTime) / 1000, - ); - firstTokenRecorded = true; - } + if (typeof t === "string" && t) { + tokenTracker.markFirstToken(); } } catch { // ignore chunk parsing issues @@ -305,8 +311,10 @@ function googleGenerativeAIStreamWrapper( }, async return(value?: any) { const endTime = Date.now(); - const duration = (endTime - startTime) / 1000; - span.setAttribute("llm.response.duration", duration); + span.setAttribute( + "llm.response.duration", + (endTime - startTime) / 1000, + ); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return iterator.return?.(value) ?? { value: undefined, done: true as const }; diff --git a/src/instrumentation/groq/wrappers.ts b/src/instrumentation/groq/wrappers.ts index 3743f32..adef38e 100644 --- a/src/instrumentation/groq/wrappers.ts +++ b/src/instrumentation/groq/wrappers.ts @@ -1,12 +1,16 @@ import { Tracer, Span, SpanKind, SpanStatusCode, context } from "@opentelemetry/api"; import { Logger } from "../../logger"; -import { setRequestAttributes, setResponseAttributes } from "./utils"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { defineHidden, modelAsDict, isPromise, shouldSuppressInstrumentation, } from "../utils"; +import { setRequestAttributes, setResponseAttributes } from "./utils"; type GroqRequestType = "chat"; @@ -98,6 +102,7 @@ function groqWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); + recordNonStreamingTimingAttributes(span, startTime, endTime); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return value; @@ -121,6 +126,7 @@ function groqWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); + recordNonStreamingTimingAttributes(span, startTime, endTime); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return response; @@ -152,12 +158,14 @@ export class StreamingWrapper implements Iterable, Iterator { private response!: any; private startTime!: number; private requestKwargs!: Record; + private tokenTracker!: FirstTokenTracker; constructor(span: Span, response: any, startTime: number, requestKwargs: Record) { defineHidden(this, "span", span); defineHidden(this, "response", response); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); } toJSON() { @@ -224,6 +232,7 @@ export class StreamingWrapper implements Iterable, Iterator { choices[index].message = { role: "assistant", content: "" }; } choices[index].message.content += String(delta.content); + this.tokenTracker.markFirstToken(); } if (choice.finish_reason) { @@ -235,16 +244,19 @@ export class StreamingWrapper implements Iterable, Iterator { if (chunkDict.usage) this.completeResponse.usage = chunkDict.usage; if (chunkDict.response?.status === "completed") { + let hasText = false; const outputs = chunkDict.response.output || []; outputs.forEach((output: any) => { const content = output.content || []; content.forEach((item: any) => { + if (item.text) hasText = true; choices.push({ message: { role: "assistant", content: item.text || "" }, }); }); }); this.completeResponse.usage = chunkDict.response.usage || {}; + if (hasText) this.tokenTracker.markFirstToken(); } this.span.addEvent("llm.content.completion.chunk"); @@ -281,12 +293,14 @@ export class AsyncStreamingWrapper private response!: any; private startTime!: number; private requestKwargs!: Record; + private tokenTracker!: FirstTokenTracker; constructor(span: Span, response: any, startTime: number, requestKwargs: Record) { defineHidden(this, "span", span); defineHidden(this, "response", response); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); } toJSON() { @@ -367,6 +381,7 @@ export class AsyncStreamingWrapper } const message = choiceEntry.message as Record; message.content = String(message.content || "") + contentPiece; + this.tokenTracker.markFirstToken(); } if (choice.finish_reason) { @@ -379,6 +394,7 @@ export class AsyncStreamingWrapper // Response API if ((chunkDict.response as any)?.status === "completed") { + let hasText = false; const response = chunkDict.response as Record; const responseOutput = (response.output || []) as Array< Record @@ -388,7 +404,7 @@ export class AsyncStreamingWrapper if (content) { for (const contentItem of content) { const assistantText = contentItem.text || ""; - // Append to choices array instead of replacing + if (contentItem.text) hasText = true; ( this.completeResponse.choices as Array> ).push({ @@ -399,6 +415,7 @@ export class AsyncStreamingWrapper const usage = response.usage || {}; this.completeResponse.usage = usage; }); + if (hasText) this.tokenTracker.markFirstToken(); } this.span.addEvent("llm.content.completion.chunk"); } diff --git a/src/instrumentation/index.ts b/src/instrumentation/index.ts index 3142785..b560fce 100644 --- a/src/instrumentation/index.ts +++ b/src/instrumentation/index.ts @@ -125,7 +125,7 @@ function patchTraceloopLangchainCallbackHandler(): void { try { const mod = require("@traceloop/instrumentation-langchain"); - Logger.debug(`Loaded @traceloop/instrumentation-langchain via require from ${require.resolve("@traceloop/instrumentation-langchain")}`); + Logger.debug("Loaded @traceloop/instrumentation-langchain"); applyPatch(mod, "require"); return; } catch (e) { @@ -395,8 +395,6 @@ export function initInstrumentations( Logger.debug(` App Name: ${config.appName}`); Logger.debug(` OTLP Endpoint: ${config.otlpEndpoint || "(default - localhost:3002)"}`); Logger.debug(` API Key: ${config.apiKey ? "***" + config.apiKey.slice(-4) : "(not set)"}`); - Logger.debug(` Trace Content: ${config.traceContent}`); - Logger.debug(` Enable Scrubbing: ${config.enableScrubbing}`); // Initialize Traceloop SDK const traceloopOptions: InitializeOptions = { diff --git a/src/instrumentation/langgraph/index.ts b/src/instrumentation/langgraph/index.ts index 473053f..a662d49 100644 --- a/src/instrumentation/langgraph/index.ts +++ b/src/instrumentation/langgraph/index.ts @@ -32,48 +32,33 @@ function findModuleInCache(moduleName: string): any { } Logger.debug( `Module ${moduleName} not found in require.cache. Cache keys containing 'langgraph':`, - Object.keys(cache).filter(k => k.includes('langgraph')), + Object.keys(cache).filter((k) => k.includes('langgraph')), ); } return null; } async function resolveLanggraph(): Promise { + const moduleName = "@langchain/langgraph"; if (LanggraphClass) return LanggraphClass; try { // First, try to find the module in require.cache (already loaded by the app) // This ensures we patch the same module instance the app is using - let langgraphModule = findModuleInCache('@langchain/langgraph'); + let langgraphModule = findModuleInCache(moduleName); - if (langgraphModule) { - Logger.debug("Found @langchain/langgraph in require.cache (using app's module instance)"); - } else { + if (!langgraphModule) { // Fallback to dynamic import if not in cache - langgraphModule = await import("@langchain/langgraph"); - Logger.debug("Loaded @langchain/langgraph via dynamic import"); + langgraphModule = await import(moduleName); } - Logger.debug("LangGraph Module Exports:", Object.keys(langgraphModule)); - LanggraphClass = langgraphModule.CompiledStateGraph ?? langgraphModule.StateGraph; - Logger.debug("Resolved LanggraphClass:", !!LanggraphClass); - Logger.debug("LanggraphClass name:", LanggraphClass?.name); - Logger.debug( - "LanggraphClass.prototype keys:", - LanggraphClass?.prototype ? Object.getOwnPropertyNames(LanggraphClass.prototype) : "no prototype", - ); - Logger.debug("Has invoke on prototype:", !!LanggraphClass?.prototype?.invoke); - Logger.debug("Has stream on prototype:", !!LanggraphClass?.prototype?.stream); + LanggraphClass = + langgraphModule.CompiledStateGraph ?? langgraphModule.StateGraph; // Check prototype chain to find where invoke is defined - Logger.debug("Checking prototype chain for invoke location:"); let proto = LanggraphClass?.prototype; while (proto) { - const hasOwn = Object.getOwnPropertyNames(proto).includes('invoke'); - Logger.debug(` ${proto.constructor?.name}: hasOwnProperty('invoke')=${hasOwn}`); - if (hasOwn) { - Logger.debug(` -> invoke is defined on: ${proto.constructor?.name}`); - break; - } + const hasOwn = Object.getOwnPropertyNames(proto).includes("invoke"); + if (hasOwn) break; proto = Object.getPrototypeOf(proto); } @@ -160,8 +145,6 @@ export class NetraLanggraphInstrumentor { return; } - Logger.debug(`Found invoke on prototype: ${targetProto.constructor?.name}`); - const originalInvoke = targetProto.invoke; originalMethods.set("langgraph.graph.invoke", originalInvoke); // Store the target prototype for uninstrumentation @@ -188,9 +171,6 @@ export class NetraLanggraphInstrumentor { // Add marker to identify patched method (patchedInvoke as any).__netra_patched = true; targetProto.invoke = patchedInvoke; - - Logger.debug(`Successfully instrumented LangGraph invoke method on ${targetProto.constructor?.name}`); - Logger.debug(`Patched Pregel class identity:`, targetProto.constructor); } catch (error) { Logger.error(`Failed to instrument langgraph invoke: ${error}`); } @@ -211,8 +191,6 @@ export class NetraLanggraphInstrumentor { return; } - Logger.debug(`Found stream on prototype: ${targetProto.constructor?.name}`); - const originalStream = targetProto.stream; originalMethods.set("langgraph.graph.stream", originalStream); // Store the target prototype for uninstrumentation @@ -236,8 +214,6 @@ export class NetraLanggraphInstrumentor { ...rest, ); }; - - Logger.debug(`Successfully instrumented LangGraph stream method on ${targetProto.constructor?.name}`); } catch (error) { Logger.error(`Failed to instrument langgraph stream: ${error}`); } diff --git a/src/instrumentation/langgraph/wrappers.ts b/src/instrumentation/langgraph/wrappers.ts index f319dce..b434193 100644 --- a/src/instrumentation/langgraph/wrappers.ts +++ b/src/instrumentation/langgraph/wrappers.ts @@ -14,55 +14,12 @@ import { } from "@opentelemetry/api"; import { Logger } from "../../logger"; +import { recordNonStreamingTimingAttributes } from "../../utils/span-timing"; import { defineHidden, setResponseAttributes as setBaseResponseAttributes, shouldSuppressInstrumentation, } from "../utils"; - -// Context key to track if we're inside a LangGraph instrumented call -// This prevents double-instrumentation when invoke internally calls stream -const LANGGRAPH_INSTRUMENTATION_ACTIVE = createContextKey("netra.langgraph.active"); - -function getContextManager(): any { - try { - if ((context as any)._getContextManager) { - return (context as any)._getContextManager(); - } - - const globalSymbols = Object.getOwnPropertySymbols(global); - const otelSymbol = globalSymbols.find(s => - s.toString().includes("opentelemetry.js.api"), - ); - if (otelSymbol) { - const globalState = (global as any)[otelSymbol]; - if (globalState?.contextManager) { - return globalState.contextManager; - } - } - - return null; - } catch { - return null; - } -} - -function enterWithContext(newContext: Context): void { - const contextManager = getContextManager(); - if (!contextManager) return; - - if (typeof contextManager.enterWith === "function") { - contextManager.enterWith(newContext); - return; - } - - if ( - contextManager._asyncLocalStorage && - typeof contextManager._asyncLocalStorage.enterWith === "function" - ) { - contextManager._asyncLocalStorage.enterWith(newContext); - } -} import { NetraLanggraphAttributes, setChainInputAttributes, @@ -73,6 +30,10 @@ import { setToolAttributes, } from "./utils"; +// Context key to track if we're inside a LangGraph instrumented call +// This prevents double-instrumentation when invoke internally calls stream +const LANGGRAPH_INSTRUMENTATION_ACTIVE = createContextKey("netra.langgraph.active"); + type AnyFunc = (...args: any[]) => any; type AsyncIterableFunc = (...args: any[]) => Promise>; @@ -83,6 +44,7 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { private nodeAttributes: Map> = new Map(); private runStack: string[] = []; private inferredParents: Map = new Map(); + private streamedRuns: Set = new Set(); constructor( private tracer: Tracer, @@ -241,10 +203,19 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { metadata, prompts, extraParams, - parentRunId: effectiveParentRunId, // Store parent ID to link back + parentRunId: effectiveParentRunId, + startTimeMs: Date.now(), }); } + async handleLLMNewToken( + _token: string, + _idx: { prompt: number; completion: number }, + runId: string, + ) { + this.streamedRuns.add(runId); + } + async handleLLMEnd( output: LLMResult, runId: string, @@ -272,9 +243,15 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { attributes.extraParams, ); setBaseResponseAttributes(span, response); + // Streaming TTFT is captured by the underlying provider instrumentation + // (e.g. OpenAI, Anthropic). We only record timing for non-streaming calls. + if (attributes.startTimeMs && !this.streamedRuns.has(runId)) { + recordNonStreamingTimingAttributes(span, attributes.startTimeMs, Date.now()); + } span.end(); this.nodeAttributes.delete(runId); + this.streamedRuns.delete(runId); } async handleLLMError( @@ -302,6 +279,7 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { span.end(); this.nodeAttributes.delete(runId); + this.streamedRuns.delete(runId); } async handleToolStart( @@ -411,14 +389,19 @@ class LanggraphStreamingWrapper implements AsyncIterable { config?: RunnableConfig, ...rest: any[] ) { - this.iterable = await originalFunc.call(instance, input, config, ...rest); + const spanContext = trace.setSpan(this.rootContext, this.rootSpan); + this.iterable = await context.with(spanContext, () => + originalFunc.call(instance, input, config, ...rest), + ); return this; } async *[Symbol.asyncIterator]() { const spanContext = trace.setSpan(this.rootContext, this.rootSpan); try { - const iterator = await this.iterable[Symbol.asyncIterator](); + const iterator = await context.with(spanContext, () => + this.iterable[Symbol.asyncIterator](), + ); while (true) { let result: any; await context.with(spanContext, async () => { @@ -427,7 +410,7 @@ class LanggraphStreamingWrapper implements AsyncIterable { if (result.done) break; const value = result?.value ?? {}; this.output = { ...this.output, ...value }; - yield result; + yield value; } this.rootSpan.setAttribute( NetraLanggraphAttributes.entityOutput, @@ -502,7 +485,6 @@ export class LanggraphWrapper { // Set the active flag to prevent nested instrumentation (e.g., when invoke calls stream internally) const ctxWithSpan = trace.setSpan(context.active(), span); const ctxWithFlag = ctxWithSpan.setValue(LANGGRAPH_INSTRUMENTATION_ACTIVE, true); - enterWithContext(ctxWithFlag); return context.with(ctxWithFlag, async () => { { @@ -561,18 +543,20 @@ export class LanggraphWrapper { try { const ctxWithSpan = trace.setSpan(context.active(), span); const ctxWithFlag = ctxWithSpan.setValue(LANGGRAPH_INSTRUMENTATION_ACTIVE, true); - enterWithContext(ctxWithFlag); const streamingWrapper = new LanggraphStreamingWrapper(span, ctxWithFlag); - return streamingWrapper.startStream( - originalFunc, - instance, - input, - updatedConfig, - ...rest, + return await context.with(ctxWithFlag, () => + streamingWrapper.startStream( + originalFunc, + instance, + input, + updatedConfig, + ...rest, + ), ); } catch (error) { span.recordException(error as Error); span.end(); + throw error; } } } diff --git a/src/instrumentation/mistralai/wrappers.ts b/src/instrumentation/mistralai/wrappers.ts index c904c3d..6794104 100644 --- a/src/instrumentation/mistralai/wrappers.ts +++ b/src/instrumentation/mistralai/wrappers.ts @@ -10,6 +10,10 @@ import { context, } from "@opentelemetry/api"; import { Logger } from "../../logger"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { defineHidden, isPromise } from "../utils"; import { modelAsDict, @@ -67,6 +71,9 @@ function mistralWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } span.setStatus({ code: SpanStatusCode.OK }); span.end(); return value; @@ -90,6 +97,9 @@ function mistralWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } span.setStatus({ code: SpanStatusCode.OK }); span.end(); return response; @@ -245,12 +255,14 @@ export class StreamingWrapper implements Iterable, Iterator { private response!: unknown; private startTime!: number; private requestKwargs!: Record; + private tokenTracker!: FirstTokenTracker; constructor(span: Span, response: unknown, startTime: number, requestKwargs: Record) { defineHidden(this, "span", span); defineHidden(this, "response", response); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); } toJSON() { @@ -354,6 +366,7 @@ export class StreamingWrapper implements Iterable, Iterator { } else { choiceEntry.text = String(choiceEntry.text || "") + contentPiece; } + this.tokenTracker.markFirstToken(); } if (choice.finishReason) { @@ -389,6 +402,7 @@ export class StreamingWrapper implements Iterable, Iterator { } else { choiceEntry.text = String(choiceEntry.text || "") + contentPiece; } + this.tokenTracker.markFirstToken(); } if (choice.finishReason) { @@ -439,17 +453,19 @@ export class AsyncStreamingWrapper private startTime!: number; private requestKwargs!: Record; private completeResponse: Record; + private tokenTracker!: FirstTokenTracker; constructor( span: Span, response: unknown, startTime: number, - requestKwargs: Record + requestKwargs: Record, ) { defineHidden(this, "span", span); defineHidden(this, "response", response); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); this.completeResponse = { choices: [], model: "" }; } @@ -572,6 +588,7 @@ export class AsyncStreamingWrapper } else { choiceEntry.text = String(choiceEntry.text || "") + contentPiece; } + this.tokenTracker.markFirstToken(); } if (choice.finishReason) { @@ -607,6 +624,7 @@ export class AsyncStreamingWrapper } else { choiceEntry.text = String(choiceEntry.text || "") + contentPiece; } + this.tokenTracker.markFirstToken(); } if (choice.finishReason) { diff --git a/src/instrumentation/openai/wrappers.ts b/src/instrumentation/openai/wrappers.ts index 5a9a455..ee3bfa2 100644 --- a/src/instrumentation/openai/wrappers.ts +++ b/src/instrumentation/openai/wrappers.ts @@ -7,6 +7,10 @@ import { context, trace, } from "@opentelemetry/api"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { defineHidden, isPromise, @@ -15,6 +19,7 @@ import { } from "../utils"; import { setRequestAttributes, setResponseAttributes } from "./utils"; import { OpenAIRequestType, StreamResponse, WrapperFn } from "./types"; +import { Logger } from "../../logger"; const SPAN_NAMES: Record = { chat: "openai.chat", @@ -45,9 +50,14 @@ function finalizeSpanSuccess( span: Span, response: Record, startTime: number, + requestType: OpenAIRequestType, ): void { + const endTime = Date.now(); setResponseAttributes(span, response); - span.setAttribute("llm.response.duration", (Date.now() - startTime) / 1000); + span.setAttribute("llm.response.duration", (endTime - startTime) / 1000); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } span.setStatus({ code: SpanStatusCode.OK }); span.end(); } @@ -58,6 +68,7 @@ abstract class BaseStreamHandler { protected span!: Span; protected startTime!: number; protected requestKwargs!: Record; + protected tokenTracker!: FirstTokenTracker; constructor( span: Span, @@ -67,6 +78,7 @@ abstract class BaseStreamHandler { defineHidden(this, "span", span); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); } toJSON(): StreamResponse { @@ -93,6 +105,7 @@ abstract class BaseStreamHandler { } const msg = entry.message as Record; msg.content = String(msg.content ?? "") + String(delta.content); + this.tokenTracker.markFirstToken(); } if (choice.finish_reason) { this.completeResponse.choices[index].finish_reason = @@ -110,6 +123,7 @@ abstract class BaseStreamHandler { | Record | undefined; if (responseChunk?.status === "completed") { + let hasText = false; const outputs = (responseChunk.output ?? []) as Array< Record >; @@ -119,6 +133,7 @@ abstract class BaseStreamHandler { | undefined; if (Array.isArray(content)) { for (const item of content) { + if (item.text) hasText = true; this.completeResponse.choices.push({ message: { role: "assistant", content: String(item.text ?? "") }, }); @@ -126,6 +141,7 @@ abstract class BaseStreamHandler { } } this.completeResponse.usage = responseChunk.usage ?? {}; + if (hasText) this.tokenTracker.markFirstToken(); } this.span.addEvent("llm.content.completion.chunk"); @@ -141,20 +157,20 @@ abstract class BaseStreamHandler { } protected finalizeSpan(code: SpanStatusCode): void { - if (code === SpanStatusCode.OK) { - finalizeSpanSuccess( + try { + setResponseAttributes( this.span, this.completeResponse as Record, - this.startTime, - ); - } else { - this.span.setAttribute( - "llm.response.duration", - (Date.now() - this.startTime) / 1000, ); - this.span.setStatus({ code }); - this.span.end(); + } catch { + Logger.debug('Failed to set response attributes'); } + this.span.setAttribute( + "llm.response.duration", + (Date.now() - this.startTime) / 1000, + ); + this.span.setStatus({ code }); + this.span.end(); } } @@ -317,7 +333,7 @@ function executeNonStreaming( if (isPromise(result)) { return result.then( (value) => { - finalizeSpanSuccess(span, modelAsDict(value), startTime); + finalizeSpanSuccess(span, modelAsDict(value), startTime, requestType); return value; }, (error) => { @@ -327,7 +343,7 @@ function executeNonStreaming( ); } - finalizeSpanSuccess(span, modelAsDict(result), startTime); + finalizeSpanSuccess(span, modelAsDict(result), startTime, requestType); return result; } catch (error) { handleSpanError(span, error); diff --git a/src/instrumentation/span-attributes.ts b/src/instrumentation/span-attributes.ts index 578810d..5500cf8 100644 --- a/src/instrumentation/span-attributes.ts +++ b/src/instrumentation/span-attributes.ts @@ -33,4 +33,10 @@ export const SpanAttributes = { LLM_IS_STREAMING: "llm.is_streaming", LLM_COMPLETIONS: "gen_ai.completion", LLM_PROMPTS: "gen_ai.prompt", + + LLM_TIME_TO_FIRST_TOKEN: "gen_ai.performance.time_to_first_token", + LLM_RELATIVE_TIME_TO_FIRST_TOKEN: + "gen_ai.performance.relative_time_to_first_token", + LLM_TIME_TO_FIRST_TOKEN_TIMESTAMP: + "gen_ai.performance.time_to_first_token.timestamp", } as const; diff --git a/src/instrumentation/utils.ts b/src/instrumentation/utils.ts index 53db4ae..81baa92 100644 --- a/src/instrumentation/utils.ts +++ b/src/instrumentation/utils.ts @@ -584,7 +584,13 @@ function setUsageAttributes( ); } - const cacheCreationTokens = usage.cache_creation_input_tokens; + const cacheCreationTokens = + usage.cache_creation_input_tokens ?? + ( + (usage.prompt_tokens_details ?? usage.input_tokens_details) as + | { cache_write_tokens?: unknown } + | undefined + )?.cache_write_tokens; if (cacheCreationTokens !== undefined) { span.setAttribute( SpanAttributes.LLM_USAGE_CACHE_CREATION_INPUT_TOKENS, diff --git a/src/processors/span-io-processor.ts b/src/processors/span-io-processor.ts index 77129ed..2d714bc 100644 --- a/src/processors/span-io-processor.ts +++ b/src/processors/span-io-processor.ts @@ -248,7 +248,17 @@ export class SpanIOProcessor implements SpanProcessor { return original(_USAGE_COMPLETION_TOKENS, value); } - // 9. Pass through + // 9. db.statement → input (DB instrumentations: TypeORM, etc.) + // Keep db.statement; never map into output. + if (key === "db.statement") { + original(key, value); + if (!inputLocked() && inputIsEmpty() && !isEmpty(value)) { + original("input", typeof value === "string" ? value : String(value)); + } + return span; + } + + // 10. Pass through return original(key, value); } catch (e) { Logger.debug(`SpanIOProcessor: error processing key=${key}`, e); diff --git a/src/simulation/api.ts b/src/simulation/api.ts index 21e7d4a..8889bb4 100644 --- a/src/simulation/api.ts +++ b/src/simulation/api.ts @@ -463,7 +463,11 @@ export class Simulation { let rawFiles: FileData[] = initialFiles ?? []; while (true) { - const span = new SpanWrapper(SPAN_NAME, {}, LOG_PREFIX); + const span = new SpanWrapper( + SPAN_NAME, + { [Config.TRACE_ORIGIN_KEY]: Config.TRACE_ORIGIN_EVALUATION }, + LOG_PREFIX, + ); span.start(); try { diff --git a/src/utils/span-timing.ts b/src/utils/span-timing.ts new file mode 100644 index 0000000..49eeda9 --- /dev/null +++ b/src/utils/span-timing.ts @@ -0,0 +1,81 @@ +import { Span } from "@opentelemetry/api"; +import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { RootSpanProcessor } from "../processors/root-span-processor"; +import { SpanAttributes } from "../instrumentation/span-attributes"; +import { Logger } from "../logger"; + +function hrTimeToMs(hrTime: [number, number]): number { + return hrTime[0] * 1000 + hrTime[1] / 1e6; +} + +function recordTimeToFirstToken( + span: Span, + nowMs: number, + startTimeMs: number, +): void { + span.setAttribute( + SpanAttributes.LLM_TIME_TO_FIRST_TOKEN, + (nowMs - startTimeMs) / 1000, + ); + span.setAttribute( + SpanAttributes.LLM_TIME_TO_FIRST_TOKEN_TIMESTAMP, + new Date(nowMs).toISOString(), + ); +} + +function recordRelativeTimeToFirstToken(span: Span, nowMs: number): void { + try { + const rootSpan = RootSpanProcessor.getRootSpan(span); + if (!rootSpan) return; + const hrStart = (rootSpan as unknown as ReadableSpan).startTime; + if (!hrStart) return; + const rootStartMs = hrTimeToMs(hrStart); + span.setAttribute( + SpanAttributes.LLM_RELATIVE_TIME_TO_FIRST_TOKEN, + (nowMs - rootStartMs) / 1000, + ); + } catch (e) { + Logger.warn("span-timing: failed to compute RTTFT", e); + } +} + +/** + * Tracks the first content token in a streaming LLM response and records + * TTFT, RTTFT, and the absolute first-token timestamp on the span. + * + * `markFirstToken()` is idempotent — only the first call writes attributes. + */ +export class FirstTokenTracker { + private _recorded = false; + + constructor( + private readonly span: Span, + private readonly startTimeMs: number, + ) {} + + markFirstToken(): void { + if (this._recorded || !this.span.isRecording()) return; + this._recorded = true; + + const now = Date.now(); + + recordTimeToFirstToken(this.span, now, this.startTimeMs); + recordRelativeTimeToFirstToken(this.span, now); + } +} + +/** + * Record TTFT + RTTFT for non-streaming LLM calls. + * For non-streaming, "first token" = full response arrival, + * so TTFT equals response duration. + */ +export function recordNonStreamingTimingAttributes( + span: Span, + startTimeMs: number, + endTimeMs: number, +): void { + if (!span.isRecording()) return; + + recordTimeToFirstToken(span, endTimeMs, startTimeMs); + recordRelativeTimeToFirstToken(span, endTimeMs); +} diff --git a/src/version.ts b/src/version.ts index f27afec..9ae0113 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const SDK_VERSION = "1.8.0"; +export const SDK_VERSION = "1.9.0-dev.0";