From 6aa62a80bb02fe36b3ee70cb686c83a9c588a201 Mon Sep 17 00:00:00 2001 From: Akhilesh Nair <72593014+AkhileshNair2201@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:06:32 +0530 Subject: [PATCH 01/20] NET-1329 feat: add opt-in TTL caching for getPrompt (#110) * feat: add opt-in TTL caching for getPrompt Co-authored-by: Cursor * fix: downgrade vitest for Node 18 and tighten cache tests Use Vitest 3.x so dev test runs align with engines.node >=18, and add getPrompt cacheTtl integration coverage plus shutdown re-init behavior. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- README.md | 36 ++ package-lock.json | 565 ++++++++++++++++++++++++++++++- package.json | 5 +- src/api/prompts/api.test.ts | 120 +++++++ src/api/prompts/api.ts | 32 +- src/api/prompts/models.ts | 4 + src/cache.test.ts | 55 +++ src/cache.ts | 35 ++ src/config.ts | 26 ++ src/index.shutdown-cache.test.ts | 43 +++ src/index.ts | 11 + 11 files changed, 919 insertions(+), 13 deletions(-) create mode 100644 src/api/prompts/api.test.ts create mode 100644 src/cache.test.ts create mode 100644 src/cache.ts create mode 100644 src/index.shutdown-cache.test.ts diff --git a/README.md b/README.md index c99b256..079be41 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ - 🔧 **Multi-Provider Support**: Works with OpenAI, Google GenAI, Mistral, Anthropic, and more - 📈 **Session Management**: Track user sessions and custom attributes - 🌐 **Automatic Instrumentation**: Zero-code instrumentation for popular frameworks and libraries +- ⚡ **Opt-in Read Caching**: In-memory TTL caching for read-heavy SDK calls (e.g. prompt fetches) ## 📦 Installation @@ -184,6 +185,40 @@ async function generateContent(prompt: string) { } ``` +## 📝 Prompts API + +Fetch prompt versions from Prompt Studio via `Netra.prompts.getPrompt()`. Caching is **opt-in per call** — omit `useCache` (or set it to `false`) to always hit the API. + +Set a global default TTL at init with `cacheTtlSeconds` (default: **60** seconds), or override TTL for a single call with `cacheTtl`. + +```typescript +import { Netra } from "netra-sdk"; + +await Netra.init({ + appName: "my-ai-app", + cacheTtlSeconds: 60, // optional; env: NETRA_CACHE_TTL_SECONDS +}); + +// Always fetches from the API (default) +const prompt = await Netra.prompts.getPrompt({ name: "my-prompt" }); + +// Cached for 60s (global default TTL) +const cached = await Netra.prompts.getPrompt({ + name: "my-prompt", + useCache: true, +}); + +// Cached for 30s for this call only +const shortLived = await Netra.prompts.getPrompt({ + name: "my-prompt", + label: "production", // default label when omitted + useCache: true, + cacheTtl: 30, +}); +``` + +> **Note**: Cached prompts may be stale for up to the TTL after dashboard edits. Use `useCache: false` when you need the latest version immediately. `Netra.shutdown()` clears in-memory caches. + ## 🔧 Environment Variables You can configure the SDK using environment variables: @@ -194,6 +229,7 @@ You can configure the SDK using environment variables: | `NETRA_APP_NAME` | Name of your application | | `NETRA_ENV` | Environment (e.g., prod, dev) | | `NETRA_TRACE_CONTENT` | Capture prompt/completion content (default: true) | +| `NETRA_CACHE_TTL_SECONDS` | Default TTL in seconds for opt-in SDK read caches (default: 60) | ## 🤝 License diff --git a/package-lock.json b/package-lock.json index 9e92f2a..bc6fb8b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,7 +28,8 @@ "@types/shimmer": "^1.2.0", "ts-node": "^10.9.2", "tsup": "^8.5.1", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^3.2.4" }, "engines": { "node": ">=18.0.0" @@ -3536,6 +3537,24 @@ "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", "license": "MIT" }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -3600,6 +3619,121 @@ "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", "license": "MIT" }, + "node_modules/@vitest/expect": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.7.tgz", + "integrity": "sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.7.tgz", + "integrity": "sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.7", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.7.tgz", + "integrity": "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.7.tgz", + "integrity": "sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.7", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.7.tgz", + "integrity": "sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.7.tgz", + "integrity": "sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.7.tgz", + "integrity": "sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/a-sync-waterfall": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz", @@ -3702,6 +3836,16 @@ "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", "license": "MIT" }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -3842,6 +3986,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -3885,6 +4046,16 @@ "node": ">=8" } }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, "node_modules/cjs-module-lexer": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", @@ -4012,6 +4183,16 @@ "node": ">=0.10.0" } }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -4099,6 +4280,13 @@ "node": ">= 0.4" } }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, "node_modules/es-object-atoms": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", @@ -4177,6 +4365,16 @@ "node": ">=6" } }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -4213,6 +4411,16 @@ "node": ">=18.0.0" } }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -4710,6 +4918,13 @@ "base64-js": "^1.5.1" } }, + "node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -4832,6 +5047,13 @@ "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", "license": "Apache-2.0" }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -4929,6 +5151,25 @@ "thenify-all": "^1.0.0" } }, + "node_modules/nanoid": { + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -5099,6 +5340,16 @@ "dev": true, "license": "MIT" }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -5107,9 +5358,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", "engines": { @@ -5141,6 +5392,35 @@ "pathe": "^2.0.1" } }, + "node_modules/postcss": { + "version": "8.5.16", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", + "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, "node_modules/postcss-load-config": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", @@ -5399,6 +5679,13 @@ "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==", "license": "BSD-2-Clause" }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, "node_modules/simple-wcswidth": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", @@ -5415,6 +5702,30 @@ "node": ">= 12" } }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, "node_modules/stream-events": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", @@ -5465,6 +5776,19 @@ "node": ">=8" } }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, "node_modules/stubs": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", @@ -5605,6 +5929,13 @@ "node": ">=0.8" } }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, "node_modules/tinyexec": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", @@ -5613,14 +5944,14 @@ "license": "MIT" }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -5629,6 +5960,36 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", + "integrity": "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -5838,6 +6199,177 @@ "dev": true, "license": "MIT" }, + "node_modules/vite": { + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", + "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0 || ^0.28.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vitest": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.7.tgz", + "integrity": "sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.7", + "@vitest/mocker": "3.2.7", + "@vitest/pretty-format": "^3.2.7", + "@vitest/runner": "3.2.7", + "@vitest/snapshot": "3.2.7", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.7", + "@vitest/ui": "3.2.7", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -5861,6 +6393,23 @@ "webidl-conversions": "^3.0.0" } }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index f0ed327..2e50dab 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "build": "tsup", "start:dev": "tsup --watch", "prepack": "npm run build", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "vitest run" }, "keywords": [ "netra", @@ -121,6 +121,7 @@ "@types/shimmer": "^1.2.0", "ts-node": "^10.9.2", "tsup": "^8.5.1", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^3.2.4" } } diff --git a/src/api/prompts/api.test.ts b/src/api/prompts/api.test.ts new file mode 100644 index 0000000..9044eb5 --- /dev/null +++ b/src/api/prompts/api.test.ts @@ -0,0 +1,120 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Config } from "../../config"; +import { Prompts } from "./api"; +import { PromptsHttpClient } from "./client"; + +describe("Prompts.getPrompt caching", () => { + let prompts: Prompts; + let getPromptVersion: ReturnType; + + beforeEach(() => { + const config = new Config({ cacheTtlSeconds: 60 }); + prompts = new Prompts(config); + getPromptVersion = vi.fn(); + (prompts as unknown as { client: PromptsHttpClient }).client = { + getPromptVersion, + } as unknown as PromptsHttpClient; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("calls HTTP on every request when useCache is omitted", async () => { + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ name: "my-prompt" }); + await prompts.getPrompt({ name: "my-prompt" }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("serves from cache on second call with same name and label when useCache is true", async () => { + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + const first = await prompts.getPrompt({ name: "my-prompt", useCache: true }); + const second = await prompts.getPrompt({ name: "my-prompt", useCache: true }); + + expect(getPromptVersion).toHaveBeenCalledTimes(1); + expect(first).toEqual({ template: "v1" }); + expect(second).toEqual({ template: "v1" }); + }); + + it("keeps separate cache entries per label when useCache is true", async () => { + getPromptVersion + .mockResolvedValueOnce({ data: { template: "prod" } }) + .mockResolvedValueOnce({ data: { template: "staging" } }); + + const prod = await prompts.getPrompt({ + name: "my-prompt", + label: "production", + useCache: true, + }); + const staging = await prompts.getPrompt({ + name: "my-prompt", + label: "staging", + useCache: true, + }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + expect(prod).toEqual({ template: "prod" }); + expect(staging).toEqual({ template: "staging" }); + }); + + it("does not cache null API responses", async () => { + getPromptVersion.mockResolvedValue(null); + + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("ignores cache when useCache is false even if cacheTtl is set", async () => { + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ + name: "my-prompt", + useCache: false, + cacheTtl: 30, + }); + await prompts.getPrompt({ + name: "my-prompt", + useCache: false, + cacheTtl: 30, + }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("expires per-call cacheTtl before the global default TTL", async () => { + vi.useFakeTimers(); + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ + name: "my-prompt", + useCache: true, + cacheTtl: 1, + }); + expect(getPromptVersion).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(1001); + + await prompts.getPrompt({ + name: "my-prompt", + useCache: true, + cacheTtl: 1, + }); + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + + it("hits HTTP again after clearCache when useCache is true", async () => { + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + prompts.clearCache(); + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/api/prompts/api.ts b/src/api/prompts/api.ts index 7709cf2..90c3423 100644 --- a/src/api/prompts/api.ts +++ b/src/api/prompts/api.ts @@ -1,3 +1,4 @@ +import { TTLCache } from "../../cache"; import { Config } from "../../config"; import { Logger } from "../../logger"; import { PromptsHttpClient } from "./client"; @@ -6,17 +7,26 @@ import { GetPromptParams, PromptResponse } from "./models"; export class Prompts { private config: Config; private client: PromptsHttpClient; + private cache: TTLCache; constructor(config: Config) { this.config = config; this.client = new PromptsHttpClient(config); + this.cache = new TTLCache(config.cacheTtlSeconds); + } + + /** Clear all cached prompt entries. */ + clearCache(): void { + this.cache.clear(); } /** * Fetch prompt version by name and label. * - * @param params.name - Name of the prompt (required) - * @param params.label - Label of the prompt version (default: "production") + * @param params.name - Name of the prompt (required) + * @param params.label - Label of the prompt version (default: "production") + * @param params.useCache - When true, read/write the in-memory cache (default: false) + * @param params.cacheTtl - Per-call cache TTL in seconds (default: init cacheTtlSeconds) */ async getPrompt(params: GetPromptParams): Promise { if (!params || typeof params.name !== "string" || !params.name) { @@ -25,12 +35,28 @@ export class Prompts { } const label = params.label ?? "production"; + const useCache = params.useCache === true; + const cacheKey = `prompt:${params.name}:${label}`; + + if (useCache) { + const cached = this.cache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + } + const result = await this.client.getPromptVersion(params.name, label); if (!result) { return null; } - return result.data ?? null; + const data = result.data ?? null; + + if (data !== null && useCache) { + this.cache.set(cacheKey, data, params.cacheTtl); + } + + return data; } } diff --git a/src/api/prompts/models.ts b/src/api/prompts/models.ts index 1ec6cb3..ae9c1d8 100644 --- a/src/api/prompts/models.ts +++ b/src/api/prompts/models.ts @@ -5,6 +5,10 @@ export interface GetPromptParams { name: string; label?: string; + /** When true, serve from in-memory cache when available (default: false). */ + useCache?: boolean; + /** Per-call TTL in seconds; falls back to init `cacheTtlSeconds` when omitted. */ + cacheTtl?: number; } /** diff --git a/src/cache.test.ts b/src/cache.test.ts new file mode 100644 index 0000000..af71f5e --- /dev/null +++ b/src/cache.test.ts @@ -0,0 +1,55 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { TTLCache } from "./cache"; + +describe("TTLCache", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("get() returns undefined for missing key", () => { + const cache = new TTLCache(); + expect(cache.get("missing")).toBeUndefined(); + }); + + it("set() + get() returns stored value before TTL expires", () => { + const cache = new TTLCache(60); + cache.set("key", "value"); + expect(cache.get("key")).toBe("value"); + }); + + it("get() returns undefined after TTL expires", () => { + vi.useFakeTimers(); + const cache = new TTLCache(1); + cache.set("key", "value"); + vi.advanceTimersByTime(1001); + expect(cache.get("key")).toBeUndefined(); + }); + + it("per-entry ttl override expires independently of default", () => { + vi.useFakeTimers(); + const cache = new TTLCache(60); + cache.set("short", "a", 1); + cache.set("long", "b", 60); + vi.advanceTimersByTime(1001); + expect(cache.get("short")).toBeUndefined(); + expect(cache.get("long")).toBe("b"); + }); + + it("clear() removes all entries", () => { + const cache = new TTLCache(); + cache.set("a", "1"); + cache.set("b", "2"); + cache.clear(); + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("b")).toBeUndefined(); + }); + + it("invalidate(key) removes single entry", () => { + const cache = new TTLCache(); + cache.set("a", "1"); + cache.set("b", "2"); + cache.invalidate("a"); + expect(cache.get("a")).toBeUndefined(); + expect(cache.get("b")).toBe("2"); + }); +}); diff --git a/src/cache.ts b/src/cache.ts new file mode 100644 index 0000000..0bbf765 --- /dev/null +++ b/src/cache.ts @@ -0,0 +1,35 @@ +/** + * In-memory TTL cache for SDK read API responses. + */ + +export class TTLCache { + private store = new Map(); + private defaultTtl: number; + + constructor(defaultTtl = 60) { + this.defaultTtl = defaultTtl; + } + + get(key: string): T | undefined { + const entry = this.store.get(key); + if (!entry) return undefined; + if (performance.now() > entry.expiresAt) { + this.store.delete(key); + return undefined; + } + return entry.value; + } + + set(key: string, value: T, ttl?: number): void { + const ttlMs = (ttl ?? this.defaultTtl) * 1000; + this.store.set(key, { value, expiresAt: performance.now() + ttlMs }); + } + + invalidate(key: string): void { + this.store.delete(key); + } + + clear(): void { + this.store.clear(); + } +} diff --git a/src/config.ts b/src/config.ts index f297a7e..5c13e39 100644 --- a/src/config.ts +++ b/src/config.ts @@ -39,6 +39,8 @@ export interface NetraConfig { * instrumentations to produce root spans (legacy behaviour). */ rootInstruments?: Set; + /** Default TTL in seconds for opt-in SDK read caches (env: NETRA_CACHE_TTL_SECONDS). */ + cacheTtlSeconds?: number; } export enum NetraInstruments { @@ -155,6 +157,7 @@ export class Config { environment: string; resourceAttributes: Record; blockedSpans?: string[]; + cacheTtlSeconds: number; constructor(config: NetraConfig = {}) { this.appName = this._getAppName(config.appName); @@ -191,6 +194,11 @@ export class Config { config.resourceAttributes, ); this.blockedSpans = config.blockedSpans; + this.cacheTtlSeconds = this._getIntConfig( + config.cacheTtlSeconds, + "NETRA_CACHE_TTL_SECONDS", + 60, + ); this._validateApiKey(); this._setupAuthentication(); @@ -270,6 +278,24 @@ export class Config { } } + private _getIntConfig( + param: number | undefined, + envVar: string, + defaultValue: number, + ): number { + if (param !== undefined) { + return param; + } + + const envValue = process.env[envVar]; + if (envValue === undefined) { + return defaultValue; + } + + const parsed = parseInt(envValue, 10); + return Number.isNaN(parsed) ? defaultValue : parsed; + } + private _getBoolConfig( param: boolean | undefined, envVar: string, diff --git a/src/index.shutdown-cache.test.ts b/src/index.shutdown-cache.test.ts new file mode 100644 index 0000000..f34bbcb --- /dev/null +++ b/src/index.shutdown-cache.test.ts @@ -0,0 +1,43 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { PromptsHttpClient } from "./api/prompts/client"; + +vi.mock("./instrumentation", () => ({ + initInstrumentations: vi.fn(() => ({})), + instrumentationsReady: Promise.resolve(), + uninstrumentAll: vi.fn(() => Promise.resolve()), +})); + +import { Netra } from "./index"; + +describe("Netra.shutdown cache clearing", () => { + afterEach(async () => { + if (Netra.isInitialized()) { + await Netra.shutdown(); + } + }); + + it("clears prompts cache on shutdown so the next cached getPrompt hits HTTP", async () => { + await Netra.init({ cacheTtlSeconds: 60 }); + + const getPromptVersion = vi + .fn() + .mockResolvedValue({ data: { template: "v1" } }); + (Netra.prompts as unknown as { client: PromptsHttpClient }).client = { + getPromptVersion, + } as unknown as PromptsHttpClient; + + await Netra.prompts.getPrompt({ name: "my-prompt", useCache: true }); + await Netra.prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(1); + + await Netra.shutdown(); + + await Netra.init({ cacheTtlSeconds: 60 }); + (Netra.prompts as unknown as { client: PromptsHttpClient }).client = { + getPromptVersion, + } as unknown as PromptsHttpClient; + + await Netra.prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/index.ts b/src/index.ts index d9393f1..e3eaf1d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -160,6 +160,11 @@ export class Netra { return this._initialized; } + /** + * Initialize the Netra SDK. + * + * @param config.cacheTtlSeconds - Default TTL in seconds for opt-in read caches (default: 60, env: NETRA_CACHE_TTL_SECONDS) + */ static async init(config: NetraConfig = {}): Promise { if (this._initialized) { Logger.warn("Netra.init() called more than once; ignoring subsequent calls."); @@ -344,6 +349,12 @@ export class Netra { Logger.error("Error during Netra trace shutdown:", e); } + try { + this.prompts?.clearCache(); + } catch (e) { + Logger.error("Error clearing SDK API caches:", e); + } + this._initialized = false; this._tracer = undefined; } From 1cfd2452a31ff090c93da085fb05e70bb1f83963 Mon Sep 17 00:00:00 2001 From: Akhilesh Nair <72593014+AkhileshNair2201@users.noreply.github.com> Date: Mon, 13 Jul 2026 10:34:37 +0530 Subject: [PATCH 02/20] NET-1329 feat(models): add getModelPricing with opt-in TTL cache (#111) * feat: add opt-in TTL caching for getPrompt Co-authored-by: Cursor * fix: downgrade vitest for Node 18 and tighten cache tests Use Vitest 3.x so dev test runs align with engines.node >=18, and add getPrompt cacheTtl integration coverage plus shutdown re-init behavior. Co-authored-by: Cursor * feat(models): add getModelPricing with opt-in TTL cache * test(models): cover getModelPricing HTTP client failure paths Co-authored-by: Cursor * NET-1329 simplify cache TTL to module const and per-call override (#112) * NET-1329 simplify cache TTL to module const and per-call override Co-authored-by: Cursor * test: cover prompt default TTL and drop removed cacheTtlSeconds Assert PROMPT_CACHE_TTL_SECONDS expiry when cacheTtl is omitted, and stop passing the removed Config option in models client tests. Co-authored-by: Cursor --------- Co-authored-by: Cursor * docs: document Models API and expand caching changelog Co-authored-by: Cursor --------- Co-authored-by: Cursor --- CHANGELOG.md | 15 ++- README.md | 42 +++++++- src/api/index.ts | 10 +- src/api/models/api.test.ts | 136 +++++++++++++++++++++++++ src/api/models/api.ts | 54 ++++++++++ src/api/models/client.test.ts | 169 +++++++++++++++++++++++++++++++ src/api/models/client.ts | 49 +++++++++ src/api/models/index.ts | 13 +++ src/api/models/models.ts | 32 ++++++ src/api/prompts/api.test.ts | 21 +++- src/api/prompts/api.ts | 10 +- src/api/prompts/index.ts | 2 +- src/api/prompts/models.ts | 8 +- src/config.ts | 26 ----- src/index.shutdown-cache.test.ts | 36 ++++++- src/index.ts | 20 +++- 16 files changed, 598 insertions(+), 45 deletions(-) create mode 100644 src/api/models/api.test.ts create mode 100644 src/api/models/api.ts create mode 100644 src/api/models/client.test.ts create mode 100644 src/api/models/client.ts create mode 100644 src/api/models/index.ts create mode 100644 src/api/models/models.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ed104e..ad68004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Simulation Lifecycle Hooks**: Added prescript/postscript support for multi-turn simulations via `SimulationHooks` (`beforeAll`, `beforeEach`, `before`, `after`, `afterEach`, `afterAll`). Hooks can return setup context passed into `BaseTask.run`, and the run uses a two-phase initialize / first-turn flow so hooks execute before any LLM spend. Execution order is `beforeAll` → `beforeEach` → item-specific `before` → task → item-specific `after` → `afterEach` → `afterAll`. `beforeAll` failure aborts the run as `prescript_failed`; item `before` failure marks only that scenario; `after`, `afterEach`, and `afterAll` failures are logged and do not affect status. +## [Unreleased] + +### Added + +- **Opt-in prompt caching** — `Netra.prompts.getPrompt()` accepts `useCache` and `cacheTtl`. When `useCache` is true, responses are served from an in-memory TTL cache (default TTL: `PROMPT_CACHE_TTL_SECONDS` = 60). Caching is off by default. +- **Models API** — `Netra.models.getModelPricing()` fetches model pricing (optional `name` filter) with the same opt-in cache pattern (`useCache`, `cacheTtl`; default TTL: `MODEL_PRICING_CACHE_TTL_SECONDS` = 300). +- **Cache lifecycle** — `Netra.shutdown()` clears prompts and models in-memory caches. `clearCache()` is also available on each client. +- **Exported cache constants** — `PROMPT_CACHE_TTL_SECONDS` and `MODEL_PRICING_CACHE_TTL_SECONDS` are public exports. + +### Changed + +- **Prompt cache TTL** — Default TTL is the module constant `PROMPT_CACHE_TTL_SECONDS` (60). Override per call with `cacheTtl`. Removed unused `cacheTtlSeconds` init config and `NETRA_CACHE_TTL_SECONDS` env var. + ## [1.6.0] - 2026-07-17 ### Added @@ -124,7 +137,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **Context Propagation Helpers**: Exported `netraExpressMiddleware` and `runWithExtractedContext` for distributed tracing. These utilities extract incoming W3C Trace Context from HTTP headers and run code within that context, covering cases where auto-instrumentation is unavailable (ESM load-order issues, missing peer dependencies, or non-Express frameworks). +- **Context Propagation Helpers**: Exported `netraExpressMiddleware` and `runWithExtractedContext` for distributed tracing. These utilities extract incoming W3C Trace Context from HTTP headers and run code within that context, covering cases where auto-instrumentation is unavailable (ESM load-order issues, missing peer dependencies, or non-Express frameworks). ### Fixed diff --git a/README.md b/README.md index 079be41..8a2b4a3 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ - 🔧 **Multi-Provider Support**: Works with OpenAI, Google GenAI, Mistral, Anthropic, and more - 📈 **Session Management**: Track user sessions and custom attributes - 🌐 **Automatic Instrumentation**: Zero-code instrumentation for popular frameworks and libraries -- ⚡ **Opt-in Read Caching**: In-memory TTL caching for read-heavy SDK calls (e.g. prompt fetches) +- ⚡ **Opt-in Read Caching**: In-memory TTL caching for read-heavy SDK calls (`getPrompt`, `getModelPricing`) ## 📦 Installation @@ -189,20 +189,19 @@ async function generateContent(prompt: string) { Fetch prompt versions from Prompt Studio via `Netra.prompts.getPrompt()`. Caching is **opt-in per call** — omit `useCache` (or set it to `false`) to always hit the API. -Set a global default TTL at init with `cacheTtlSeconds` (default: **60** seconds), or override TTL for a single call with `cacheTtl`. +Default TTL is **60 seconds** (`PROMPT_CACHE_TTL_SECONDS`). Override TTL for a single call with `cacheTtl`. ```typescript import { Netra } from "netra-sdk"; await Netra.init({ appName: "my-ai-app", - cacheTtlSeconds: 60, // optional; env: NETRA_CACHE_TTL_SECONDS }); // Always fetches from the API (default) const prompt = await Netra.prompts.getPrompt({ name: "my-prompt" }); -// Cached for 60s (global default TTL) +// Cached for 60s (default TTL) const cached = await Netra.prompts.getPrompt({ name: "my-prompt", useCache: true, @@ -219,6 +218,40 @@ const shortLived = await Netra.prompts.getPrompt({ > **Note**: Cached prompts may be stale for up to the TTL after dashboard edits. Use `useCache: false` when you need the latest version immediately. `Netra.shutdown()` clears in-memory caches. +## 💰 Models API + +Fetch model pricing via `Netra.models.getModelPricing()`. Caching is **opt-in per call** — omit `useCache` (or set it to `false`) to always hit the API. + +Default TTL is **300 seconds** (`MODEL_PRICING_CACHE_TTL_SECONDS`). Override TTL for a single call with `cacheTtl`. + +```typescript +import { Netra } from "netra-sdk"; + +await Netra.init({ + appName: "my-ai-app", +}); + +// Always fetches from the API (default) +const pricing = await Netra.models.getModelPricing(); + +// Optional name filter +const gptPricing = await Netra.models.getModelPricing({ name: "gpt-4o" }); + +// Cached for 300s (default TTL) +const cached = await Netra.models.getModelPricing({ + useCache: true, +}); + +// Cached for 60s for this call only +const shortLived = await Netra.models.getModelPricing({ + name: "gpt-4o", + useCache: true, + cacheTtl: 60, +}); +``` + +> **Note**: Cached pricing may be stale for up to the TTL after dashboard edits. Use `useCache: false` when you need the latest values immediately. `Netra.shutdown()` clears in-memory caches. + ## 🔧 Environment Variables You can configure the SDK using environment variables: @@ -229,7 +262,6 @@ You can configure the SDK using environment variables: | `NETRA_APP_NAME` | Name of your application | | `NETRA_ENV` | Environment (e.g., prod, dev) | | `NETRA_TRACE_CONTENT` | Capture prompt/completion content (default: true) | -| `NETRA_CACHE_TTL_SECONDS` | Default TTL in seconds for opt-in SDK read caches (default: 60) | ## 🤝 License diff --git a/src/api/index.ts b/src/api/index.ts index 6dc1451..249df25 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -66,5 +66,13 @@ export type { } from "./dashboard"; // Prompts API -export { Prompts } from "./prompts"; +export { Prompts, PROMPT_CACHE_TTL_SECONDS } from "./prompts"; export type { GetPromptParams, PromptResponse } from "./prompts"; + +// Models API +export { Models, MODEL_PRICING_CACHE_TTL_SECONDS } from "./models"; +export type { + GetModelPricingParams, + ModelPrice, + ModelPricing, +} from "./models"; diff --git a/src/api/models/api.test.ts b/src/api/models/api.test.ts new file mode 100644 index 0000000..3acf8ff --- /dev/null +++ b/src/api/models/api.test.ts @@ -0,0 +1,136 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Config } from "../../config"; +import { Models } from "./api"; +import { ModelsHttpClient } from "./client"; +import { ModelPricing } from "./models"; + +const samplePricing: ModelPricing[] = [ + { + name: "gpt-4", + projectId: null, + matchPattern: "gpt-4*", + prices: [ + { + usageType: "input", + minUnits: 0, + maxUnits: 1000, + price: 0.03, + unitValue: 1000, + }, + ], + }, +]; + +describe("Models.getModelPricing caching", () => { + let models: Models; + let getModelPricing: ReturnType; + + beforeEach(() => { + const config = new Config(); + models = new Models(config); + getModelPricing = vi.fn(); + (models as unknown as { client: ModelsHttpClient }).client = { + getModelPricing, + } as unknown as ModelsHttpClient; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("calls HTTP on every request when useCache is omitted", async () => { + getModelPricing.mockResolvedValue(samplePricing); + + await models.getModelPricing(); + await models.getModelPricing(); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); + + it("serves from cache on second call with same name when useCache is true", async () => { + getModelPricing.mockResolvedValue(samplePricing); + + const first = await models.getModelPricing({ + name: "gpt-4", + useCache: true, + }); + const second = await models.getModelPricing({ + name: "gpt-4", + useCache: true, + }); + + expect(getModelPricing).toHaveBeenCalledTimes(1); + expect(getModelPricing).toHaveBeenCalledWith("gpt-4"); + expect(first).toEqual(samplePricing); + expect(second).toEqual(samplePricing); + }); + + it("keeps separate cache entries for different name and all", async () => { + const allPricing: ModelPricing[] = []; + getModelPricing + .mockResolvedValueOnce(samplePricing) + .mockResolvedValueOnce(allPricing); + + const named = await models.getModelPricing({ + name: "gpt-4", + useCache: true, + }); + const all = await models.getModelPricing({ useCache: true }); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + expect(getModelPricing).toHaveBeenNthCalledWith(1, "gpt-4"); + expect(getModelPricing).toHaveBeenNthCalledWith(2, undefined); + expect(named).toEqual(samplePricing); + expect(all).toEqual(allPricing); + }); + + it("does not cache null API responses", async () => { + getModelPricing.mockResolvedValue(null); + + await models.getModelPricing({ useCache: true }); + await models.getModelPricing({ useCache: true }); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); + + it("caches empty arrays as successful responses", async () => { + getModelPricing.mockResolvedValue([]); + + await models.getModelPricing({ useCache: true }); + await models.getModelPricing({ useCache: true }); + + expect(getModelPricing).toHaveBeenCalledTimes(1); + }); + + it("ignores cache when useCache is false even if cacheTtl is set", async () => { + getModelPricing.mockResolvedValue(samplePricing); + + await models.getModelPricing({ useCache: false, cacheTtl: 30 }); + await models.getModelPricing({ useCache: false, cacheTtl: 30 }); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); + + it("expires per-call cacheTtl before the models default TTL", async () => { + vi.useFakeTimers(); + getModelPricing.mockResolvedValue(samplePricing); + + await models.getModelPricing({ useCache: true, cacheTtl: 1 }); + expect(getModelPricing).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(1001); + + await models.getModelPricing({ useCache: true, cacheTtl: 1 }); + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); + + it("hits HTTP again after clearCache when useCache is true", async () => { + getModelPricing.mockResolvedValue(samplePricing); + + await models.getModelPricing({ useCache: true }); + models.clearCache(); + await models.getModelPricing({ useCache: true }); + + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/api/models/api.ts b/src/api/models/api.ts new file mode 100644 index 0000000..f7ee692 --- /dev/null +++ b/src/api/models/api.ts @@ -0,0 +1,54 @@ +import { TTLCache } from "../../cache"; +import { Config } from "../../config"; +import { ModelsHttpClient } from "./client"; +import { + GetModelPricingParams, + MODEL_PRICING_CACHE_TTL_SECONDS, + ModelPricing, +} from "./models"; + +export class Models { + private config: Config; + private client: ModelsHttpClient; + private cache: TTLCache; + + constructor(config: Config) { + this.config = config; + this.client = new ModelsHttpClient(config); + this.cache = new TTLCache(MODEL_PRICING_CACHE_TTL_SECONDS); + } + + /** Clear all cached model pricing entries. */ + clearCache(): void { + this.cache.clear(); + } + + /** + * Fetch model pricing from the backend. + * + * @param params.name - Optional model name filter + * @param params.useCache - When true, read/write the in-memory cache (default: false) + * @param params.cacheTtl - Per-call cache TTL in seconds (default: MODEL_PRICING_CACHE_TTL_SECONDS) + */ + async getModelPricing( + params: GetModelPricingParams = {}, + ): Promise { + const useCache = params.useCache === true; + const cacheKey = `model:pricing:${params.name ?? "all"}`; + + if (useCache) { + const cached = this.cache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + } + + const data = await this.client.getModelPricing(params.name); + + if (data !== null && useCache) { + this.cache.set(cacheKey, data, params.cacheTtl); + } + + return data; + } +} diff --git a/src/api/models/client.test.ts b/src/api/models/client.test.ts new file mode 100644 index 0000000..3fe917d --- /dev/null +++ b/src/api/models/client.test.ts @@ -0,0 +1,169 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { Config } from "../../config"; +import { Logger } from "../../logger"; +import { ModelsHttpClient } from "./client"; +import { ModelPricing } from "./models"; + +const samplePricing: ModelPricing[] = [ + { + name: "gpt-4", + projectId: null, + matchPattern: "gpt-4*", + prices: [ + { + usageType: "input", + minUnits: 0, + maxUnits: 1000, + price: 0.03, + unitValue: 1000, + }, + ], + }, +]; + +describe("ModelsHttpClient.getModelPricing", () => { + let client: ModelsHttpClient; + let get: ReturnType; + let isInitialized: ReturnType; + let logError: ReturnType; + + beforeEach(() => { + client = new ModelsHttpClient(new Config()); + get = vi.spyOn(client, "get"); + isInitialized = vi.spyOn(client, "isInitialized").mockReturnValue(true); + logError = vi.spyOn(Logger, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe("when the request succeeds", () => { + it("returns the pricing array from an enveloped API response", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: samplePricing }, + }); + + const result = await client.getModelPricing(); + + expect(result).toEqual(samplePricing); + expect(get).toHaveBeenCalledWith("/sdk/models", undefined); + }); + + it("calls GET /sdk/models with the name query param when name is provided", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: samplePricing }, + }); + + await client.getModelPricing("gpt-4"); + + expect(get).toHaveBeenCalledWith("/sdk/models", { name: "gpt-4" }); + }); + + it("calls GET /sdk/models with no query params when name is omitted", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: samplePricing }, + }); + + await client.getModelPricing(); + + expect(get).toHaveBeenCalledWith("/sdk/models", undefined); + }); + + it("returns an empty array when the API returns an empty list", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: [] }, + }); + + const result = await client.getModelPricing(); + + expect(result).toEqual([]); + expect(logError).not.toHaveBeenCalled(); + }); + }); + + describe("when the client is not initialized", () => { + it("returns null without calling GET", async () => { + isInitialized.mockReturnValue(false); + + const result = await client.getModelPricing("gpt-4"); + + expect(result).toBeNull(); + expect(get).not.toHaveBeenCalled(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Models client is not initialized; cannot fetch model pricing", + ); + }); + }); + + describe("when the HTTP response fails", () => { + it("returns null and logs the API error message when response is not ok", async () => { + get.mockResolvedValue({ + ok: false, + status: 500, + data: { error: { message: "backend unavailable" } }, + }); + + const result = await client.getModelPricing(); + + expect(result).toBeNull(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Failed to fetch model pricing: backend unavailable", + ); + }); + + it("returns null and logs a generic message when the error payload has no message", async () => { + get.mockResolvedValue({ + ok: false, + status: 502, + data: {}, + }); + + const result = await client.getModelPricing(); + + expect(result).toBeNull(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Failed to fetch model pricing: Unknown error", + ); + }); + }); + + describe("when the response payload is unexpected", () => { + it("returns null and logs when the unwrapped payload is not an array", async () => { + get.mockResolvedValue({ + ok: true, + status: 200, + data: { data: { name: "not-an-array" } }, + }); + + const result = await client.getModelPricing(); + + expect(result).toBeNull(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Unexpected response format; expected an array", + ); + }); + }); + + describe("when GET throws", () => { + it("returns null and logs the thrown error message", async () => { + get.mockRejectedValue(new Error("network down")); + + const result = await client.getModelPricing(); + + expect(result).toBeNull(); + expect(logError).toHaveBeenCalledWith( + "netra.models: Failed to fetch model pricing:", + "network down", + ); + }); + }); +}); diff --git a/src/api/models/client.ts b/src/api/models/client.ts new file mode 100644 index 0000000..9dfa4d3 --- /dev/null +++ b/src/api/models/client.ts @@ -0,0 +1,49 @@ +/** + * Internal HTTP client for Models APIs + */ + +import { Config } from "../../config"; +import { Logger } from "../../logger"; +import { NetraHttpClient } from "../http-client"; +import { ModelPricing } from "./models"; + +export class ModelsHttpClient extends NetraHttpClient { + constructor(config: Config) { + super(config, "NETRA_MODELS_TIMEOUT", 10.0); + } + + async getModelPricing(name?: string): Promise { + if (!this.isInitialized()) { + Logger.error( + "netra.models: Models client is not initialized; cannot fetch model pricing", + ); + return null; + } + + try { + const params = name ? { name } : undefined; + const response = await this.get("/sdk/models", params); + + if (!response.ok) { + const errorMessage = response.data?.error?.message ?? "Unknown error"; + Logger.error( + `netra.models: Failed to fetch model pricing: ${errorMessage}`, + ); + return null; + } + + const items = this.extractData(response, null); + if (items !== null && !Array.isArray(items)) { + Logger.error( + "netra.models: Unexpected response format; expected an array", + ); + return null; + } + return items; + } catch (err: any) { + const message = err?.response?.data?.error?.message ?? err?.message ?? ""; + Logger.error("netra.models: Failed to fetch model pricing:", message); + return null; + } + } +} diff --git a/src/api/models/index.ts b/src/api/models/index.ts new file mode 100644 index 0000000..138c4b9 --- /dev/null +++ b/src/api/models/index.ts @@ -0,0 +1,13 @@ +/** + * Models API exports + */ + +export { Models } from "./api"; + +export type { + GetModelPricingParams, + ModelPrice, + ModelPricing, +} from "./models"; + +export { MODEL_PRICING_CACHE_TTL_SECONDS } from "./models"; diff --git a/src/api/models/models.ts b/src/api/models/models.ts new file mode 100644 index 0000000..4ccf3a8 --- /dev/null +++ b/src/api/models/models.ts @@ -0,0 +1,32 @@ +/** + * Models API Models + */ + +/** Default TTL (seconds) for model pricing cache when useCache is true and cacheTtl is omitted. */ +export const MODEL_PRICING_CACHE_TTL_SECONDS = 300; + +export interface ModelPrice { + usageType: string; + minUnits: number; + maxUnits: number; + price: number; + unitValue: number; +} + +export interface ModelPricing { + name: string; + projectId: string | null; + matchPattern: string; + prices: ModelPrice[]; +} + +export interface GetModelPricingParams { + name?: string; + /** When true, read/write in-memory cache (default: false). */ + useCache?: boolean; + /** + * Per-call TTL in seconds. + * When omitted with useCache: true, uses MODEL_PRICING_CACHE_TTL_SECONDS (300). + */ + cacheTtl?: number; +} diff --git a/src/api/prompts/api.test.ts b/src/api/prompts/api.test.ts index 9044eb5..75294de 100644 --- a/src/api/prompts/api.test.ts +++ b/src/api/prompts/api.test.ts @@ -2,13 +2,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { Config } from "../../config"; import { Prompts } from "./api"; import { PromptsHttpClient } from "./client"; +import { PROMPT_CACHE_TTL_SECONDS } from "./models"; describe("Prompts.getPrompt caching", () => { let prompts: Prompts; let getPromptVersion: ReturnType; beforeEach(() => { - const config = new Config({ cacheTtlSeconds: 60 }); + const config = new Config(); prompts = new Prompts(config); getPromptVersion = vi.fn(); (prompts as unknown as { client: PromptsHttpClient }).client = { @@ -87,7 +88,7 @@ describe("Prompts.getPrompt caching", () => { expect(getPromptVersion).toHaveBeenCalledTimes(2); }); - it("expires per-call cacheTtl before the global default TTL", async () => { + it("expires per-call cacheTtl before the module default TTL", async () => { vi.useFakeTimers(); getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); @@ -108,6 +109,22 @@ describe("Prompts.getPrompt caching", () => { expect(getPromptVersion).toHaveBeenCalledTimes(2); }); + it("expires cached prompts after PROMPT_CACHE_TTL_SECONDS when cacheTtl is omitted", async () => { + vi.useFakeTimers(); + getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); + + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(PROMPT_CACHE_TTL_SECONDS * 1000 - 1); + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(2); + await prompts.getPrompt({ name: "my-prompt", useCache: true }); + expect(getPromptVersion).toHaveBeenCalledTimes(2); + }); + it("hits HTTP again after clearCache when useCache is true", async () => { getPromptVersion.mockResolvedValue({ data: { template: "v1" } }); diff --git a/src/api/prompts/api.ts b/src/api/prompts/api.ts index 90c3423..3d8ca6d 100644 --- a/src/api/prompts/api.ts +++ b/src/api/prompts/api.ts @@ -2,7 +2,11 @@ import { TTLCache } from "../../cache"; import { Config } from "../../config"; import { Logger } from "../../logger"; import { PromptsHttpClient } from "./client"; -import { GetPromptParams, PromptResponse } from "./models"; +import { + GetPromptParams, + PROMPT_CACHE_TTL_SECONDS, + PromptResponse, +} from "./models"; export class Prompts { private config: Config; @@ -12,7 +16,7 @@ export class Prompts { constructor(config: Config) { this.config = config; this.client = new PromptsHttpClient(config); - this.cache = new TTLCache(config.cacheTtlSeconds); + this.cache = new TTLCache(PROMPT_CACHE_TTL_SECONDS); } /** Clear all cached prompt entries. */ @@ -26,7 +30,7 @@ export class Prompts { * @param params.name - Name of the prompt (required) * @param params.label - Label of the prompt version (default: "production") * @param params.useCache - When true, read/write the in-memory cache (default: false) - * @param params.cacheTtl - Per-call cache TTL in seconds (default: init cacheTtlSeconds) + * @param params.cacheTtl - Per-call cache TTL in seconds (default: PROMPT_CACHE_TTL_SECONDS) */ async getPrompt(params: GetPromptParams): Promise { if (!params || typeof params.name !== "string" || !params.name) { diff --git a/src/api/prompts/index.ts b/src/api/prompts/index.ts index bedb08f..0847ecf 100644 --- a/src/api/prompts/index.ts +++ b/src/api/prompts/index.ts @@ -3,5 +3,5 @@ */ export { Prompts } from "./api"; - +export { PROMPT_CACHE_TTL_SECONDS } from "./models"; export type { GetPromptParams, PromptResponse } from "./models"; diff --git a/src/api/prompts/models.ts b/src/api/prompts/models.ts index ae9c1d8..50ed7cc 100644 --- a/src/api/prompts/models.ts +++ b/src/api/prompts/models.ts @@ -2,12 +2,18 @@ * Prompts API Models */ +/** Default TTL (seconds) for prompt cache when useCache is true and cacheTtl is omitted. */ +export const PROMPT_CACHE_TTL_SECONDS = 60; + export interface GetPromptParams { name: string; label?: string; /** When true, serve from in-memory cache when available (default: false). */ useCache?: boolean; - /** Per-call TTL in seconds; falls back to init `cacheTtlSeconds` when omitted. */ + /** + * Per-call TTL in seconds. + * When omitted with useCache: true, uses PROMPT_CACHE_TTL_SECONDS (60). + */ cacheTtl?: number; } diff --git a/src/config.ts b/src/config.ts index 5c13e39..f297a7e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -39,8 +39,6 @@ export interface NetraConfig { * instrumentations to produce root spans (legacy behaviour). */ rootInstruments?: Set; - /** Default TTL in seconds for opt-in SDK read caches (env: NETRA_CACHE_TTL_SECONDS). */ - cacheTtlSeconds?: number; } export enum NetraInstruments { @@ -157,7 +155,6 @@ export class Config { environment: string; resourceAttributes: Record; blockedSpans?: string[]; - cacheTtlSeconds: number; constructor(config: NetraConfig = {}) { this.appName = this._getAppName(config.appName); @@ -194,11 +191,6 @@ export class Config { config.resourceAttributes, ); this.blockedSpans = config.blockedSpans; - this.cacheTtlSeconds = this._getIntConfig( - config.cacheTtlSeconds, - "NETRA_CACHE_TTL_SECONDS", - 60, - ); this._validateApiKey(); this._setupAuthentication(); @@ -278,24 +270,6 @@ export class Config { } } - private _getIntConfig( - param: number | undefined, - envVar: string, - defaultValue: number, - ): number { - if (param !== undefined) { - return param; - } - - const envValue = process.env[envVar]; - if (envValue === undefined) { - return defaultValue; - } - - const parsed = parseInt(envValue, 10); - return Number.isNaN(parsed) ? defaultValue : parsed; - } - private _getBoolConfig( param: boolean | undefined, envVar: string, diff --git a/src/index.shutdown-cache.test.ts b/src/index.shutdown-cache.test.ts index f34bbcb..ed9076e 100644 --- a/src/index.shutdown-cache.test.ts +++ b/src/index.shutdown-cache.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { ModelsHttpClient } from "./api/models/client"; import { PromptsHttpClient } from "./api/prompts/client"; vi.mock("./instrumentation", () => ({ @@ -17,7 +18,7 @@ describe("Netra.shutdown cache clearing", () => { }); it("clears prompts cache on shutdown so the next cached getPrompt hits HTTP", async () => { - await Netra.init({ cacheTtlSeconds: 60 }); + await Netra.init({}); const getPromptVersion = vi .fn() @@ -32,7 +33,7 @@ describe("Netra.shutdown cache clearing", () => { await Netra.shutdown(); - await Netra.init({ cacheTtlSeconds: 60 }); + await Netra.init({}); (Netra.prompts as unknown as { client: PromptsHttpClient }).client = { getPromptVersion, } as unknown as PromptsHttpClient; @@ -40,4 +41,35 @@ describe("Netra.shutdown cache clearing", () => { await Netra.prompts.getPrompt({ name: "my-prompt", useCache: true }); expect(getPromptVersion).toHaveBeenCalledTimes(2); }); + + it("clears models cache on shutdown so the next cached getModelPricing hits HTTP", async () => { + await Netra.init({}); + + const pricing = [ + { + name: "gpt-4", + projectId: null, + matchPattern: "gpt-4*", + prices: [], + }, + ]; + const getModelPricing = vi.fn().mockResolvedValue(pricing); + (Netra.models as unknown as { client: ModelsHttpClient }).client = { + getModelPricing, + } as unknown as ModelsHttpClient; + + await Netra.models.getModelPricing({ useCache: true }); + await Netra.models.getModelPricing({ useCache: true }); + expect(getModelPricing).toHaveBeenCalledTimes(1); + + await Netra.shutdown(); + + await Netra.init({}); + (Netra.models as unknown as { client: ModelsHttpClient }).client = { + getModelPricing, + } as unknown as ModelsHttpClient; + + await Netra.models.getModelPricing({ useCache: true }); + expect(getModelPricing).toHaveBeenCalledTimes(2); + }); }); diff --git a/src/index.ts b/src/index.ts index e3eaf1d..d7506ee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,7 @@ import { context, Span, SpanKind, trace } from "@opentelemetry/api"; import { createRequire } from "module"; -import { Prompts, Dashboard, Evaluation, Usage } from "./api"; +import { Prompts, Dashboard, Evaluation, Usage, Models } from "./api"; import { Config, NetraConfig } from "./config"; import { initInstrumentations, instrumentationsReady, uninstrumentAll } from "./instrumentation"; import { Logger } from "./logger"; @@ -68,6 +68,10 @@ export { Usage, // Prompts API Prompts, + PROMPT_CACHE_TTL_SECONDS, + // Models API + Models, + MODEL_PRICING_CACHE_TTL_SECONDS, } from "./api"; export type { @@ -107,6 +111,10 @@ export type { TraceSummary, GetPromptParams, PromptResponse, + // Models API + GetModelPricingParams, + ModelPrice, + ModelPricing, } from "./api"; // Export simulation types and classes @@ -148,6 +156,7 @@ export class Netra { static dashboard: Dashboard; static simulation: Simulation; static prompts: Prompts; + static models: Models; static getConfig(): Config { if (!this._config) { @@ -162,8 +171,6 @@ export class Netra { /** * Initialize the Netra SDK. - * - * @param config.cacheTtlSeconds - Default TTL in seconds for opt-in read caches (default: 60, env: NETRA_CACHE_TTL_SECONDS) */ static async init(config: NetraConfig = {}): Promise { if (this._initialized) { @@ -219,6 +226,12 @@ export class Netra { Logger.warn("Netra: failed to initialize prompts client:", e); } + try { + this.models = new Models(cfg); + } catch (e) { + Logger.warn("Netra: failed to initialize models client:", e); + } + this._initialized = true; Logger.info("Netra successfully initialized."); @@ -351,6 +364,7 @@ export class Netra { try { this.prompts?.clearCache(); + this.models?.clearCache(); } catch (e) { Logger.error("Error clearing SDK API caches:", e); } From 218b50e7265f8b06566454581241c136d8735a10 Mon Sep 17 00:00:00 2001 From: akash-vijay-kv Date: Mon, 13 Jul 2026 11:08:56 +0530 Subject: [PATCH 03/20] chore: Update CHANGELOG with all latest changes --- CHANGELOG.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad68004..85720e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,55 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Prompt cache TTL** — Default TTL is the module constant `PROMPT_CACHE_TTL_SECONDS` (60). Override per call with `cacheTtl`. Removed unused `cacheTtlSeconds` init config and `NETRA_CACHE_TTL_SECONDS` env var. +## [1.5.0] - 2026-07-10 + +### Added + +- **Google GenAI Instrumentation**: Added instrumentation support for the Google GenAI (`@google/genai`) SDK, with a shared `base-instrumentor` abstraction and a separate `google-generative-ai` module. + +### Fixed + +- **OpenAI Agents Cache Tokens**: Captured cache token details (`cached_tokens`, `cache_write_tokens`, `reasoning_tokens`) for Claude models in OpenAI Agents instrumentation. +- **Raw Chat Completions Response**: Added a helper to handle usage from the Chat Completions API naming (`prompt_tokens`/`completion_tokens` and their token details) alongside the Responses API naming, so `OpenAIChatCompletionsModel` usage is captured correctly. +- **Span Input/Output on Active and Root Spans**: Updated utility functions so input and output attributes are set correctly on both the active span and the root span. + +## [1.4.0] - 2026-07-02 + +### Added + +- **Anthropic Tool Runner Support**: Added span capture for the Anthropic tool-runner flow, so tool-execution turns within an agentic Anthropic run are traced. +- **Attribute Size Limit Processor**: New `AttributeSizeLimitProcessor` enforces a hard per-attribute size cap (default 32KB, configurable via `NETRA_SPAN_ATTRIBUTE_MAX_SIZE`) by wrapping `setAttribute` on span start, preventing "entity too large" errors during export. +- **Resource Attributes Propagation**: `Config` now sets `OTEL_RESOURCE_ATTRIBUTES` so the `TracerProvider` resource carries `service.name` and `deployment.environment`, with precedence order: pre-existing env value > config `resourceAttributes` > Netra defaults. + +### Changed + +- **SDK Version Source**: `Config.LIBRARY_VERSION` now derives from `SDK_VERSION` in `src/version.ts` instead of a hardcoded string, keeping the reported library version in sync with the package version. + +## [1.3.0] - 2026-07-01 + +### Added + +- **NativeTracingMode**: Replaced the boolean `disableNativeTracing` option with a `NativeTracingMode` (`"both" | "netra" | "netra-strict"`) for granular control over where OpenAI Agents traces are sent. Configurable per-instrument or via the `NATIVE_TRACING_MODE` environment variable. `"netra"` routes traces to Netra only (falling back to additive mode with a warning if SDK APIs are unavailable), `"netra-strict"` (default) never falls back to native, and `"both"` sends to Netra and native. + +### Fixed + +- **Netra-only OpenAI Agents Tracing**: Added `canSet` gating and a restore fallback so the OpenAI Agents processor can be swapped safely, and included `OPENAI_AGENTS` in the root-span instrument allowlist (`DEFAULT_INSTRUMENTS_FOR_ROOT`) so agent runs are captured as root spans. + +## [1.2.0] - 2026-06-23 + +### Added + +- **Simulation File Handling**: The simulation workflow now supports file attachments on user messages. The SDK parses `attachments` metadata from the API (`FileData`), downloads the referenced content via pre-signed URLs, and delivers it to user tasks as base64-encoded `ProcessedFile` objects. `ProcessedFile` is now exported from the package root. + +## [1.1.0] - 2026-06-16 + +### Added + +- **Context Propagation Helpers**: Exported `netraExpressMiddleware` and `runWithExtractedContext` for distributed tracing. These utilities extract incoming W3C Trace Context from HTTP headers and run code within that context, covering cases where auto-instrumentation is unavailable (ESM load-order issues, missing peer dependencies, or non-Express frameworks). + +### Fixed + +- **CommonJS (CJS) Support for LLM Provider Instrumentations**: Fixed instrumentation for Anthropic, Google GenAI, Groq, and Mistral AI so they patch correctly under CommonJS module resolution, resolving instrumentation failures in CJS applications. ## [1.6.0] - 2026-07-17 From 54d08a6861cf15cf3598027661d083a4aaa3ed9c Mon Sep 17 00:00:00 2001 From: Akhilesh Nair <72593014+AkhileshNair2201@users.noreply.github.com> Date: Mon, 13 Jul 2026 10:34:37 +0530 Subject: [PATCH 04/20] NET-1329 feat(models): add getModelPricing with opt-in TTL cache (#111) * feat: add opt-in TTL caching for getPrompt Co-authored-by: Cursor * fix: downgrade vitest for Node 18 and tighten cache tests Use Vitest 3.x so dev test runs align with engines.node >=18, and add getPrompt cacheTtl integration coverage plus shutdown re-init behavior. Co-authored-by: Cursor * feat(models): add getModelPricing with opt-in TTL cache * test(models): cover getModelPricing HTTP client failure paths Co-authored-by: Cursor * NET-1329 simplify cache TTL to module const and per-call override (#112) * NET-1329 simplify cache TTL to module const and per-call override Co-authored-by: Cursor * test: cover prompt default TTL and drop removed cacheTtlSeconds Assert PROMPT_CACHE_TTL_SECONDS expiry when cacheTtl is omitted, and stop passing the removed Config option in models client tests. Co-authored-by: Cursor --------- Co-authored-by: Cursor * docs: document Models API and expand caching changelog Co-authored-by: Cursor --------- Co-authored-by: Cursor --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85720e6..ea9f666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Prompt cache TTL** — Default TTL is the module constant `PROMPT_CACHE_TTL_SECONDS` (60). Override per call with `cacheTtl`. Removed unused `cacheTtlSeconds` init config and `NETRA_CACHE_TTL_SECONDS` env var. + ## [1.5.0] - 2026-07-10 ### Added From 852f0a123905a08d319d19ea5604a6106cd77769 Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Fri, 17 Jul 2026 10:15:53 +0530 Subject: [PATCH 05/20] [NET-1333] fix: Standardize span attribute serialization and fix addConversation persistence (#114) --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea9f666..8e5d921 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Prompt cache TTL** — Default TTL is the module constant `PROMPT_CACHE_TTL_SECONDS` (60). Override per call with `cacheTtl`. Removed unused `cacheTtlSeconds` init config and `NETRA_CACHE_TTL_SECONDS` env var. +## [1.6.0] - 2026-07-14 + +### Added + +- **SerializationSpanProcessor** — New span processor that serializes and truncates span attributes at write time via `setAttribute` wrapping. Uses `jsonrepair`-based truncation for JSON values and `...[TRUNCATED]` suffix for plain strings. + +### Changed + +- **Span processor registration order** — `ScrubbingSpanProcessor` is now registered before `SerializationSpanProcessor` so that serialization executes first in the `setAttribute` wrapper chain, converting values to strings before the scrubber runs its regex-based redaction. +- **Lazy env var loading** — `Config.SPAN_ATTRIBUTE_MAX_SIZE` is now a lazy getter that reads `process.env` on first access rather than at import time, so environment variables set before `Netra.init()` are always respected. Removed the `dotenv` dependency — env loading is the application's responsibility. + +### Fixed + +- **Conversation accumulation** — `addConversation` now correctly reads existing entries from `span.attributes` instead of the internal `_attributes` property, so all conversation entries are persisted rather than only the last one. +- **Error data truncation** — `span.setStatus({ message })` in the OpenAI Agents processor now enforces a 4 KB limit on error data using `safeStringify`, preventing unbounded error messages. + ## [1.5.0] - 2026-07-10 ### Added From ad57b52fd5dd35c86ad130004d0857324f3cf76d Mon Sep 17 00:00:00 2001 From: Akash Vijay Date: Thu, 23 Jul 2026 10:01:17 +0530 Subject: [PATCH 06/20] [NET-1386] feat: Add label on the root span for evaluation/simulation traces (#121) --- CHANGELOG.md | 66 --------------------------------------- src/api/evaluation/api.ts | 6 +++- src/config.ts | 5 +++ src/simulation/api.ts | 6 +++- 4 files changed, 15 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e5d921..ad68004 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,72 +30,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Prompt cache TTL** — Default TTL is the module constant `PROMPT_CACHE_TTL_SECONDS` (60). Override per call with `cacheTtl`. Removed unused `cacheTtlSeconds` init config and `NETRA_CACHE_TTL_SECONDS` env var. -## [1.6.0] - 2026-07-14 - -### Added - -- **SerializationSpanProcessor** — New span processor that serializes and truncates span attributes at write time via `setAttribute` wrapping. Uses `jsonrepair`-based truncation for JSON values and `...[TRUNCATED]` suffix for plain strings. - -### Changed - -- **Span processor registration order** — `ScrubbingSpanProcessor` is now registered before `SerializationSpanProcessor` so that serialization executes first in the `setAttribute` wrapper chain, converting values to strings before the scrubber runs its regex-based redaction. -- **Lazy env var loading** — `Config.SPAN_ATTRIBUTE_MAX_SIZE` is now a lazy getter that reads `process.env` on first access rather than at import time, so environment variables set before `Netra.init()` are always respected. Removed the `dotenv` dependency — env loading is the application's responsibility. - -### Fixed - -- **Conversation accumulation** — `addConversation` now correctly reads existing entries from `span.attributes` instead of the internal `_attributes` property, so all conversation entries are persisted rather than only the last one. -- **Error data truncation** — `span.setStatus({ message })` in the OpenAI Agents processor now enforces a 4 KB limit on error data using `safeStringify`, preventing unbounded error messages. - -## [1.5.0] - 2026-07-10 - -### Added - -- **Google GenAI Instrumentation**: Added instrumentation support for the Google GenAI (`@google/genai`) SDK, with a shared `base-instrumentor` abstraction and a separate `google-generative-ai` module. - -### Fixed - -- **OpenAI Agents Cache Tokens**: Captured cache token details (`cached_tokens`, `cache_write_tokens`, `reasoning_tokens`) for Claude models in OpenAI Agents instrumentation. -- **Raw Chat Completions Response**: Added a helper to handle usage from the Chat Completions API naming (`prompt_tokens`/`completion_tokens` and their token details) alongside the Responses API naming, so `OpenAIChatCompletionsModel` usage is captured correctly. -- **Span Input/Output on Active and Root Spans**: Updated utility functions so input and output attributes are set correctly on both the active span and the root span. - -## [1.4.0] - 2026-07-02 - -### Added - -- **Anthropic Tool Runner Support**: Added span capture for the Anthropic tool-runner flow, so tool-execution turns within an agentic Anthropic run are traced. -- **Attribute Size Limit Processor**: New `AttributeSizeLimitProcessor` enforces a hard per-attribute size cap (default 32KB, configurable via `NETRA_SPAN_ATTRIBUTE_MAX_SIZE`) by wrapping `setAttribute` on span start, preventing "entity too large" errors during export. -- **Resource Attributes Propagation**: `Config` now sets `OTEL_RESOURCE_ATTRIBUTES` so the `TracerProvider` resource carries `service.name` and `deployment.environment`, with precedence order: pre-existing env value > config `resourceAttributes` > Netra defaults. - -### Changed - -- **SDK Version Source**: `Config.LIBRARY_VERSION` now derives from `SDK_VERSION` in `src/version.ts` instead of a hardcoded string, keeping the reported library version in sync with the package version. - -## [1.3.0] - 2026-07-01 - -### Added - -- **NativeTracingMode**: Replaced the boolean `disableNativeTracing` option with a `NativeTracingMode` (`"both" | "netra" | "netra-strict"`) for granular control over where OpenAI Agents traces are sent. Configurable per-instrument or via the `NATIVE_TRACING_MODE` environment variable. `"netra"` routes traces to Netra only (falling back to additive mode with a warning if SDK APIs are unavailable), `"netra-strict"` (default) never falls back to native, and `"both"` sends to Netra and native. - -### Fixed - -- **Netra-only OpenAI Agents Tracing**: Added `canSet` gating and a restore fallback so the OpenAI Agents processor can be swapped safely, and included `OPENAI_AGENTS` in the root-span instrument allowlist (`DEFAULT_INSTRUMENTS_FOR_ROOT`) so agent runs are captured as root spans. - -## [1.2.0] - 2026-06-23 - -### Added - -- **Simulation File Handling**: The simulation workflow now supports file attachments on user messages. The SDK parses `attachments` metadata from the API (`FileData`), downloads the referenced content via pre-signed URLs, and delivers it to user tasks as base64-encoded `ProcessedFile` objects. `ProcessedFile` is now exported from the package root. - -## [1.1.0] - 2026-06-16 - -### Added - -- **Context Propagation Helpers**: Exported `netraExpressMiddleware` and `runWithExtractedContext` for distributed tracing. These utilities extract incoming W3C Trace Context from HTTP headers and run code within that context, covering cases where auto-instrumentation is unavailable (ESM load-order issues, missing peer dependencies, or non-Express frameworks). - -### Fixed - -- **CommonJS (CJS) Support for LLM Provider Instrumentations**: Fixed instrumentation for Anthropic, Google GenAI, Groq, and Mistral AI so they patch correctly under CommonJS module resolution, resolving instrumentation failures in CJS applications. - ## [1.6.0] - 2026-07-17 ### Added diff --git a/src/api/evaluation/api.ts b/src/api/evaluation/api.ts index dae512a..3edda31 100644 --- a/src/api/evaluation/api.ts +++ b/src/api/evaluation/api.ts @@ -368,7 +368,11 @@ export class Evaluation { ): Promise<{ status: string }> { const spanName = `TestRun.${runName}`; - const span = new SpanWrapper(spanName, {}, "netra.evaluation"); + const span = new SpanWrapper( + spanName, + { [Config.TRACE_ORIGIN_KEY]: Config.TRACE_ORIGIN_EVALUATION }, + "netra.evaluation", + ); span.start(); try { diff --git a/src/config.ts b/src/config.ts index f297a7e..31b12a8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -127,6 +127,11 @@ export class Config { static readonly LIBRARY_VERSION = SDK_VERSION; static readonly TRIAL_BLOCK_DURATION_SECONDS = 900; // 15 minutes + // Root-span attribute marking traces produced by evaluation/simulation runs + // so the FE/BE can distinguish them from normal workflow invocations. + static readonly TRACE_ORIGIN_KEY = "netra.trace.origin"; + static readonly TRACE_ORIGIN_EVALUATION = "evaluation"; + private static _spanAttributeMaxSize: number | undefined; /** diff --git a/src/simulation/api.ts b/src/simulation/api.ts index 21e7d4a..8889bb4 100644 --- a/src/simulation/api.ts +++ b/src/simulation/api.ts @@ -463,7 +463,11 @@ export class Simulation { let rawFiles: FileData[] = initialFiles ?? []; while (true) { - const span = new SpanWrapper(SPAN_NAME, {}, LOG_PREFIX); + const span = new SpanWrapper( + SPAN_NAME, + { [Config.TRACE_ORIGIN_KEY]: Config.TRACE_ORIGIN_EVALUATION }, + LOG_PREFIX, + ); span.start(); try { From 7d7c5933b89fa77295e010dd56e808c0e3ffecb3 Mon Sep 17 00:00:00 2001 From: Akhilesh Nair <72593014+AkhileshNair2201@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:45:25 +0530 Subject: [PATCH 07/20] NET-1072 fix(sdk-js): map db.statement to span input (#120) Co-authored-by: Cursor --- src/processors/span-io-processor.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/processors/span-io-processor.ts b/src/processors/span-io-processor.ts index 77129ed..2d714bc 100644 --- a/src/processors/span-io-processor.ts +++ b/src/processors/span-io-processor.ts @@ -248,7 +248,17 @@ export class SpanIOProcessor implements SpanProcessor { return original(_USAGE_COMPLETION_TOKENS, value); } - // 9. Pass through + // 9. db.statement → input (DB instrumentations: TypeORM, etc.) + // Keep db.statement; never map into output. + if (key === "db.statement") { + original(key, value); + if (!inputLocked() && inputIsEmpty() && !isEmpty(value)) { + original("input", typeof value === "string" ? value : String(value)); + } + return span; + } + + // 10. Pass through return original(key, value); } catch (e) { Logger.debug(`SpanIOProcessor: error processing key=${key}`, e); From 64cfca8e3c6f726ffb2bd475d7435a8ea55504de Mon Sep 17 00:00:00 2001 From: Akash Vijay Date: Mon, 27 Jul 2026 15:13:23 +0530 Subject: [PATCH 08/20] [NET-1380] refactor: Remove enable root span configuration (#123) --- .env.sample | 3 --- src/config.ts | 13 ++----------- src/index.ts | 54 +-------------------------------------------------- 3 files changed, 3 insertions(+), 67 deletions(-) diff --git a/.env.sample b/.env.sample index 0130c64..e747ca8 100644 --- a/.env.sample +++ b/.env.sample @@ -10,9 +10,6 @@ NETRA_DISABLE_BATCH= # Debug Mode NETRA_DEBUG= -# Root Span Configuration -NETRA_ENABLE_ROOT_SPAN= - # Scrubbing Configuration NETRA_ENABLE_SCRUBBING= diff --git a/src/config.ts b/src/config.ts index 31b12a8..c4ca815 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,7 +11,6 @@ export interface NetraConfig { disableBatch?: boolean; traceContent?: boolean; debugMode?: boolean; - enableRootSpan?: boolean; resourceAttributes?: Record; environment?: string; enableScrubbing?: boolean; @@ -33,10 +32,8 @@ export interface NetraConfig { * link is remote (cross-process) is kept as a root so an upstream distributed * trace is not severed. * - * Note: when `enableRootSpan` is true, Netra attaches its own root span and - * every auto-instrumentation span becomes its child, so no reparenting - * occurs. Pass a set containing `NetraInstruments.ALL` to allow all - * instrumentations to produce root spans (legacy behaviour). + * Pass a set containing `NetraInstruments.ALL` to allow all instrumentations + * to produce root spans (legacy behaviour). */ rootInstruments?: Set; } @@ -155,7 +152,6 @@ export class Config { disableBatch: boolean; traceContent: boolean; debugMode: boolean; - enableRootSpan: boolean; enableScrubbing: boolean; environment: string; resourceAttributes: Record; @@ -181,11 +177,6 @@ export class Config { "NETRA_DEBUG", false, ); - this.enableRootSpan = this._getBoolConfig( - config.enableRootSpan, - "NETRA_ENABLE_ROOT_SPAN", - false, - ); this.enableScrubbing = this._getBoolConfig( config.enableScrubbing, "NETRA_ENABLE_SCRUBBING", diff --git a/src/index.ts b/src/index.ts index d7506ee..00d96b8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,7 +4,7 @@ * Built on top of OpenTelemetry and Traceloop */ -import { context, Span, SpanKind, trace } from "@opentelemetry/api"; +import { trace } from "@opentelemetry/api"; import { createRequire } from "module"; import { Prompts, Dashboard, Evaluation, Usage, Models } from "./api"; import { Config, NetraConfig } from "./config"; @@ -148,7 +148,6 @@ export class Netra { private static _initialized = false; private static _config: Config | undefined; private static _tracer: any; - private static _rootSpan: Span | undefined; private static _metricsEnabled = false; static usage: Usage; @@ -283,33 +282,6 @@ export class Netra { // Handle crashes process.once("uncaughtException", handleUncaughtException); - // Create root span if enabled - if (cfg.enableRootSpan) { - // Use the effective tracer if available - const tracer = this._tracer || trace.getTracer("netra.root.span"); - const rootName = `${Config.LIBRARY_NAME}.root.span`; - - // Create the root span - this._rootSpan = tracer.startSpan(rootName, { - kind: SpanKind.INTERNAL, - }); - - if (this._rootSpan) { - if (cfg.appName) { - this._rootSpan.setAttribute("service.name", cfg.appName); - } - this._rootSpan.setAttribute("netra.environment", cfg.environment); - this._rootSpan.setAttribute( - "netra.library.version", - Config.LIBRARY_VERSION, - ); - - Logger.info( - "Netra root span created. Use Netra.runWithRootSpan() to parent spans under it.", - ); - } - } - // Wait for all async instrumentations to be ready await instrumentationsReady; } @@ -326,15 +298,6 @@ export class Netra { Logger.error("Error during uninstrumentAll:", e); } - if (this._rootSpan) { - try { - this._rootSpan.end(); - } catch (e) { - } finally { - this._rootSpan = undefined; - } - } - const FLUSH_TIMEOUT_MS = 5000; try { @@ -393,21 +356,6 @@ export class Netra { SessionManager.setRootOutput(value); } - /** - * Run a function with the root span as the active parent context. - * All spans created within this function will be children of the root span. - * Note: required in JS because OTel JS has no persistent context.attach() - */ - static runWithRootSpan(fn: () => T): T { - if (!this._rootSpan) { - Logger.warn( - "runWithRootSpan: No root span available. Running function without parent context.", - ); - return fn(); - } - return context.with(trace.setSpan(context.active(), this._rootSpan), fn); - } - static setSessionId(sessionId: string): void { if (typeof sessionId !== "string") { Logger.error(`setSessionId: sessionId must be a string, got ${typeof sessionId}`); From b9621d18607e252c5cb61a8b88c792d1e4e4a479 Mon Sep 17 00:00:00 2001 From: Akash Vijay Date: Mon, 27 Jul 2026 16:24:41 +0530 Subject: [PATCH 09/20] feat: Add support for capturing cache-write tokens in OpenAI instrumentation (#116) --- src/instrumentation/utils.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/instrumentation/utils.ts b/src/instrumentation/utils.ts index 53db4ae..81baa92 100644 --- a/src/instrumentation/utils.ts +++ b/src/instrumentation/utils.ts @@ -584,7 +584,13 @@ function setUsageAttributes( ); } - const cacheCreationTokens = usage.cache_creation_input_tokens; + const cacheCreationTokens = + usage.cache_creation_input_tokens ?? + ( + (usage.prompt_tokens_details ?? usage.input_tokens_details) as + | { cache_write_tokens?: unknown } + | undefined + )?.cache_write_tokens; if (cacheCreationTokens !== undefined) { span.setAttribute( SpanAttributes.LLM_USAGE_CACHE_CREATION_INPUT_TOKENS, From e13bfd93184c0dd2e1f0f7544c0861d09878d4ab Mon Sep 17 00:00:00 2001 From: Chanchal Delson Date: Tue, 4 Aug 2026 17:09:49 +0530 Subject: [PATCH 10/20] [NET-1421] feat: session details implementation (#131) * feat: session details implementation * feat: session details implementation- pr comments fixes * feat: session details implementation- pr comments fixes 2 --- src/api/dashboard/api.ts | 18 ++++++++++++++++ src/api/dashboard/client.ts | 42 +++++++++++++++++++++++++++++++++++++ src/api/dashboard/index.ts | 3 +++ src/api/dashboard/models.ts | 37 ++++++++++++++++++++++++++++++++ src/api/index.ts | 3 +++ src/index.ts | 3 +++ 6 files changed, 106 insertions(+) diff --git a/src/api/dashboard/api.ts b/src/api/dashboard/api.ts index 1c2fea8..b684be9 100644 --- a/src/api/dashboard/api.ts +++ b/src/api/dashboard/api.ts @@ -14,6 +14,7 @@ import { QueryDataParams, QueryResponse, Scope, + SessionDetailsResponse, SessionFilter, SessionFilterConfig, SessionStatsData, @@ -196,6 +197,23 @@ async getSessionStats( return result.data ?? {}; } + async getSessionDetails(sessionId: string): Promise { + if (!sessionId) { + Logger.error( + "netra.dashboard: session_id is required to fetch session details", + ); + return null; + } + + const result = await this.client.getSessionDetails(sessionId); + + if (!result) { + return null; + } + + return result.data ?? result; + } + private isValidScope(scope: any): scope is Scope { return Object.values(Scope).includes(scope); } diff --git a/src/api/dashboard/client.ts b/src/api/dashboard/client.ts index 026c33f..fc4bb7e 100644 --- a/src/api/dashboard/client.ts +++ b/src/api/dashboard/client.ts @@ -244,4 +244,46 @@ export class DashboardHttpClient extends NetraHttpClient { return null; } } + + /** + * Get detailed information for a specific session including all its traces. + * + * Args: + * sessionId: The session ID to retrieve details for. + * + * Returns: + * The session details response data or null on error. + */ + async getSessionDetails(sessionId: string): Promise { + if (!this.isInitialized()) { + Logger.error( + "netra.dashboard: Dashboard client is not initialized; cannot get session details", + ); + return null; + } + + try { + const url = `/public/dashboard/session/${encodeURIComponent(sessionId)}`; + + const response = await this.get(url); + + if (!response.ok) { + const errorMessage = response.data?.error?.message ?? "Unknown error"; + Logger.error( + `netra.dashboard: Failed to fetch session details: ${errorMessage}`, + ); + return null; + } + + return response.data; + } catch (err: any) { + const message = err?.response?.data?.error?.message ?? ""; + + Logger.error( + "netra.dashboard: Failed to fetch session details:", + message, + ); + return null; + } + } } diff --git a/src/api/dashboard/index.ts b/src/api/dashboard/index.ts index dfde5e8..c98f100 100644 --- a/src/api/dashboard/index.ts +++ b/src/api/dashboard/index.ts @@ -35,6 +35,9 @@ export type { NumberResponse, QueryDataParams, QueryResponse, + SessionDetailsResponse, + SessionDetailsToolCall, + SessionDetailsTrace, TimeRange, TimeSeriesDataPoint, TimeSeriesResponse, diff --git a/src/api/dashboard/models.ts b/src/api/dashboard/models.ts index 5a8aa0d..7674715 100644 --- a/src/api/dashboard/models.ts +++ b/src/api/dashboard/models.ts @@ -228,3 +228,40 @@ export interface SessionStatsData { session_duration: string; cursor: string; } + +// Session Detail +export interface SessionDetailsToolCall { + toolName: string; + toolCallCount: number; +} + +export interface SessionDetailsTrace { + traceId: string; + traceName: string; + startTime: string; + endTime: string; + latencyMs: number; + input: string | null; + output: string | null; + tokens: { + promptTokens: number; + completionTokens: number; + cachedTokens: number; + cacheCreationTokens: number; + totalTokens: number; + }; + cost: { + promptTokensCost: number; + completionTokensCost: number; + cachedTokensCost: number; + cacheCreationTokensCost: number; + totalCost: number; + }; + models: string[]; + toolCalls: SessionDetailsToolCall[]; +} + +export interface SessionDetailsResponse { + sessionId: string; + traces: SessionDetailsTrace[]; +} diff --git a/src/api/index.ts b/src/api/index.ts index 249df25..520f136 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -59,6 +59,9 @@ export type { NumberResponse, QueryDataParams, QueryResponse, + SessionDetailsResponse, + SessionDetailsToolCall, + SessionDetailsTrace, TimeRange, TimeSeriesDataPoint, TimeSeriesResponse, diff --git a/src/index.ts b/src/index.ts index 00d96b8..ea42421 100644 --- a/src/index.ts +++ b/src/index.ts @@ -97,6 +97,9 @@ export type { QueryDataParams, QueryResponse, Run, + SessionDetailsResponse, + SessionDetailsToolCall, + SessionDetailsTrace, SessionUsageData, SpansPage, TaskFunction, From b8d006f18fa81699374825328e01a791077fd5dc Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Wed, 5 Aug 2026 12:29:49 +0530 Subject: [PATCH 11/20] [NET-1093] feat: Add TTFT and RTTFT to LLM spans --- CHANGELOG.md | 26 +++--- package-lock.json | 4 +- package.json | 2 +- src/instrumentation/anthropic/utils.ts | 3 + src/instrumentation/anthropic/wrappers.ts | 24 +++++- src/instrumentation/google-genai/utils.ts | 7 +- src/instrumentation/google-genai/wrappers.ts | 20 ++++- .../google-generative-ai/wrappers.ts | 54 +++++++----- src/instrumentation/groq/wrappers.ts | 16 +++- src/instrumentation/mistralai/wrappers.ts | 16 +++- src/instrumentation/openai/wrappers.ts | 12 ++- src/instrumentation/span-attributes.ts | 6 ++ src/utils/span-timing.ts | 85 +++++++++++++++++++ src/version.ts | 2 +- 14 files changed, 228 insertions(+), 49 deletions(-) create mode 100644 src/utils/span-timing.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ad68004..afd3fdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,21 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.8.0] - 2026-08-03 - -### Added - -- **Postscript Failed Status**: After-hook failures (`after`, `afterEach`, `afterAll`) on otherwise successful scenarios are now reported as `postscript_failed` instead of being logged and ignored. Existing failure statuses (`failed` / `prescript_failed`) are never overwritten. `after` and `afterEach` always both attempt to run; `afterAll` failure marks only successfully completed scenarios. - -## [1.7.0] - 2026-07-27 +## [1.9.0] - Unreleased ### Added -- **Simulation Lifecycle Hooks**: Added prescript/postscript support for multi-turn simulations via `SimulationHooks` (`beforeAll`, `beforeEach`, `before`, `after`, `afterEach`, `afterAll`). Hooks can return setup context passed into `BaseTask.run`, and the run uses a two-phase initialize / first-turn flow so hooks execute before any LLM spend. Execution order is `beforeAll` → `beforeEach` → item-specific `before` → task → item-specific `after` → `afterEach` → `afterAll`. `beforeAll` failure aborts the run as `prescript_failed`; item `before` failure marks only that scenario; `after`, `afterEach`, and `afterAll` failures are logged and do not affect status. - -## [Unreleased] - -### Added +- **Time to First Token (TTFT) & Relative TTFT**: All LLM generation spans now record `gen_ai.performance.time_to_first_token` (seconds from call start to first content chunk), `gen_ai.performance.relative_time_to_first_token` (seconds from trace root span start to first content chunk), and `gen_ai.performance.time_to_first_token.timestamp` (absolute ISO 8601 UTC time of first token). Supported across OpenAI, Anthropic, Groq, Mistral, Google GenAI, and Google Generative AI for both streaming and non-streaming calls. RTTFT is silently skipped when no root span exists. - **Opt-in prompt caching** — `Netra.prompts.getPrompt()` accepts `useCache` and `cacheTtl`. When `useCache` is true, responses are served from an in-memory TTL cache (default TTL: `PROMPT_CACHE_TTL_SECONDS` = 60). Caching is off by default. - **Models API** — `Netra.models.getModelPricing()` fetches model pricing (optional `name` filter) with the same opt-in cache pattern (`useCache`, `cacheTtl`; default TTL: `MODEL_PRICING_CACHE_TTL_SECONDS` = 300). @@ -30,6 +20,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Prompt cache TTL** — Default TTL is the module constant `PROMPT_CACHE_TTL_SECONDS` (60). Override per call with `cacheTtl`. Removed unused `cacheTtlSeconds` init config and `NETRA_CACHE_TTL_SECONDS` env var. +## [1.8.0] - 2026-08-03 + +### Added + +- **Postscript Failed Status**: After-hook failures (`after`, `afterEach`, `afterAll`) on otherwise successful scenarios are now reported as `postscript_failed` instead of being logged and ignored. Existing failure statuses (`failed` / `prescript_failed`) are never overwritten. `after` and `afterEach` always both attempt to run; `afterAll` failure marks only successfully completed scenarios. + +## [1.7.0] - 2026-07-27 + +### Added + +- **Simulation Lifecycle Hooks**: Added prescript/postscript support for multi-turn simulations via `SimulationHooks` (`beforeAll`, `beforeEach`, `before`, `after`, `afterEach`, `afterAll`). Hooks can return setup context passed into `BaseTask.run`, and the run uses a two-phase initialize / first-turn flow so hooks execute before any LLM spend. Execution order is `beforeAll` → `beforeEach` → item-specific `before` → task → item-specific `after` → `afterEach` → `afterAll`. `beforeAll` failure aborts the run as `prescript_failed`; item `before` failure marks only that scenario; `after`, `afterEach`, and `afterAll` failures are logged and do not affect status. + ## [1.6.0] - 2026-07-17 ### Added diff --git a/package-lock.json b/package-lock.json index bc6fb8b..804b9ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "netra-sdk", - "version": "1.8.0", + "version": "1.9.0-dev.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "netra-sdk", - "version": "1.8.0", + "version": "1.9.0-dev.0", "license": "Apache-2.0", "dependencies": { "@opentelemetry/api": "^1.9.0", diff --git a/package.json b/package.json index 2e50dab..890925f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "netra-sdk", - "version": "1.8.0", + "version": "1.9.0-dev.0", "description": "A comprehensive TypeScript/JavaScript SDK for AI application observability built on top of OpenTelemetry and Traceloop", "type": "module", "main": "./dist/index.cjs", diff --git a/src/instrumentation/anthropic/utils.ts b/src/instrumentation/anthropic/utils.ts index b378226..0ff521f 100644 --- a/src/instrumentation/anthropic/utils.ts +++ b/src/instrumentation/anthropic/utils.ts @@ -1,5 +1,6 @@ import { Span, SpanStatusCode } from "@opentelemetry/api"; import { Logger } from "../../logger"; +import type { FirstTokenTracker } from "../../utils/span-timing"; import { setRequestAttributes as setBaseRequestAttributes, setResponseAttributes as setBaseResponseAttributes, @@ -13,6 +14,7 @@ export function processStreamChunk( completeResponse: Record, chunk: any, span: Span, + tokenTracker?: FirstTokenTracker, ): void { try { switch (chunk.type) { @@ -67,6 +69,7 @@ export function processStreamChunk( targetBlock.input += chunk.delta.partial_json ?? ""; } else if (chunk.delta?.text) { targetBlock.text += chunk.delta.text; + tokenTracker?.markFirstToken(); } break; } diff --git a/src/instrumentation/anthropic/wrappers.ts b/src/instrumentation/anthropic/wrappers.ts index 73267b0..93182e2 100644 --- a/src/instrumentation/anthropic/wrappers.ts +++ b/src/instrumentation/anthropic/wrappers.ts @@ -10,6 +10,10 @@ import { import { Logger } from "../../logger"; import { wrapResponse } from "../../utils/response-handler"; import { safeStringify } from "../../utils/serialization"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { SpanAttributes } from "../span-attributes"; import { defineHidden, @@ -38,6 +42,7 @@ const WRAPPER_OWN_PROPS = new Set([ "spanFinalized", "completionPending", "listenerMap", + "tokenTracker", ]); const EVENT_EMITTER_METHODS = new Set([ @@ -152,6 +157,7 @@ class MessageStreamWrapper { private spanFinalized = false; private completionPending = false; private listenerMap = new WeakMap>(); + private tokenTracker!: FirstTokenTracker; constructor( span: Span, @@ -165,6 +171,7 @@ class MessageStreamWrapper { defineHidden(this, "messageStream", messageStream); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); defineHidden( this, "spanContext", @@ -319,7 +326,12 @@ class MessageStreamWrapper { let errorOccurred = false; try { for await (const chunk of this.messageStream) { - processStreamChunk(this.completeResponse, chunk, this.span); + processStreamChunk( + this.completeResponse, + chunk, + this.span, + this.tokenTracker, + ); yield chunk; } } catch (err) { @@ -350,6 +362,7 @@ class MessageStreamWrapper { this.completeResponse.currentText = ""; } this.completeResponse.currentText += data; + this.tokenTracker.markFirstToken(); break; case "contentBlock": @@ -444,13 +457,19 @@ function anthropicWrapper( model: "", usage: {}, }; + const tokenTracker = new FirstTokenTracker(span, startTime); return wrapResponse( response, { withContext: (fn) => context.with(spanContext, fn), onChunk: (chunk) => - processStreamChunk(completeResponse, chunk, span), + processStreamChunk( + completeResponse, + chunk, + span, + tokenTracker, + ), onError: (error) => { Logger.error("netra.instrumentation.anthropic:", error); span.setStatus({ @@ -468,6 +487,7 @@ function anthropicWrapper( "llm.response.duration", (endTime - startTime) / 1000, ); + recordNonStreamingTimingAttributes(span, startTime, endTime); }, finalize: (status) => { const hasStreamData = diff --git a/src/instrumentation/google-genai/utils.ts b/src/instrumentation/google-genai/utils.ts index e2308ab..91840db 100644 --- a/src/instrumentation/google-genai/utils.ts +++ b/src/instrumentation/google-genai/utils.ts @@ -14,6 +14,7 @@ import { Span } from "@opentelemetry/api"; import { Logger } from "../../logger"; import { safeStringify } from "../../utils/serialization"; +import type { FirstTokenTracker } from "../../utils/span-timing"; import { SpanAttributes } from "../span-attributes"; import { TracedMessage, @@ -488,6 +489,7 @@ export function processStreamChunk( chunk: any, span: Span, startTime: number, + tokenTracker?: FirstTokenTracker, ): void { try { if (chunk.modelVersion) { @@ -502,10 +504,7 @@ export function processStreamChunk( if (chunkText && chunkText.length > 0) { if (!accumulated._text) { accumulated._text = chunkText; - span.setAttribute( - "gen_ai.performance.time_to_first_token", - (Date.now() - startTime) / 1000, - ); + tokenTracker?.markFirstToken(); } else { accumulated._text += chunkText; } diff --git a/src/instrumentation/google-genai/wrappers.ts b/src/instrumentation/google-genai/wrappers.ts index 724fc2e..3a5fd96 100644 --- a/src/instrumentation/google-genai/wrappers.ts +++ b/src/instrumentation/google-genai/wrappers.ts @@ -21,6 +21,10 @@ import { } from "@opentelemetry/api"; import { Logger } from "../../logger"; import { wrapResponse } from "../../utils/response-handler"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { createSuppressedContext, modelAsDict, @@ -127,6 +131,13 @@ function genericWrapperFactory( SpanAttributes.LLM_RESPONSE_DURATION, (endTime - startTime) / 1000, ); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes( + span, + startTime, + endTime, + ); + } }, onError: (error) => { span.setStatus({ @@ -194,6 +205,7 @@ function streamWrapperFactory( (span: Span) => { const startTime = Date.now(); const accumulated: Record = {}; + const tokenTracker = new FirstTokenTracker(span, startTime); try { setRequestAttributes(span, params, requestType); @@ -209,7 +221,13 @@ function streamWrapperFactory( { withContext: (fn) => context.with(spanContext, fn), onChunk: (chunk) => { - processStreamChunk(accumulated, chunk, span, startTime); + processStreamChunk( + accumulated, + chunk, + span, + startTime, + tokenTracker, + ); }, onError: (error) => { span.setStatus({ diff --git a/src/instrumentation/google-generative-ai/wrappers.ts b/src/instrumentation/google-generative-ai/wrappers.ts index e60a8a3..7eb0811 100644 --- a/src/instrumentation/google-generative-ai/wrappers.ts +++ b/src/instrumentation/google-generative-ai/wrappers.ts @@ -6,6 +6,10 @@ import { context, } from "@opentelemetry/api"; import { Logger } from "../../logger"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { isPromise, modelAsDict, @@ -99,12 +103,15 @@ function googleGenerativeAIWrapper( const endTime = Date.now(); const responseDict = modelAsDict(value); setResponseAttributes(span, responseDict); - const duration = (endTime - startTime) / 1000; - span.setAttribute("llm.response.duration", duration); + span.setAttribute( + "llm.response.duration", + (endTime - startTime) / 1000, + ); if (requestType !== "embedding") { - span.setAttribute( - "gen_ai.performance.time_to_first_token", - duration, + recordNonStreamingTimingAttributes( + span, + startTime, + endTime, ); } } catch (e) { @@ -133,6 +140,9 @@ function googleGenerativeAIWrapper( "llm.response.duration", (endTime - startTime) / 1000, ); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } } catch (e) { Logger.error(`${LOG_PREFIX}:`, e); } @@ -196,6 +206,8 @@ function googleGenerativeAIStreamWrapper( return response; } + const tokenTracker = new FirstTokenTracker(span, startTime); + return (async () => { try { const streamResult: any = await response; @@ -210,11 +222,14 @@ function googleGenerativeAIStreamWrapper( const endTime = Date.now(); const responseDict = modelAsDict(streamResult); setResponseAttributes(span, responseDict); - const duration = (endTime - startTime) / 1000; - span.setAttribute("llm.response.duration", duration); span.setAttribute( - "gen_ai.performance.time_to_first_token", - duration, + "llm.response.duration", + (endTime - startTime) / 1000, + ); + recordNonStreamingTimingAttributes( + span, + startTime, + endTime, ); } catch (e) { Logger.error(`${LOG_PREFIX}:`, e); @@ -224,8 +239,6 @@ function googleGenerativeAIStreamWrapper( return streamResult; } - let firstTokenRecorded = false; - const wrappedStream: AsyncIterable = { [Symbol.asyncIterator]() { const iterator = originalStream[Symbol.asyncIterator](); @@ -256,10 +269,9 @@ function googleGenerativeAIStreamWrapper( setResponseAttributes(span, responseDict); } - const duration = (endTime - startTime) / 1000; span.setAttribute( "llm.response.duration", - duration, + (endTime - startTime) / 1000, ); } catch (e) { Logger.error(`${LOG_PREFIX}:`, e); @@ -276,14 +288,8 @@ function googleGenerativeAIStreamWrapper( typeof chunk?.text === "function" ? chunk.text() : chunk?.text; - if (typeof t === "string") { - if (t && !firstTokenRecorded) { - span.setAttribute( - "gen_ai.performance.time_to_first_token", - (Date.now() - startTime) / 1000, - ); - firstTokenRecorded = true; - } + if (typeof t === "string" && t) { + tokenTracker.markFirstToken(); } } catch { // ignore chunk parsing issues @@ -305,8 +311,10 @@ function googleGenerativeAIStreamWrapper( }, async return(value?: any) { const endTime = Date.now(); - const duration = (endTime - startTime) / 1000; - span.setAttribute("llm.response.duration", duration); + span.setAttribute( + "llm.response.duration", + (endTime - startTime) / 1000, + ); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return iterator.return?.(value) ?? { value: undefined, done: true as const }; diff --git a/src/instrumentation/groq/wrappers.ts b/src/instrumentation/groq/wrappers.ts index 3743f32..bf8bdf1 100644 --- a/src/instrumentation/groq/wrappers.ts +++ b/src/instrumentation/groq/wrappers.ts @@ -1,12 +1,16 @@ import { Tracer, Span, SpanKind, SpanStatusCode, context } from "@opentelemetry/api"; import { Logger } from "../../logger"; -import { setRequestAttributes, setResponseAttributes } from "./utils"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { defineHidden, modelAsDict, isPromise, shouldSuppressInstrumentation, } from "../utils"; +import { setRequestAttributes, setResponseAttributes } from "./utils"; type GroqRequestType = "chat"; @@ -98,6 +102,7 @@ function groqWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); + recordNonStreamingTimingAttributes(span, startTime, endTime); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return value; @@ -121,6 +126,7 @@ function groqWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); + recordNonStreamingTimingAttributes(span, startTime, endTime); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return response; @@ -152,12 +158,14 @@ export class StreamingWrapper implements Iterable, Iterator { private response!: any; private startTime!: number; private requestKwargs!: Record; + private tokenTracker!: FirstTokenTracker; constructor(span: Span, response: any, startTime: number, requestKwargs: Record) { defineHidden(this, "span", span); defineHidden(this, "response", response); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); } toJSON() { @@ -224,6 +232,7 @@ export class StreamingWrapper implements Iterable, Iterator { choices[index].message = { role: "assistant", content: "" }; } choices[index].message.content += String(delta.content); + this.tokenTracker.markFirstToken(); } if (choice.finish_reason) { @@ -245,6 +254,7 @@ export class StreamingWrapper implements Iterable, Iterator { }); }); this.completeResponse.usage = chunkDict.response.usage || {}; + this.tokenTracker.markFirstToken(); } this.span.addEvent("llm.content.completion.chunk"); @@ -281,12 +291,14 @@ export class AsyncStreamingWrapper private response!: any; private startTime!: number; private requestKwargs!: Record; + private tokenTracker!: FirstTokenTracker; constructor(span: Span, response: any, startTime: number, requestKwargs: Record) { defineHidden(this, "span", span); defineHidden(this, "response", response); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); } toJSON() { @@ -367,6 +379,7 @@ export class AsyncStreamingWrapper } const message = choiceEntry.message as Record; message.content = String(message.content || "") + contentPiece; + this.tokenTracker.markFirstToken(); } if (choice.finish_reason) { @@ -399,6 +412,7 @@ export class AsyncStreamingWrapper const usage = response.usage || {}; this.completeResponse.usage = usage; }); + this.tokenTracker.markFirstToken(); } this.span.addEvent("llm.content.completion.chunk"); } diff --git a/src/instrumentation/mistralai/wrappers.ts b/src/instrumentation/mistralai/wrappers.ts index c904c3d..1470aad 100644 --- a/src/instrumentation/mistralai/wrappers.ts +++ b/src/instrumentation/mistralai/wrappers.ts @@ -10,6 +10,10 @@ import { context, } from "@opentelemetry/api"; import { Logger } from "../../logger"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { defineHidden, isPromise } from "../utils"; import { modelAsDict, @@ -67,6 +71,7 @@ function mistralWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); + recordNonStreamingTimingAttributes(span, startTime, endTime); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return value; @@ -90,6 +95,7 @@ function mistralWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); + recordNonStreamingTimingAttributes(span, startTime, endTime); span.setStatus({ code: SpanStatusCode.OK }); span.end(); return response; @@ -245,12 +251,14 @@ export class StreamingWrapper implements Iterable, Iterator { private response!: unknown; private startTime!: number; private requestKwargs!: Record; + private tokenTracker!: FirstTokenTracker; constructor(span: Span, response: unknown, startTime: number, requestKwargs: Record) { defineHidden(this, "span", span); defineHidden(this, "response", response); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); } toJSON() { @@ -354,6 +362,7 @@ export class StreamingWrapper implements Iterable, Iterator { } else { choiceEntry.text = String(choiceEntry.text || "") + contentPiece; } + this.tokenTracker.markFirstToken(); } if (choice.finishReason) { @@ -389,6 +398,7 @@ export class StreamingWrapper implements Iterable, Iterator { } else { choiceEntry.text = String(choiceEntry.text || "") + contentPiece; } + this.tokenTracker.markFirstToken(); } if (choice.finishReason) { @@ -439,17 +449,19 @@ export class AsyncStreamingWrapper private startTime!: number; private requestKwargs!: Record; private completeResponse: Record; + private tokenTracker!: FirstTokenTracker; constructor( span: Span, response: unknown, startTime: number, - requestKwargs: Record + requestKwargs: Record, ) { defineHidden(this, "span", span); defineHidden(this, "response", response); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); this.completeResponse = { choices: [], model: "" }; } @@ -572,6 +584,7 @@ export class AsyncStreamingWrapper } else { choiceEntry.text = String(choiceEntry.text || "") + contentPiece; } + this.tokenTracker.markFirstToken(); } if (choice.finishReason) { @@ -607,6 +620,7 @@ export class AsyncStreamingWrapper } else { choiceEntry.text = String(choiceEntry.text || "") + contentPiece; } + this.tokenTracker.markFirstToken(); } if (choice.finishReason) { diff --git a/src/instrumentation/openai/wrappers.ts b/src/instrumentation/openai/wrappers.ts index 5a9a455..d367d35 100644 --- a/src/instrumentation/openai/wrappers.ts +++ b/src/instrumentation/openai/wrappers.ts @@ -7,6 +7,10 @@ import { context, trace, } from "@opentelemetry/api"; +import { + FirstTokenTracker, + recordNonStreamingTimingAttributes, +} from "../../utils/span-timing"; import { defineHidden, isPromise, @@ -46,8 +50,10 @@ function finalizeSpanSuccess( response: Record, startTime: number, ): void { + const endTime = Date.now(); setResponseAttributes(span, response); - span.setAttribute("llm.response.duration", (Date.now() - startTime) / 1000); + span.setAttribute("llm.response.duration", (endTime - startTime) / 1000); + recordNonStreamingTimingAttributes(span, startTime, endTime); span.setStatus({ code: SpanStatusCode.OK }); span.end(); } @@ -58,6 +64,7 @@ abstract class BaseStreamHandler { protected span!: Span; protected startTime!: number; protected requestKwargs!: Record; + protected tokenTracker!: FirstTokenTracker; constructor( span: Span, @@ -67,6 +74,7 @@ abstract class BaseStreamHandler { defineHidden(this, "span", span); defineHidden(this, "startTime", startTime); defineHidden(this, "requestKwargs", requestKwargs); + defineHidden(this, "tokenTracker", new FirstTokenTracker(span, startTime)); } toJSON(): StreamResponse { @@ -93,6 +101,7 @@ abstract class BaseStreamHandler { } const msg = entry.message as Record; msg.content = String(msg.content ?? "") + String(delta.content); + this.tokenTracker.markFirstToken(); } if (choice.finish_reason) { this.completeResponse.choices[index].finish_reason = @@ -126,6 +135,7 @@ abstract class BaseStreamHandler { } } this.completeResponse.usage = responseChunk.usage ?? {}; + this.tokenTracker.markFirstToken(); } this.span.addEvent("llm.content.completion.chunk"); diff --git a/src/instrumentation/span-attributes.ts b/src/instrumentation/span-attributes.ts index 578810d..5500cf8 100644 --- a/src/instrumentation/span-attributes.ts +++ b/src/instrumentation/span-attributes.ts @@ -33,4 +33,10 @@ export const SpanAttributes = { LLM_IS_STREAMING: "llm.is_streaming", LLM_COMPLETIONS: "gen_ai.completion", LLM_PROMPTS: "gen_ai.prompt", + + LLM_TIME_TO_FIRST_TOKEN: "gen_ai.performance.time_to_first_token", + LLM_RELATIVE_TIME_TO_FIRST_TOKEN: + "gen_ai.performance.relative_time_to_first_token", + LLM_TIME_TO_FIRST_TOKEN_TIMESTAMP: + "gen_ai.performance.time_to_first_token.timestamp", } as const; diff --git a/src/utils/span-timing.ts b/src/utils/span-timing.ts new file mode 100644 index 0000000..89905db --- /dev/null +++ b/src/utils/span-timing.ts @@ -0,0 +1,85 @@ +import { Span } from "@opentelemetry/api"; +import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; +import { RootSpanProcessor } from "../processors/root-span-processor"; +import { SpanAttributes } from "../instrumentation/span-attributes"; +import { Logger } from "../logger"; + +function hrTimeToMs(hrTime: [number, number]): number { + return hrTime[0] * 1000 + hrTime[1] / 1e6; +} + +function recordTimeToFirstToken( + span: Span, + nowMs: number, + startTimeMs: number, +): void { + span.setAttribute( + SpanAttributes.LLM_TIME_TO_FIRST_TOKEN, + (nowMs - startTimeMs) / 1000, + ); + span.setAttribute( + SpanAttributes.LLM_TIME_TO_FIRST_TOKEN_TIMESTAMP, + new Date(nowMs).toISOString(), + ); +} + +function recordRelativeTimeToFirstToken(span: Span, nowMs: number): void { + try { + const rootSpan = RootSpanProcessor.getRootSpan(span); + if (!rootSpan) return; + const hrStart = (rootSpan as unknown as ReadableSpan).startTime; + if (!hrStart) return; + const rootStartMs = hrTimeToMs(hrStart); + span.setAttribute( + SpanAttributes.LLM_RELATIVE_TIME_TO_FIRST_TOKEN, + (nowMs - rootStartMs) / 1000, + ); + } catch (e) { + Logger.warn("span-timing: failed to compute RTTFT", e); + } +} + +/** + * Tracks the first content token in a streaming LLM response and records + * TTFT, RTTFT, and the absolute first-token timestamp on the span. + * + * `markFirstToken()` is idempotent — only the first call writes attributes. + */ +export class FirstTokenTracker { + private _recorded = false; + + constructor( + private readonly span: Span, + private readonly startTimeMs: number, + ) {} + + get recorded(): boolean { + return this._recorded; + } + + markFirstToken(): void { + if (this._recorded || !this.span.isRecording()) return; + this._recorded = true; + + const now = Date.now(); + + recordTimeToFirstToken(this.span, now, this.startTimeMs); + recordRelativeTimeToFirstToken(this.span, now); + } +} + +/** + * Record TTFT + RTTFT for non-streaming LLM calls. + * For non-streaming, "first token" = full response arrival, + * so TTFT equals response duration. + */ +export function recordNonStreamingTimingAttributes( + span: Span, + startTimeMs: number, + endTimeMs: number, +): void { + if (!span.isRecording()) return; + + recordTimeToFirstToken(span, endTimeMs, startTimeMs); + recordRelativeTimeToFirstToken(span, endTimeMs); +} diff --git a/src/version.ts b/src/version.ts index f27afec..9ae0113 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const SDK_VERSION = "1.8.0"; +export const SDK_VERSION = "1.9.0-dev.0"; From 791fed24535ac828c6addd38d7371e1dd8321e6f Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Wed, 5 Aug 2026 13:48:13 +0530 Subject: [PATCH 12/20] [NET-1093] chore: Code cleanup --- src/instrumentation/anthropic/wrappers.ts | 8 ++++++++ src/instrumentation/groq/wrappers.ts | 9 ++++++--- src/instrumentation/openai/wrappers.ts | 4 +++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/instrumentation/anthropic/wrappers.ts b/src/instrumentation/anthropic/wrappers.ts index 93182e2..aa363e6 100644 --- a/src/instrumentation/anthropic/wrappers.ts +++ b/src/instrumentation/anthropic/wrappers.ts @@ -261,12 +261,20 @@ class MessageStreamWrapper { const result = await method.call(target.messageStream, ...args); if (prop === "finalMessage" || prop === "done") { + if (result) { + const hasText = Array.isArray(result.content) && + result.content.some((b: any) => b.type === "text" && b.text); + if (hasText) { + target.tokenTracker.markFirstToken(); + } + } target.finalizeSpanFromMessage(result); } else if (prop === "finalText") { if (typeof result === "string" && result.length > 0) { target.completeResponse.content = [ { type: "text", text: result }, ]; + target.tokenTracker.markFirstToken(); } else { target.flushCurrentText(); } diff --git a/src/instrumentation/groq/wrappers.ts b/src/instrumentation/groq/wrappers.ts index bf8bdf1..adef38e 100644 --- a/src/instrumentation/groq/wrappers.ts +++ b/src/instrumentation/groq/wrappers.ts @@ -244,17 +244,19 @@ export class StreamingWrapper implements Iterable, Iterator { if (chunkDict.usage) this.completeResponse.usage = chunkDict.usage; if (chunkDict.response?.status === "completed") { + let hasText = false; const outputs = chunkDict.response.output || []; outputs.forEach((output: any) => { const content = output.content || []; content.forEach((item: any) => { + if (item.text) hasText = true; choices.push({ message: { role: "assistant", content: item.text || "" }, }); }); }); this.completeResponse.usage = chunkDict.response.usage || {}; - this.tokenTracker.markFirstToken(); + if (hasText) this.tokenTracker.markFirstToken(); } this.span.addEvent("llm.content.completion.chunk"); @@ -392,6 +394,7 @@ export class AsyncStreamingWrapper // Response API if ((chunkDict.response as any)?.status === "completed") { + let hasText = false; const response = chunkDict.response as Record; const responseOutput = (response.output || []) as Array< Record @@ -401,7 +404,7 @@ export class AsyncStreamingWrapper if (content) { for (const contentItem of content) { const assistantText = contentItem.text || ""; - // Append to choices array instead of replacing + if (contentItem.text) hasText = true; ( this.completeResponse.choices as Array> ).push({ @@ -412,7 +415,7 @@ export class AsyncStreamingWrapper const usage = response.usage || {}; this.completeResponse.usage = usage; }); - this.tokenTracker.markFirstToken(); + if (hasText) this.tokenTracker.markFirstToken(); } this.span.addEvent("llm.content.completion.chunk"); } diff --git a/src/instrumentation/openai/wrappers.ts b/src/instrumentation/openai/wrappers.ts index d367d35..4631665 100644 --- a/src/instrumentation/openai/wrappers.ts +++ b/src/instrumentation/openai/wrappers.ts @@ -119,6 +119,7 @@ abstract class BaseStreamHandler { | Record | undefined; if (responseChunk?.status === "completed") { + let hasText = false; const outputs = (responseChunk.output ?? []) as Array< Record >; @@ -128,6 +129,7 @@ abstract class BaseStreamHandler { | undefined; if (Array.isArray(content)) { for (const item of content) { + if (item.text) hasText = true; this.completeResponse.choices.push({ message: { role: "assistant", content: String(item.text ?? "") }, }); @@ -135,7 +137,7 @@ abstract class BaseStreamHandler { } } this.completeResponse.usage = responseChunk.usage ?? {}; - this.tokenTracker.markFirstToken(); + if (hasText) this.tokenTracker.markFirstToken(); } this.span.addEvent("llm.content.completion.chunk"); From 90fe7c33805bc67b0d6fb7eb1b059090adbbf64f Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Wed, 5 Aug 2026 14:54:13 +0530 Subject: [PATCH 13/20] [NET-1093] fix: Remove overwriting of TTFT attributes in openai --- src/instrumentation/anthropic/wrappers.ts | 2 +- src/instrumentation/mistralai/wrappers.ts | 8 ++++-- src/instrumentation/openai/wrappers.ts | 33 +++++++++++------------ 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/src/instrumentation/anthropic/wrappers.ts b/src/instrumentation/anthropic/wrappers.ts index aa363e6..136b5ed 100644 --- a/src/instrumentation/anthropic/wrappers.ts +++ b/src/instrumentation/anthropic/wrappers.ts @@ -370,7 +370,7 @@ class MessageStreamWrapper { this.completeResponse.currentText = ""; } this.completeResponse.currentText += data; - this.tokenTracker.markFirstToken(); + if (data) this.tokenTracker.markFirstToken(); break; case "contentBlock": diff --git a/src/instrumentation/mistralai/wrappers.ts b/src/instrumentation/mistralai/wrappers.ts index 1470aad..6794104 100644 --- a/src/instrumentation/mistralai/wrappers.ts +++ b/src/instrumentation/mistralai/wrappers.ts @@ -71,7 +71,9 @@ function mistralWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); - recordNonStreamingTimingAttributes(span, startTime, endTime); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } span.setStatus({ code: SpanStatusCode.OK }); span.end(); return value; @@ -95,7 +97,9 @@ function mistralWrapper( "llm.response.duration", (endTime - startTime) / 1000 ); - recordNonStreamingTimingAttributes(span, startTime, endTime); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } span.setStatus({ code: SpanStatusCode.OK }); span.end(); return response; diff --git a/src/instrumentation/openai/wrappers.ts b/src/instrumentation/openai/wrappers.ts index 4631665..89a6213 100644 --- a/src/instrumentation/openai/wrappers.ts +++ b/src/instrumentation/openai/wrappers.ts @@ -49,11 +49,14 @@ function finalizeSpanSuccess( span: Span, response: Record, startTime: number, + requestType: OpenAIRequestType, ): void { const endTime = Date.now(); setResponseAttributes(span, response); span.setAttribute("llm.response.duration", (endTime - startTime) / 1000); - recordNonStreamingTimingAttributes(span, startTime, endTime); + if (requestType !== "embedding") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } span.setStatus({ code: SpanStatusCode.OK }); span.end(); } @@ -153,20 +156,16 @@ abstract class BaseStreamHandler { } protected finalizeSpan(code: SpanStatusCode): void { - if (code === SpanStatusCode.OK) { - finalizeSpanSuccess( - this.span, - this.completeResponse as Record, - this.startTime, - ); - } else { - this.span.setAttribute( - "llm.response.duration", - (Date.now() - this.startTime) / 1000, - ); - this.span.setStatus({ code }); - this.span.end(); - } + setResponseAttributes( + this.span, + this.completeResponse as Record, + ); + this.span.setAttribute( + "llm.response.duration", + (Date.now() - this.startTime) / 1000, + ); + this.span.setStatus({ code }); + this.span.end(); } } @@ -329,7 +328,7 @@ function executeNonStreaming( if (isPromise(result)) { return result.then( (value) => { - finalizeSpanSuccess(span, modelAsDict(value), startTime); + finalizeSpanSuccess(span, modelAsDict(value), startTime, requestType); return value; }, (error) => { @@ -339,7 +338,7 @@ function executeNonStreaming( ); } - finalizeSpanSuccess(span, modelAsDict(result), startTime); + finalizeSpanSuccess(span, modelAsDict(result), startTime, requestType); return result; } catch (error) { handleSpanError(span, error); From aab98e50b33a45afd2df57fc899b9085c347074b Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Wed, 5 Aug 2026 17:49:31 +0530 Subject: [PATCH 14/20] [NET-1093] fix: Add TTFT and RTTFT capturing for anthropic beta stream calls --- src/instrumentation/anthropic/wrappers.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/instrumentation/anthropic/wrappers.ts b/src/instrumentation/anthropic/wrappers.ts index 136b5ed..56bfc6f 100644 --- a/src/instrumentation/anthropic/wrappers.ts +++ b/src/instrumentation/anthropic/wrappers.ts @@ -301,6 +301,13 @@ class MessageStreamWrapper { try { if (typeof this.messageStream?.on !== "function") return; + // Capture TTFT directly from the raw stream so it is recorded + // even when the consumer (e.g. tool runner) never registers its + // own "text" listener through our proxy. + this.messageStream.on("text", (data: any) => { + if (data) this.tokenTracker.markFirstToken(); + }); + this.messageStream.on("end", () => { if (!this.completionPending) { this.finalizeSpanOnce(SpanStatusCode.OK); From 2154022baf1973edd2481ae9c675105114bee167 Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Fri, 7 Aug 2026 14:19:56 +0530 Subject: [PATCH 15/20] [NET-1093] fix: Add timing attributes to langgraph specific llm calls --- src/instrumentation/anthropic/wrappers.ts | 16 ++++++-- src/instrumentation/langgraph/wrappers.ts | 48 +++-------------------- 2 files changed, 18 insertions(+), 46 deletions(-) diff --git a/src/instrumentation/anthropic/wrappers.ts b/src/instrumentation/anthropic/wrappers.ts index 56bfc6f..0562e68 100644 --- a/src/instrumentation/anthropic/wrappers.ts +++ b/src/instrumentation/anthropic/wrappers.ts @@ -43,6 +43,7 @@ const WRAPPER_OWN_PROPS = new Set([ "completionPending", "listenerMap", "tokenTracker", + "ttftListener", ]); const EVENT_EMITTER_METHODS = new Set([ @@ -158,6 +159,7 @@ class MessageStreamWrapper { private completionPending = false; private listenerMap = new WeakMap>(); private tokenTracker!: FirstTokenTracker; + private ttftListener!: (data: any) => void; constructor( span: Span, @@ -245,7 +247,11 @@ class MessageStreamWrapper { if (prop === "removeAllListeners") { return function (event?: string) { target.listenerMap = new WeakMap(); - return method.call(target.messageStream, event); + const result = method.call(target.messageStream, event); + if (!event || event === "text") { + target.messageStream.on("text", target.ttftListener); + } + return result; }; } return method.bind(target.messageStream); @@ -303,10 +309,12 @@ class MessageStreamWrapper { // Capture TTFT directly from the raw stream so it is recorded // even when the consumer (e.g. tool runner) never registers its - // own "text" listener through our proxy. - this.messageStream.on("text", (data: any) => { + // own "text" listener through our proxy. Stored on the instance + // so removeAllListeners can re-attach it. + this.ttftListener = (data: any) => { if (data) this.tokenTracker.markFirstToken(); - }); + }; + this.messageStream.on("text", this.ttftListener); this.messageStream.on("end", () => { if (!this.completionPending) { diff --git a/src/instrumentation/langgraph/wrappers.ts b/src/instrumentation/langgraph/wrappers.ts index f319dce..8522834 100644 --- a/src/instrumentation/langgraph/wrappers.ts +++ b/src/instrumentation/langgraph/wrappers.ts @@ -19,50 +19,12 @@ import { setResponseAttributes as setBaseResponseAttributes, shouldSuppressInstrumentation, } from "../utils"; +import { recordNonStreamingTimingAttributes } from "../../utils/span-timing"; // Context key to track if we're inside a LangGraph instrumented call // This prevents double-instrumentation when invoke internally calls stream const LANGGRAPH_INSTRUMENTATION_ACTIVE = createContextKey("netra.langgraph.active"); -function getContextManager(): any { - try { - if ((context as any)._getContextManager) { - return (context as any)._getContextManager(); - } - - const globalSymbols = Object.getOwnPropertySymbols(global); - const otelSymbol = globalSymbols.find(s => - s.toString().includes("opentelemetry.js.api"), - ); - if (otelSymbol) { - const globalState = (global as any)[otelSymbol]; - if (globalState?.contextManager) { - return globalState.contextManager; - } - } - - return null; - } catch { - return null; - } -} - -function enterWithContext(newContext: Context): void { - const contextManager = getContextManager(); - if (!contextManager) return; - - if (typeof contextManager.enterWith === "function") { - contextManager.enterWith(newContext); - return; - } - - if ( - contextManager._asyncLocalStorage && - typeof contextManager._asyncLocalStorage.enterWith === "function" - ) { - contextManager._asyncLocalStorage.enterWith(newContext); - } -} import { NetraLanggraphAttributes, setChainInputAttributes, @@ -241,7 +203,8 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { metadata, prompts, extraParams, - parentRunId: effectiveParentRunId, // Store parent ID to link back + parentRunId: effectiveParentRunId, + startTimeMs: Date.now(), }); } @@ -272,6 +235,9 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { attributes.extraParams, ); setBaseResponseAttributes(span, response); + if (attributes.startTimeMs) { + recordNonStreamingTimingAttributes(span, attributes.startTimeMs, Date.now()); + } span.end(); this.nodeAttributes.delete(runId); @@ -502,7 +468,6 @@ export class LanggraphWrapper { // Set the active flag to prevent nested instrumentation (e.g., when invoke calls stream internally) const ctxWithSpan = trace.setSpan(context.active(), span); const ctxWithFlag = ctxWithSpan.setValue(LANGGRAPH_INSTRUMENTATION_ACTIVE, true); - enterWithContext(ctxWithFlag); return context.with(ctxWithFlag, async () => { { @@ -561,7 +526,6 @@ export class LanggraphWrapper { try { const ctxWithSpan = trace.setSpan(context.active(), span); const ctxWithFlag = ctxWithSpan.setValue(LANGGRAPH_INSTRUMENTATION_ACTIVE, true); - enterWithContext(ctxWithFlag); const streamingWrapper = new LanggraphStreamingWrapper(span, ctxWithFlag); return streamingWrapper.startStream( originalFunc, From a5fcc7d6455f36e3a1de7f351ecbb554a4632f14 Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Fri, 7 Aug 2026 14:21:39 +0530 Subject: [PATCH 16/20] [NET-1093] chore: Cleanup logs --- src/index.ts | 18 +--------- src/instrumentation/index.ts | 4 +-- src/instrumentation/langgraph/index.ts | 42 +++++------------------ src/instrumentation/langgraph/wrappers.ts | 23 ++++++++----- 4 files changed, 26 insertions(+), 61 deletions(-) diff --git a/src/index.ts b/src/index.ts index ea42421..2cfe47e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -235,23 +235,6 @@ export class Netra { } this._initialized = true; - Logger.info("Netra successfully initialized."); - - { - let pkgVersion = Config.LIBRARY_VERSION; - let pkgPath = "unknown"; - try { - const req = createRequire(import.meta.url); - pkgPath = req.resolve("../package.json"); - const pkg = req("../package.json"); - pkgVersion = pkg?.version || pkgVersion; - } catch { - // keep defaults - } - Logger.debug( - `SDK version=${pkgVersion} libraryVersion=${Config.LIBRARY_VERSION} build=langgraph-parenting-v3 packageJson=${pkgPath}`, - ); - } // Graceful shutdown logic const handleSignal = async (signal: string) => { @@ -287,6 +270,7 @@ export class Netra { // Wait for all async instrumentations to be ready await instrumentationsReady; + Logger.info("Netra successfully initialized."); } static async shutdown(): Promise { diff --git a/src/instrumentation/index.ts b/src/instrumentation/index.ts index 3142785..b560fce 100644 --- a/src/instrumentation/index.ts +++ b/src/instrumentation/index.ts @@ -125,7 +125,7 @@ function patchTraceloopLangchainCallbackHandler(): void { try { const mod = require("@traceloop/instrumentation-langchain"); - Logger.debug(`Loaded @traceloop/instrumentation-langchain via require from ${require.resolve("@traceloop/instrumentation-langchain")}`); + Logger.debug("Loaded @traceloop/instrumentation-langchain"); applyPatch(mod, "require"); return; } catch (e) { @@ -395,8 +395,6 @@ export function initInstrumentations( Logger.debug(` App Name: ${config.appName}`); Logger.debug(` OTLP Endpoint: ${config.otlpEndpoint || "(default - localhost:3002)"}`); Logger.debug(` API Key: ${config.apiKey ? "***" + config.apiKey.slice(-4) : "(not set)"}`); - Logger.debug(` Trace Content: ${config.traceContent}`); - Logger.debug(` Enable Scrubbing: ${config.enableScrubbing}`); // Initialize Traceloop SDK const traceloopOptions: InitializeOptions = { diff --git a/src/instrumentation/langgraph/index.ts b/src/instrumentation/langgraph/index.ts index 473053f..a662d49 100644 --- a/src/instrumentation/langgraph/index.ts +++ b/src/instrumentation/langgraph/index.ts @@ -32,48 +32,33 @@ function findModuleInCache(moduleName: string): any { } Logger.debug( `Module ${moduleName} not found in require.cache. Cache keys containing 'langgraph':`, - Object.keys(cache).filter(k => k.includes('langgraph')), + Object.keys(cache).filter((k) => k.includes('langgraph')), ); } return null; } async function resolveLanggraph(): Promise { + const moduleName = "@langchain/langgraph"; if (LanggraphClass) return LanggraphClass; try { // First, try to find the module in require.cache (already loaded by the app) // This ensures we patch the same module instance the app is using - let langgraphModule = findModuleInCache('@langchain/langgraph'); + let langgraphModule = findModuleInCache(moduleName); - if (langgraphModule) { - Logger.debug("Found @langchain/langgraph in require.cache (using app's module instance)"); - } else { + if (!langgraphModule) { // Fallback to dynamic import if not in cache - langgraphModule = await import("@langchain/langgraph"); - Logger.debug("Loaded @langchain/langgraph via dynamic import"); + langgraphModule = await import(moduleName); } - Logger.debug("LangGraph Module Exports:", Object.keys(langgraphModule)); - LanggraphClass = langgraphModule.CompiledStateGraph ?? langgraphModule.StateGraph; - Logger.debug("Resolved LanggraphClass:", !!LanggraphClass); - Logger.debug("LanggraphClass name:", LanggraphClass?.name); - Logger.debug( - "LanggraphClass.prototype keys:", - LanggraphClass?.prototype ? Object.getOwnPropertyNames(LanggraphClass.prototype) : "no prototype", - ); - Logger.debug("Has invoke on prototype:", !!LanggraphClass?.prototype?.invoke); - Logger.debug("Has stream on prototype:", !!LanggraphClass?.prototype?.stream); + LanggraphClass = + langgraphModule.CompiledStateGraph ?? langgraphModule.StateGraph; // Check prototype chain to find where invoke is defined - Logger.debug("Checking prototype chain for invoke location:"); let proto = LanggraphClass?.prototype; while (proto) { - const hasOwn = Object.getOwnPropertyNames(proto).includes('invoke'); - Logger.debug(` ${proto.constructor?.name}: hasOwnProperty('invoke')=${hasOwn}`); - if (hasOwn) { - Logger.debug(` -> invoke is defined on: ${proto.constructor?.name}`); - break; - } + const hasOwn = Object.getOwnPropertyNames(proto).includes("invoke"); + if (hasOwn) break; proto = Object.getPrototypeOf(proto); } @@ -160,8 +145,6 @@ export class NetraLanggraphInstrumentor { return; } - Logger.debug(`Found invoke on prototype: ${targetProto.constructor?.name}`); - const originalInvoke = targetProto.invoke; originalMethods.set("langgraph.graph.invoke", originalInvoke); // Store the target prototype for uninstrumentation @@ -188,9 +171,6 @@ export class NetraLanggraphInstrumentor { // Add marker to identify patched method (patchedInvoke as any).__netra_patched = true; targetProto.invoke = patchedInvoke; - - Logger.debug(`Successfully instrumented LangGraph invoke method on ${targetProto.constructor?.name}`); - Logger.debug(`Patched Pregel class identity:`, targetProto.constructor); } catch (error) { Logger.error(`Failed to instrument langgraph invoke: ${error}`); } @@ -211,8 +191,6 @@ export class NetraLanggraphInstrumentor { return; } - Logger.debug(`Found stream on prototype: ${targetProto.constructor?.name}`); - const originalStream = targetProto.stream; originalMethods.set("langgraph.graph.stream", originalStream); // Store the target prototype for uninstrumentation @@ -236,8 +214,6 @@ export class NetraLanggraphInstrumentor { ...rest, ); }; - - Logger.debug(`Successfully instrumented LangGraph stream method on ${targetProto.constructor?.name}`); } catch (error) { Logger.error(`Failed to instrument langgraph stream: ${error}`); } diff --git a/src/instrumentation/langgraph/wrappers.ts b/src/instrumentation/langgraph/wrappers.ts index 8522834..3998fca 100644 --- a/src/instrumentation/langgraph/wrappers.ts +++ b/src/instrumentation/langgraph/wrappers.ts @@ -377,14 +377,19 @@ class LanggraphStreamingWrapper implements AsyncIterable { config?: RunnableConfig, ...rest: any[] ) { - this.iterable = await originalFunc.call(instance, input, config, ...rest); + const spanContext = trace.setSpan(this.rootContext, this.rootSpan); + this.iterable = await context.with(spanContext, () => + originalFunc.call(instance, input, config, ...rest), + ); return this; } async *[Symbol.asyncIterator]() { const spanContext = trace.setSpan(this.rootContext, this.rootSpan); try { - const iterator = await this.iterable[Symbol.asyncIterator](); + const iterator = await context.with(spanContext, () => + this.iterable[Symbol.asyncIterator](), + ); while (true) { let result: any; await context.with(spanContext, async () => { @@ -527,12 +532,14 @@ export class LanggraphWrapper { const ctxWithSpan = trace.setSpan(context.active(), span); const ctxWithFlag = ctxWithSpan.setValue(LANGGRAPH_INSTRUMENTATION_ACTIVE, true); const streamingWrapper = new LanggraphStreamingWrapper(span, ctxWithFlag); - return streamingWrapper.startStream( - originalFunc, - instance, - input, - updatedConfig, - ...rest, + return await context.with(ctxWithFlag, () => + streamingWrapper.startStream( + originalFunc, + instance, + input, + updatedConfig, + ...rest, + ), ); } catch (error) { span.recordException(error as Error); From 1276b71e048d37756d18030ed1c7f240ef597210 Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Fri, 7 Aug 2026 14:22:56 +0530 Subject: [PATCH 17/20] [NET-1093] chore: Update peer dependencies list to make them optional --- package.json | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 890925f..9692bc3 100644 --- a/package.json +++ b/package.json @@ -79,40 +79,49 @@ "openai": "^4.0.0 || ^5.0.0 || ^6.0.0" }, "peerDependenciesMeta": { - "@opentelemetry/instrumentation": { + "@anthropic-ai/sdk": { "optional": true }, - "@opentelemetry/instrumentation-http": { + "@google/genai": { "optional": true }, - "@opentelemetry/instrumentation-express": { + "@google/generative-ai": { "optional": true }, - "@opentelemetry/instrumentation-undici": { + "@langchain/langchain": { "optional": true }, - "@prisma/instrumentation": { + "@langchain/langgraph": { "optional": true }, - "openai": { + "@langchain/ollama": { "optional": true }, - "groq-sdk": { + "@mistralai/mistralai": { "optional": true }, - "@mistralai/mistralai": { + "@openai/agents": { "optional": true }, - "@google/generative-ai": { + "@opentelemetry/instrumentation": { "optional": true }, - "@google/genai": { + "@opentelemetry/instrumentation-express": { "optional": true }, - "@anthropic-ai/sdk": { + "@opentelemetry/instrumentation-http": { "optional": true }, - "@openai/agents": { + "@opentelemetry/instrumentation-undici": { + "optional": true + }, + "@prisma/instrumentation": { + "optional": true + }, + "groq-sdk": { + "optional": true + }, + "openai": { "optional": true } }, From 2173b40018a9a9ec0b2ce5b5588c1e21ef746c45 Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Fri, 7 Aug 2026 14:30:32 +0530 Subject: [PATCH 18/20] [NET-1093] chore: Code cleanup --- package.json | 3 --- src/instrumentation/langgraph/wrappers.ts | 13 ++++++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 9692bc3..454e74b 100644 --- a/package.json +++ b/package.json @@ -88,9 +88,6 @@ "@google/generative-ai": { "optional": true }, - "@langchain/langchain": { - "optional": true - }, "@langchain/langgraph": { "optional": true }, diff --git a/src/instrumentation/langgraph/wrappers.ts b/src/instrumentation/langgraph/wrappers.ts index 3998fca..f936811 100644 --- a/src/instrumentation/langgraph/wrappers.ts +++ b/src/instrumentation/langgraph/wrappers.ts @@ -45,6 +45,7 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { private nodeAttributes: Map> = new Map(); private runStack: string[] = []; private inferredParents: Map = new Map(); + private streamedRuns: Set = new Set(); constructor( private tracer: Tracer, @@ -208,6 +209,14 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { }); } + async handleLLMNewToken( + _token: string, + _idx: { prompt: number; completion: number }, + runId: string, + ) { + this.streamedRuns.add(runId); + } + async handleLLMEnd( output: LLMResult, runId: string, @@ -235,12 +244,13 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { attributes.extraParams, ); setBaseResponseAttributes(span, response); - if (attributes.startTimeMs) { + if (attributes.startTimeMs && !this.streamedRuns.has(runId)) { recordNonStreamingTimingAttributes(span, attributes.startTimeMs, Date.now()); } span.end(); this.nodeAttributes.delete(runId); + this.streamedRuns.delete(runId); } async handleLLMError( @@ -268,6 +278,7 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { span.end(); this.nodeAttributes.delete(runId); + this.streamedRuns.delete(runId); } async handleToolStart( From 3f189e9e31d95fbfa0384a3b1ed65ae242fecc65 Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Fri, 7 Aug 2026 15:27:38 +0530 Subject: [PATCH 19/20] [NET-1093] chore: Resolve bugs --- src/instrumentation/anthropic/wrappers.ts | 61 ++++++++++++----------- src/instrumentation/langgraph/wrappers.ts | 14 +++--- src/instrumentation/openai/wrappers.ts | 13 +++-- src/utils/span-timing.ts | 4 -- 4 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/instrumentation/anthropic/wrappers.ts b/src/instrumentation/anthropic/wrappers.ts index 0562e68..41642aa 100644 --- a/src/instrumentation/anthropic/wrappers.ts +++ b/src/instrumentation/anthropic/wrappers.ts @@ -248,7 +248,9 @@ class MessageStreamWrapper { return function (event?: string) { target.listenerMap = new WeakMap(); const result = method.call(target.messageStream, event); - if (!event || event === "text") { + if (!event) { + target.attachSafetyNetListeners(); + } else if (event === "text") { target.messageStream.on("text", target.ttftListener); } return result; @@ -303,36 +305,35 @@ class MessageStreamWrapper { }); } + private attachSafetyNetListeners(): void { + this.ttftListener = (data: any) => { + if (data) this.tokenTracker.markFirstToken(); + }; + this.messageStream.on("text", this.ttftListener); + + this.messageStream.on("end", () => { + if (!this.completionPending) { + this.finalizeSpanOnce(SpanStatusCode.OK); + } + }); + this.messageStream.on("error", (err: any) => { + if (err && !this.spanFinalized) { + this.span.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : String(err), + }); + this.span.recordException( + err instanceof Error ? err : new Error(String(err)), + ); + } + this.finalizeSpanOnce(SpanStatusCode.ERROR); + }); + } + private registerSafetyNetListeners(): void { try { if (typeof this.messageStream?.on !== "function") return; - - // Capture TTFT directly from the raw stream so it is recorded - // even when the consumer (e.g. tool runner) never registers its - // own "text" listener through our proxy. Stored on the instance - // so removeAllListeners can re-attach it. - this.ttftListener = (data: any) => { - if (data) this.tokenTracker.markFirstToken(); - }; - this.messageStream.on("text", this.ttftListener); - - this.messageStream.on("end", () => { - if (!this.completionPending) { - this.finalizeSpanOnce(SpanStatusCode.OK); - } - }); - this.messageStream.on("error", (err: any) => { - if (err && !this.spanFinalized) { - this.span.setStatus({ - code: SpanStatusCode.ERROR, - message: err instanceof Error ? err.message : String(err), - }); - this.span.recordException( - err instanceof Error ? err : new Error(String(err)), - ); - } - this.finalizeSpanOnce(SpanStatusCode.ERROR); - }); + this.attachSafetyNetListeners(); } catch (e) { Logger.error( "netra.instrumentation.anthropic: safety net listener registration failed", @@ -510,7 +511,9 @@ function anthropicWrapper( "llm.response.duration", (endTime - startTime) / 1000, ); - recordNonStreamingTimingAttributes(span, startTime, endTime); + if (requestType !== "batches") { + recordNonStreamingTimingAttributes(span, startTime, endTime); + } }, finalize: (status) => { const hasStreamData = diff --git a/src/instrumentation/langgraph/wrappers.ts b/src/instrumentation/langgraph/wrappers.ts index f936811..de7652c 100644 --- a/src/instrumentation/langgraph/wrappers.ts +++ b/src/instrumentation/langgraph/wrappers.ts @@ -14,17 +14,12 @@ import { } from "@opentelemetry/api"; import { Logger } from "../../logger"; +import { recordNonStreamingTimingAttributes } from "../../utils/span-timing"; import { defineHidden, setResponseAttributes as setBaseResponseAttributes, shouldSuppressInstrumentation, } from "../utils"; -import { recordNonStreamingTimingAttributes } from "../../utils/span-timing"; - -// Context key to track if we're inside a LangGraph instrumented call -// This prevents double-instrumentation when invoke internally calls stream -const LANGGRAPH_INSTRUMENTATION_ACTIVE = createContextKey("netra.langgraph.active"); - import { NetraLanggraphAttributes, setChainInputAttributes, @@ -35,6 +30,10 @@ import { setToolAttributes, } from "./utils"; +// Context key to track if we're inside a LangGraph instrumented call +// This prevents double-instrumentation when invoke internally calls stream +const LANGGRAPH_INSTRUMENTATION_ACTIVE = createContextKey("netra.langgraph.active"); + type AnyFunc = (...args: any[]) => any; type AsyncIterableFunc = (...args: any[]) => Promise>; @@ -244,6 +243,8 @@ class NetraLanggraphCallbackHandler extends BaseCallbackHandler { attributes.extraParams, ); setBaseResponseAttributes(span, response); + // Streaming TTFT is captured by the underlying provider instrumentation + // (e.g. OpenAI, Anthropic). We only record timing for non-streaming calls. if (attributes.startTimeMs && !this.streamedRuns.has(runId)) { recordNonStreamingTimingAttributes(span, attributes.startTimeMs, Date.now()); } @@ -555,6 +556,7 @@ export class LanggraphWrapper { } catch (error) { span.recordException(error as Error); span.end(); + throw error; } } } diff --git a/src/instrumentation/openai/wrappers.ts b/src/instrumentation/openai/wrappers.ts index 89a6213..ee3bfa2 100644 --- a/src/instrumentation/openai/wrappers.ts +++ b/src/instrumentation/openai/wrappers.ts @@ -19,6 +19,7 @@ import { } from "../utils"; import { setRequestAttributes, setResponseAttributes } from "./utils"; import { OpenAIRequestType, StreamResponse, WrapperFn } from "./types"; +import { Logger } from "../../logger"; const SPAN_NAMES: Record = { chat: "openai.chat", @@ -156,10 +157,14 @@ abstract class BaseStreamHandler { } protected finalizeSpan(code: SpanStatusCode): void { - setResponseAttributes( - this.span, - this.completeResponse as Record, - ); + try { + setResponseAttributes( + this.span, + this.completeResponse as Record, + ); + } catch { + Logger.debug('Failed to set response attributes'); + } this.span.setAttribute( "llm.response.duration", (Date.now() - this.startTime) / 1000, diff --git a/src/utils/span-timing.ts b/src/utils/span-timing.ts index 89905db..49eeda9 100644 --- a/src/utils/span-timing.ts +++ b/src/utils/span-timing.ts @@ -53,10 +53,6 @@ export class FirstTokenTracker { private readonly startTimeMs: number, ) {} - get recorded(): boolean { - return this._recorded; - } - markFirstToken(): void { if (this._recorded || !this.span.isRecording()) return; this._recorded = true; From 1d0eaa2d3f6a3a0569219c31b41070ce85cae2b3 Mon Sep 17 00:00:00 2001 From: Nithish-KV Date: Fri, 7 Aug 2026 16:09:45 +0530 Subject: [PATCH 20/20] [NET-1093] fix: Fix the async iterator result for langgraph instrumentation --- src/instrumentation/langgraph/wrappers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/instrumentation/langgraph/wrappers.ts b/src/instrumentation/langgraph/wrappers.ts index de7652c..b434193 100644 --- a/src/instrumentation/langgraph/wrappers.ts +++ b/src/instrumentation/langgraph/wrappers.ts @@ -410,7 +410,7 @@ class LanggraphStreamingWrapper implements AsyncIterable { if (result.done) break; const value = result?.value ?? {}; this.output = { ...this.output, ...value }; - yield result; + yield value; } this.rootSpan.setAttribute( NetraLanggraphAttributes.entityOutput,