diff --git a/docs/storybook/stories/agents-ui/AgentAudioVisualizerAura.stories.tsx b/docs/storybook/stories/agents-ui/AgentAudioVisualizerAura.stories.tsx index cf8b03508..049e5368c 100644 --- a/docs/storybook/stories/agents-ui/AgentAudioVisualizerAura.stories.tsx +++ b/docs/storybook/stories/agents-ui/AgentAudioVisualizerAura.stories.tsx @@ -4,6 +4,7 @@ import { AgentSessionProvider } from '../../.storybook/lk-decorators/AgentSessio import { AgentAudioVisualizerAura, AgentAudioVisualizerAuraProps } from '@livekit/agents-ui'; import { useTheme } from 'next-themes'; import { useAgent } from '@livekit/components-react'; +import { useSimulatedVolumeBands } from './useSimulatedVolumeBands'; export default { component: AgentAudioVisualizerAura, @@ -64,3 +65,25 @@ export default { export const Default: StoryObj = { args: {}, }; + +// Demonstrates the `volume` override prop with a simulated speech waveform instead of +// live audio. +export const OverrideVolume: StoryObj = { + args: { + state: 'speaking', + }, + render: (args: AgentAudioVisualizerAuraProps) => { + const { microphoneTrack } = useAgent(); + const { resolvedTheme = 'dark' } = useTheme(); + const [volume] = useSimulatedVolumeBands(1); + + return ( + + ); + }, +}; diff --git a/docs/storybook/stories/agents-ui/AgentAudioVisualizerBar.stories.tsx b/docs/storybook/stories/agents-ui/AgentAudioVisualizerBar.stories.tsx index c63ad9e42..710d4094d 100644 --- a/docs/storybook/stories/agents-ui/AgentAudioVisualizerBar.stories.tsx +++ b/docs/storybook/stories/agents-ui/AgentAudioVisualizerBar.stories.tsx @@ -3,6 +3,7 @@ import { StoryObj } from '@storybook/react-vite'; import { useAgent } from '@livekit/components-react'; import { AgentSessionProvider } from '../../.storybook/lk-decorators/AgentSessionProvider'; import { AgentAudioVisualizerBar, AgentAudioVisualizerBarProps } from '@livekit/agents-ui'; +import { useSimulatedVolumeBands } from './useSimulatedVolumeBands'; export default { component: AgentAudioVisualizerBar, @@ -54,8 +55,20 @@ export const Default: StoryObj = { args: {}, }; -export const Demo1: StoryObj = { +// Demonstrates the `volumeBands` override prop with a simulated speech waveform instead +// of live audio. The array length doesn't need to match `barCount` — it's trimmed or +// padded automatically. +export const OverrideVolumeBands: StoryObj = { args: { - color: '#F9B11F', + state: 'speaking', + barCount: 5, + }, + render: (args: AgentAudioVisualizerBarProps) => { + const { microphoneTrack } = useAgent(); + const volumeBands = useSimulatedVolumeBands(args.barCount ?? 5); + + return ( + + ); }, }; diff --git a/docs/storybook/stories/agents-ui/AgentAudioVisualizerGrid.stories.tsx b/docs/storybook/stories/agents-ui/AgentAudioVisualizerGrid.stories.tsx index 4dcd7ade3..430509d6f 100644 --- a/docs/storybook/stories/agents-ui/AgentAudioVisualizerGrid.stories.tsx +++ b/docs/storybook/stories/agents-ui/AgentAudioVisualizerGrid.stories.tsx @@ -8,6 +8,7 @@ import { } from '@livekit/agents-ui'; import { useAgent } from '@livekit/components-react'; import { cn } from '@/lib/utils'; +import { useSimulatedVolumeBands } from './useSimulatedVolumeBands'; export default { component: AgentAudioVisualizerGrid, @@ -135,3 +136,21 @@ export const Demo4: StoryObj = { ), }, }; + +// Demonstrates the `volumeBands` override prop with a simulated speech waveform instead +// of live audio. The array length doesn't need to match `columnCount` — it's trimmed or +// padded automatically. +export const OverrideVolumeBands: StoryObj = { + args: { + state: 'speaking', + columnCount: 9, + }, + render: (args: AgentAudioVisualizerGridProps) => { + const { microphoneTrack } = useAgent(); + const volumeBands = useSimulatedVolumeBands(args.columnCount ?? 9); + + return ( + + ); + }, +}; diff --git a/docs/storybook/stories/agents-ui/AgentAudioVisualizerRadial.stories.tsx b/docs/storybook/stories/agents-ui/AgentAudioVisualizerRadial.stories.tsx index f25785c75..cd793c2b9 100644 --- a/docs/storybook/stories/agents-ui/AgentAudioVisualizerRadial.stories.tsx +++ b/docs/storybook/stories/agents-ui/AgentAudioVisualizerRadial.stories.tsx @@ -3,6 +3,7 @@ import { StoryObj } from '@storybook/react-vite'; import { AgentSessionProvider } from '../../.storybook/lk-decorators/AgentSessionProvider'; import { AgentAudioVisualizerRadial, AgentAudioVisualizerRadialProps } from '@livekit/agents-ui'; import { useAgent } from '@livekit/components-react'; +import { useSimulatedVolumeBands } from './useSimulatedVolumeBands'; export default { component: AgentAudioVisualizerRadial, @@ -57,3 +58,25 @@ export default { export const Default: StoryObj = { args: {}, }; + +// Demonstrates the `volumeBands` override prop with a simulated speech waveform instead +// of live audio. The array length doesn't need to match `barCount` — it's trimmed or +// padded automatically. +export const OverrideVolumeBands: StoryObj = { + args: { + state: 'speaking', + barCount: 24, + }, + render: (args: AgentAudioVisualizerRadialProps) => { + const { microphoneTrack } = useAgent(); + const volumeBands = useSimulatedVolumeBands(args.barCount ?? 24); + + return ( + + ); + }, +}; diff --git a/docs/storybook/stories/agents-ui/AgentAudioVisualizerWave.stories.tsx b/docs/storybook/stories/agents-ui/AgentAudioVisualizerWave.stories.tsx index e9df7cf65..ceb754d95 100644 --- a/docs/storybook/stories/agents-ui/AgentAudioVisualizerWave.stories.tsx +++ b/docs/storybook/stories/agents-ui/AgentAudioVisualizerWave.stories.tsx @@ -3,6 +3,7 @@ import { StoryObj } from '@storybook/react-vite'; import { AgentSessionProvider } from '../../.storybook/lk-decorators/AgentSessionProvider'; import { AgentAudioVisualizerWave, AgentAudioVisualizerWaveProps } from '@livekit/agents-ui'; import { useAgent } from '@livekit/components-react'; +import { useSimulatedVolumeBands } from './useSimulatedVolumeBands'; export default { component: AgentAudioVisualizerWave, @@ -63,3 +64,17 @@ export default { export const Default: StoryObj = { args: {}, }; + +// Demonstrates the `volume` override prop with a simulated speech waveform instead of +// live audio. +export const OverrideVolume: StoryObj = { + args: { + state: 'speaking', + }, + render: (args: AgentAudioVisualizerWaveProps) => { + const { microphoneTrack } = useAgent(); + const [volume] = useSimulatedVolumeBands(1); + + return ; + }, +}; diff --git a/docs/storybook/stories/agents-ui/useSimulatedVolumeBands.ts b/docs/storybook/stories/agents-ui/useSimulatedVolumeBands.ts new file mode 100644 index 000000000..5e3a4df25 --- /dev/null +++ b/docs/storybook/stories/agents-ui/useSimulatedVolumeBands.ts @@ -0,0 +1,140 @@ +'use client'; + +import { useEffect, useState } from 'react'; + +const TICK_MS = 16; +const FRAME_CYCLE = 240; +const MIN_V = 0.4; +const MAX_V = 1.0; +const RANGE = MAX_V - MIN_V; +const ATTACK = 0.6; +const DECAY = 0.1; +const BELL_FLOOR = 0.1; +const SPEECH_TICKS = [90, 200]; +const PAUSE_TICKS = [18, 55]; +const NOTIFY_EVERY = 2; +const RHYTHM_BEATS = 12; +const PHI = 1.618033988749; + +type VolumeListener = (volumes: number[]) => void; + +let frameIndex = 0; +let interval: ReturnType | undefined; +let speechScale = 1; +let speechTicksLeft = 120; +let speechSeed = 12345; + +const listenersByBarCount = new Map>(); +const volumesByBarCount = new Map(); +const cache = new Map< + number, + { phases: number[]; baselines: number[]; variation: number[]; bell: number[] } +>(); + +function nextDuration(isPause: boolean): number { + speechSeed = (speechSeed * 1103515245 + 12345) >>> 0; + const [min, max] = (isPause ? PAUSE_TICKS : SPEECH_TICKS) as [number, number]; + return min + (speechSeed % (max - min + 1)); +} + +function getBarConfig(barCount: number) { + let c = cache.get(barCount); + if (c) return c; + const n = barCount; + const last = Math.max(1, n - 1); + const phases = Array.from({ length: n }, (_, i) => (i * 2.1 + n * 0.7) * Math.PI); + const baselines = Array.from({ length: n }, (_, i) => 0.08 + 0.25 * ((i * PHI) % 1)); + const variation = + n <= 1 + ? [1] + : Array.from({ length: n }, (_, i) => 0.25 + 0.75 * (1 - Math.sin((Math.PI * i) / last))); + const bell = Array.from({ length: n }, (_, i) => { + const x = i / last; + return BELL_FLOOR + (1 - BELL_FLOOR) * Math.pow(Math.sin(Math.PI * x), 1.5); + }); + c = { phases, baselines, variation, bell }; + cache.set(barCount, c); + return c; +} + +function generateVolumes(barCount: number, prev: number[]): number[] { + const t = frameIndex / FRAME_CYCLE; + const phaseBase = t * Math.PI * 2; + const { phases, baselines, variation, bell } = getBarConfig(barCount); + let rhythm = 1; + if (speechScale > 0.5) { + const beat = t * RHYTHM_BEATS * Math.PI * 2; + const wobble = 0.52 * Math.sin(t * 5 * Math.PI * 2); + const swing = 0.8 * Math.sin(beat * 1.31); + const wave = Math.max(-1, Math.min(1, Math.sin(beat + wobble) + swing)); + rhythm = 0.6 + 0.4 * (0.5 + 0.5 * wave); + } + + return Array.from({ length: barCount }, (_, i) => { + const phase = phaseBase + (phases[i] ?? 0); + const h = + 0.45 + + 0.3 * Math.sin(phase * 1.7) + + 0.2 * Math.sin(phase * 3.2) + + 0.12 * Math.sin(phase * 5.1) + + (baselines[i] ?? 0.2); + const sharp = Math.pow(Math.min(1, Math.max(0, h)), 0.5); + const activity = 0.5 + (sharp - 0.5) * (variation[i] ?? 1); + const target = (MIN_V + activity * RANGE * (bell[i] ?? 1)) * speechScale * rhythm; + const p = prev[i] ?? 0; + const f = target > p ? ATTACK : DECAY; + return p + (target - p) * f; + }); +} + +function tick() { + frameIndex = (frameIndex + 1) % FRAME_CYCLE; + + if (--speechTicksLeft <= 0) { + const toPause = speechScale > 0.5; + speechScale = toPause ? 0 : 1; + speechTicksLeft = nextDuration(toPause); + } + + const notify = frameIndex % NOTIFY_EVERY === 0; + for (const [barCount, listeners] of listenersByBarCount.entries()) { + if (listeners.size === 0) continue; + const next = generateVolumes(barCount, volumesByBarCount.get(barCount) ?? []); + volumesByBarCount.set(barCount, next); + if (notify) for (const fn of listeners) fn(next); + } +} + +function subscribe(barCount: number, listener: VolumeListener): () => void { + const set = listenersByBarCount.get(barCount) ?? new Set(); + set.add(listener); + listenersByBarCount.set(barCount, set); + + const existing = volumesByBarCount.get(barCount); + if (existing) listener(existing); + else { + const init = generateVolumes(barCount, []); + volumesByBarCount.set(barCount, init); + listener(init); + } + + if (!interval) interval = setInterval(tick, TICK_MS); + + return () => { + const s = listenersByBarCount.get(barCount); + if (!s) return; + s.delete(listener); + if (s.size === 0) listenersByBarCount.delete(barCount); + const hasAny = [...listenersByBarCount.values()].some((set) => set.size > 0); + if (!hasAny && interval) { + clearInterval(interval); + interval = undefined; + } + }; +} + +export function useSimulatedVolumeBands(barCount: number) { + const [volumes, setVolumes] = useState([]); + useEffect(() => subscribe(barCount, setVolumes), [barCount]); + return volumes; +} diff --git a/packages/shadcn/components/agents-ui/agent-audio-visualizer-aura.tsx b/packages/shadcn/components/agents-ui/agent-audio-visualizer-aura.tsx index 77496ea7a..c01e39cc3 100644 --- a/packages/shadcn/components/agents-ui/agent-audio-visualizer-aura.tsx +++ b/packages/shadcn/components/agents-ui/agent-audio-visualizer-aura.tsx @@ -106,21 +106,21 @@ vec2 turb(vec2 pos, float t, float it) { mat2 rotation = mat2(0.6, -0.25, 0.25, 0.9); // Secondary rotation applied each iteration (approx 53 degree rotation) mat2 layerRotation = mat2(0.6, -0.8, 0.8, 0.6); - + float frequency = mix(2.0, 15.0, uFrequency); float amplitude = uAmplitude; float frequencyGrowth = 1.4; float animTime = t * 0.1 * uSpeed; - + const int LAYERS = 4; for(int i = 0; i < LAYERS; i++) { // Calculate wave displacement for this layer vec2 rotatedPos = pos * rotation; vec2 wave = sin(frequency * rotatedPos + float(i) * animTime + it); - + // Apply displacement along rotation direction pos += (amplitude / frequency) * rotation[0] * wave; - + // Evolve parameters for next layer rotation *= layerRotation; amplitude *= mix(1.0, max(wave.x, wave.y), uVariance); @@ -134,12 +134,12 @@ const float ITERATIONS = 36.0; void mainImage(out vec4 fragColor, in vec2 fragCoord) { vec2 uv = fragCoord / iResolution.xy; - + vec3 pp = vec3(0.0); vec3 bloom = vec3(0.0); float t = iTime * 0.5; vec2 pos = uv - 0.5; - + vec2 prevPos = turb(pos, t, 0.0 - 1.0 / ITERATIONS); float spacing = mix(1.0, TAU, uSpacing); @@ -151,25 +151,25 @@ void mainImage(out vec4 fragColor, in vec2 fragCoord) { prevPos = st; float dynamicBlur = exp2(pd * 2.0 * 1.4426950408889634) - 1.0; float ds = smoothstep(0.0, uBlur * 0.05 + max(dynamicBlur * uSmoothing, 0.001), d); - + // Shift color based on iteration using uColorScale vec3 color = uColor; if(uColorShift > 0.01) { vec3 hsv = rgb2hsv(color); // Shift hue by iteration - hsv.x = fract(hsv.x + (1.0 - iter) * uColorShift * 0.3); + hsv.x = fract(hsv.x + (1.0 - iter) * uColorShift * 0.3); color = hsv2rgb(hsv); } - + float invd = 1.0 / max(d + dynamicBlur, 0.001); pp += (ds - 1.0) * color; bloom += clamp(invd, 0.0, 250.0) * color; } pp *= 1.0 / ITERATIONS; - + vec3 color; - + // Dark mode (default) if(uMode < 0.5) { // use bloom effect @@ -180,32 +180,32 @@ void mainImage(out vec4 fragColor, in vec2 fragCoord) { float alpha = luma(color) * uMix; fragColor = vec4(color * uMix, alpha); } - - // Light mode + + // Light mode else { // no bloom effect color = -pp; color += (randFibo(fragCoord).x - 0.5) / 255.0; - + // Preserve hue by tone mapping brightness only float brightness = length(color); vec3 direction = brightness > 0.0 ? color / brightness : color; - + // Reinhard on brightness float factor = 2.0; float mappedBrightness = (brightness * factor) / (1.0 + brightness * factor); color = direction * mappedBrightness; - + // Boost saturation to compensate for white background bleed-through // When alpha < 1.0, white bleeds through making colors look desaturated // So we increase saturation to maintain vibrant appearance float gray = dot(color, vec3(0.2, 0.5, 0.1)); float saturationBoost = 3.0; color = mix(vec3(gray), color, saturationBoost); - + // Clamp between 0-1 color = clamp(color, 0.0, 1.0); - + float alpha = mappedBrightness * clamp(uMix, 1.0, 2.0); fragColor = vec4(color, alpha); } @@ -392,6 +392,10 @@ export interface AgentAudioVisualizerAuraProps { * The audio track to visualize. Can be a local/remote audio track or a track reference. */ audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + /** + * Volume value (0-1) to use instead of the value computed from the audioTrack. + */ + volume?: number; } /** @@ -416,6 +420,7 @@ export function AgentAudioVisualizerAura({ color = DEFAULT_COLOR, colorShift = 0.05, audioTrack, + volume, themeMode, className, ref, @@ -426,6 +431,7 @@ export function AgentAudioVisualizerAura({ const { speed, scale, amplitude, frequency, brightness } = useAgentAudioVisualizerAura( state, audioTrack, + volume, ); return ( diff --git a/packages/shadcn/components/agents-ui/agent-audio-visualizer-bar.tsx b/packages/shadcn/components/agents-ui/agent-audio-visualizer-bar.tsx index 3f68603c5..d82c35a6a 100644 --- a/packages/shadcn/components/agents-ui/agent-audio-visualizer-bar.tsx +++ b/packages/shadcn/components/agents-ui/agent-audio-visualizer-bar.tsx @@ -17,7 +17,7 @@ import { useMultibandTrackVolume, } from '@livekit/components-react'; import { useAgentAudioVisualizerBarAnimator } from '@/hooks/agents-ui/use-agent-audio-visualizer-bar'; -import { cn } from '@/lib/utils'; +import { cn, normalizeVolumeBands } from '@/lib/utils'; function cloneSingleChild( children: ReactNode | ReactNode[], @@ -106,6 +106,11 @@ export interface AgentAudioVisualizerBarProps { * The audio track to visualize. Can be a local/remote audio track or a track reference. */ audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + /** + * Volume values (0-1) to use instead of the values computed from the audioTrack. + * The volumeBands.length should match barCount. + */ + volumeBands?: number[]; /** * Additional CSS class names to apply to the container. */ @@ -139,6 +144,7 @@ export function AgentAudioVisualizerBar({ color, barCount, audioTrack, + volumeBands, className, children, style, @@ -159,11 +165,14 @@ export function AgentAudioVisualizerBar({ } }, [barCount, size]); - const volumeBands = useMultibandTrackVolume(audioTrack, { + const multibandVolume = useMultibandTrackVolume(audioTrack, { bands: _barCount, loPass: 100, hiPass: 200, }); + const resolvedVolumeBands = volumeBands + ? normalizeVolumeBands(volumeBands, _barCount) + : multibandVolume; const sequencerInterval = useMemo(() => { switch (state) { @@ -187,8 +196,8 @@ export function AgentAudioVisualizerBar({ ); const bands = useMemo( - () => (state === 'speaking' ? volumeBands : new Array(_barCount).fill(0)), - [state, volumeBands, _barCount], + () => (state === 'speaking' ? resolvedVolumeBands : new Array(_barCount).fill(0)), + [state, resolvedVolumeBands, _barCount], ); if (children && Array.isArray(children)) { diff --git a/packages/shadcn/components/agents-ui/agent-audio-visualizer-grid.tsx b/packages/shadcn/components/agents-ui/agent-audio-visualizer-grid.tsx index 597639de1..d67acc4a3 100644 --- a/packages/shadcn/components/agents-ui/agent-audio-visualizer-grid.tsx +++ b/packages/shadcn/components/agents-ui/agent-audio-visualizer-grid.tsx @@ -21,7 +21,7 @@ import { type Coordinate, useAgentAudioVisualizerGridAnimator, } from '@/hooks/agents-ui/use-agent-audio-visualizer-grid'; -import { cn } from '@/lib/utils'; +import { cn, normalizeVolumeBands } from '@/lib/utils'; function cloneSingleChild( children: ReactNode | ReactNode[], @@ -206,6 +206,11 @@ export type AgentAudioVisualizerGridProps = GridOptions & { * The audio track to visualize. Can be a local/remote audio track or a track reference. */ audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + /** + * Volume values (0-1) to use instead of the values computed from the audioTrack. + * The volumeBands.length should match columnCount. + */ + volumeBands?: number[]; /** * Additional CSS class names to apply to the container. */ @@ -246,6 +251,7 @@ export function AgentAudioVisualizerGrid({ className, children, audioTrack, + volumeBands, style, ...props }: AgentAudioVisualizerGridProps & ComponentProps<'div'>) { @@ -257,11 +263,14 @@ export function AgentAudioVisualizerGrid({ interval, radius, ); - const volumeBands = useMultibandTrackVolume(audioTrack, { + const multibandVolume = useMultibandTrackVolume(audioTrack, { bands: columnCount, loPass: 100, hiPass: 200, }); + const resolvedVolumeBands = volumeBands + ? normalizeVolumeBands(volumeBands, columnCount) + : multibandVolume; if (children && Array.isArray(children)) { throw new Error('AgentAudioVisualizerGrid children must be a single element.'); @@ -284,7 +293,7 @@ export function AgentAudioVisualizerGrid({ interval={interval} rowCount={rowCount} columnCount={columnCount} - volumeBands={volumeBands} + volumeBands={resolvedVolumeBands} highlightedCoordinate={highlightedCoordinate} > {children ??
} diff --git a/packages/shadcn/components/agents-ui/agent-audio-visualizer-radial.tsx b/packages/shadcn/components/agents-ui/agent-audio-visualizer-radial.tsx index 8c6ca27ab..a7facf69f 100644 --- a/packages/shadcn/components/agents-ui/agent-audio-visualizer-radial.tsx +++ b/packages/shadcn/components/agents-ui/agent-audio-visualizer-radial.tsx @@ -8,7 +8,7 @@ import { type TrackReferenceOrPlaceholder, useMultibandTrackVolume, } from '@livekit/components-react'; -import { cn } from '@/lib/utils'; +import { cn, normalizeVolumeBands } from '@/lib/utils'; import { useAgentAudioVisualizerRadialAnimator } from '@/hooks/agents-ui/use-agent-audio-visualizer-radial'; export const AgentAudioVisualizerRadialVariants = cva( @@ -71,6 +71,11 @@ export interface AgentAudioVisualizerRadialProps { * The audio track to visualize. Can be a local/remote audio track or a track reference. */ audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + /** + * Volume values (0-1) to use instead of the values computed from the audioTrack. + * The volumeBands.length should match barCount. + */ + volumeBands?: number[]; /** * Additional CSS class names to apply to the container. */ @@ -101,6 +106,7 @@ export function AgentAudioVisualizerRadial({ radius, barCount, audioTrack, + volumeBands, className, style, ...props @@ -120,11 +126,14 @@ export function AgentAudioVisualizerRadial({ } }, [barCount, size]); - const volumeBands = useMultibandTrackVolume(audioTrack, { + const multibandVolume = useMultibandTrackVolume(audioTrack, { bands: _barCount, loPass: 100, hiPass: 200, }); + const resolvedVolumeBands = volumeBands + ? normalizeVolumeBands(volumeBands, _barCount) + : multibandVolume; const sequencerInterval = useMemo(() => { switch (state) { @@ -168,10 +177,6 @@ export function AgentAudioVisualizerRadial({ _barCount, sequencerInterval, ); - const bands = useMemo( - () => (audioTrack ? volumeBands : new Array(_barCount).fill(0)), - [audioTrack, volumeBands, _barCount], - ); const dotSize = useMemo(() => { return (distanceFromCenter * Math.PI) / _barCount; @@ -184,7 +189,7 @@ export function AgentAudioVisualizerRadial({ style={{ ...style, color } as CSSProperties} {...props} > - {bands.map((band, idx) => { + {resolvedVolumeBands.map((band, idx) => { const angle = (idx / _barCount) * Math.PI * 2; return ( diff --git a/packages/shadcn/components/agents-ui/agent-audio-visualizer-wave.tsx b/packages/shadcn/components/agents-ui/agent-audio-visualizer-wave.tsx index 25dd651e3..1d2190674 100644 --- a/packages/shadcn/components/agents-ui/agent-audio-visualizer-wave.tsx +++ b/packages/shadcn/components/agents-ui/agent-audio-visualizer-wave.tsx @@ -74,54 +74,54 @@ float oscilloscopeWave(float x, float centerX, float time) { float relativeX = x - centerX; float maxDistance = centerX; float distanceFromCenter = abs(relativeX); - + // Apply bell curve for amplitude attenuation float bell = bellCurve(distanceFromCenter, maxDistance); - + // Calculate wave with uniforms and bell curve attenuation float wave = sin(relativeX * uFrequency + time * uSpeed) * uAmplitude * bell; - + return wave; } void mainImage(out vec4 fragColor, in vec2 fragCoord) { vec2 uv = fragCoord / iResolution.xy; vec2 pos = uv - 0.5; - + // Calculate center and positions float centerX = 0.5; float centerY = 0.5; float x = uv.x; float y = uv.y; - + // Convert line width from pixels to UV space // Use the average of width and height to handle aspect ratio float pixelSize = 2.0 / (iResolution.x + iResolution.y); float lineWidthUV = uLineWidth * pixelSize; float smoothingUV = uSmoothing * pixelSize; - + // Find minimum distance to the wave by sampling nearby points // This gives us consistent line width without high-frequency artifacts const int NUM_SAMPLES = 50; // Must be const for GLSL loop float minDist = 1000.0; float sampleRange = 0.02; // Range to search for closest point - + for(int i = 0; i < NUM_SAMPLES; i++) { float offset = (float(i) / float(NUM_SAMPLES - 1) - 0.5) * sampleRange; float sampleX = x + offset; float waveY = centerY + oscilloscopeWave(sampleX, centerX, iTime); - + // Calculate distance from current pixel to this point on the wave vec2 wavePoint = vec2(sampleX, waveY); vec2 currentPoint = vec2(x, y); float dist = distance(currentPoint, wavePoint); - + minDist = min(minDist, dist); } - + // Solid line with smooth edges using minimum distance float line = smoothstep(lineWidthUV + smoothingUV, lineWidthUV - smoothingUV, minDist); - + vec3 color = uColor; if(abs(uColorShift) > 0.01) { // Keep the center 50% at base color, then ramp shift across outer 25% on each side. @@ -134,16 +134,16 @@ void mainImage(out vec4 fragColor, in vec2 fragCoord) { hsv.x = fract(hsv.x + edgeFactor * uColorShift * 0.3); color = hsv2rgb(hsv); } - + // Apply line intensity color *= line; - + // Add dithering for smoother gradients // color += (randFibo(fragCoord).x - 0.5) / 255.0; - + // Calculate alpha based on line intensity float alpha = line * uMix; - + fragColor = vec4(color * uMix, alpha); }`; @@ -289,6 +289,10 @@ export interface AgentAudioVisualizerWaveProps { * The audio track to visualize. Can be a local/remote audio track or a track reference. */ audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + /** + * Volume value (0-1) to use instead of the value computed from the audioTrack. + */ + volume?: number; /** * Additional CSS class names to apply to the container. */ @@ -322,6 +326,7 @@ export function AgentAudioVisualizerWave({ lineWidth, blur, audioTrack, + volume, className, ref, ...props @@ -344,6 +349,7 @@ export function AgentAudioVisualizerWave({ const { speed, amplitude, frequency, opacity } = useAgentAudioVisualizerWave({ state, audioTrack, + volume, }); return ( diff --git a/packages/shadcn/hooks/agents-ui/use-agent-audio-visualizer-aura.ts b/packages/shadcn/hooks/agents-ui/use-agent-audio-visualizer-aura.ts index 4fbc9abeb..5fb857a39 100644 --- a/packages/shadcn/hooks/agents-ui/use-agent-audio-visualizer-aura.ts +++ b/packages/shadcn/hooks/agents-ui/use-agent-audio-visualizer-aura.ts @@ -46,6 +46,7 @@ function useAnimatedValue(initialValue: T) { export function useAgentAudioVisualizerAura( state: AgentState | undefined, audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder, + volumeProp?: number, ) { const [speed, setSpeed] = useState(DEFAULT_SPEED); const { @@ -57,10 +58,11 @@ export function useAgentAudioVisualizerAura( const { value: frequency, animate: animateFrequency } = useAnimatedValue(DEFAULT_FREQUENCY); const { value: brightness, animate: animateBrightness } = useAnimatedValue(DEFAULT_BRIGHTNESS); - const volume = useTrackVolume(audioTrack as TrackReference, { + const trackVolume = useTrackVolume(audioTrack as TrackReference, { fftSize: 512, smoothingTimeConstant: 0.55, }); + const volume = volumeProp ?? trackVolume; useEffect(() => { switch (state) { diff --git a/packages/shadcn/hooks/agents-ui/use-agent-audio-visualizer-wave.ts b/packages/shadcn/hooks/agents-ui/use-agent-audio-visualizer-wave.ts index 0e037fa00..8e6b22f6e 100644 --- a/packages/shadcn/hooks/agents-ui/use-agent-audio-visualizer-wave.ts +++ b/packages/shadcn/hooks/agents-ui/use-agent-audio-visualizer-wave.ts @@ -38,21 +38,24 @@ function useAnimatedValue(initialValue: T) { interface UseAgentAudioVisualizerWaveAnimatorArgs { state?: AgentState; audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + volume?: number; } export function useAgentAudioVisualizerWave({ state, audioTrack, + volume: volumeProp, }: UseAgentAudioVisualizerWaveAnimatorArgs) { const [speed, setSpeed] = useState(DEFAULT_SPEED); const { value: amplitude, animate: animateAmplitude } = useAnimatedValue(DEFAULT_AMPLITUDE); const { value: frequency, animate: animateFrequency } = useAnimatedValue(DEFAULT_FREQUENCY); const { value: opacity, animate: animateOpacity } = useAnimatedValue(1.0); - const volume = useTrackVolume(audioTrack as TrackReference, { + const trackVolume = useTrackVolume(audioTrack as TrackReference, { fftSize: 512, smoothingTimeConstant: 0.55, }); + const volume = volumeProp ?? trackVolume; useEffect(() => { switch (state) { diff --git a/packages/shadcn/lib/utils.ts b/packages/shadcn/lib/utils.ts index 2819a830d..afba5effb 100644 --- a/packages/shadcn/lib/utils.ts +++ b/packages/shadcn/lib/utils.ts @@ -4,3 +4,15 @@ import { twMerge } from 'tailwind-merge'; export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)); } + +/** + * Resizes an array of per-band volume values to exactly `count` entries. + * Excess values are trimmed from the end; if there are too few, the last + * value is duplicated to fill the remainder. An empty array is padded with 0s. + */ +export function normalizeVolumeBands(bands: number[], count: number): number[] { + if (bands.length === count) return bands; + if (bands.length > count) return bands.slice(0, count); + const lastValue = bands[bands.length - 1] ?? 0; + return [...bands, ...new Array(count - bands.length).fill(lastValue)]; +} diff --git a/packages/shadcn/tests/agent-audio-visualizer-aura.test.tsx b/packages/shadcn/tests/agent-audio-visualizer-aura.test.tsx index 9b8802aa9..91c128e48 100644 --- a/packages/shadcn/tests/agent-audio-visualizer-aura.test.tsx +++ b/packages/shadcn/tests/agent-audio-visualizer-aura.test.tsx @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { render, screen, fireEvent } from '@testing-library/react'; import { AgentAudioVisualizerAura } from '@/components/agents-ui/agent-audio-visualizer-aura'; +import { useAgentAudioVisualizerAura } from '@/hooks/agents-ui/use-agent-audio-visualizer-aura'; vi.mock('@/hooks/agents-ui/use-agent-audio-visualizer-aura', () => ({ useAgentAudioVisualizerAura: vi.fn(() => ({ @@ -49,4 +50,10 @@ describe('AgentAudioVisualizerAura', () => { render(); expect(screen.getByTestId('aura-viz')).toHaveAttribute('data-lk-state', 'listening'); }); + + it('forwards volume to the underlying hook', () => { + render(); + + expect(useAgentAudioVisualizerAura).toHaveBeenCalledWith('speaking', undefined, 0.6); + }); }); diff --git a/packages/shadcn/tests/agent-audio-visualizer-bar.test.tsx b/packages/shadcn/tests/agent-audio-visualizer-bar.test.tsx index 0d48482dc..cdb0857f0 100644 --- a/packages/shadcn/tests/agent-audio-visualizer-bar.test.tsx +++ b/packages/shadcn/tests/agent-audio-visualizer-bar.test.tsx @@ -95,6 +95,49 @@ describe('AgentAudioVisualizerBar', () => { }); }); + it('uses volumeBands prop instead of the hook value when speaking', () => { + vi.mocked(LiveKitComponents.useMultibandTrackVolume).mockReturnValue([0.1, 0.1, 0.1]); + const { container } = render( + , + ); + const bars = container.querySelectorAll('[data-lk-index]'); + expect(bars[0]).toHaveStyle({ height: '100%' }); + expect(bars[1]).toHaveStyle({ height: '50%' }); + expect(bars[2]).toHaveStyle({ height: '0%' }); + }); + + it('still zeroes bars when not speaking even if volumeBands is supplied', () => { + const { container } = render( + , + ); + const bars = container.querySelectorAll('[data-lk-index]'); + bars.forEach((bar) => { + expect(bar).toHaveStyle({ height: '0%' }); + }); + }); + + it('trims excess volumeBands values to match barCount', () => { + const { container } = render( + , + ); + const bars = container.querySelectorAll('[data-lk-index]'); + expect(bars).toHaveLength(2); + expect(bars[0]).toHaveStyle({ height: '100%' }); + expect(bars[1]).toHaveStyle({ height: '50%' }); + }); + + it('pads volumeBands by duplicating the last value to match barCount', () => { + const { container } = render( + , + ); + const bars = container.querySelectorAll('[data-lk-index]'); + expect(bars).toHaveLength(4); + expect(bars[0]).toHaveStyle({ height: '100%' }); + expect(bars[1]).toHaveStyle({ height: '50%' }); + expect(bars[2]).toHaveStyle({ height: '50%' }); + expect(bars[3]).toHaveStyle({ height: '50%' }); + }); + it('throws when children is not a single element', () => { expect(() => render( diff --git a/packages/shadcn/tests/agent-audio-visualizer-grid.test.tsx b/packages/shadcn/tests/agent-audio-visualizer-grid.test.tsx index c3323427e..4f2548f58 100644 --- a/packages/shadcn/tests/agent-audio-visualizer-grid.test.tsx +++ b/packages/shadcn/tests/agent-audio-visualizer-grid.test.tsx @@ -104,6 +104,81 @@ describe('AgentAudioVisualizerGrid', () => { expect(cells[5]).toHaveAttribute('data-lk-highlighted', 'false'); }); + it('uses volumeBands prop instead of the hook value when speaking', () => { + mockUseMultibandTrackVolume.mockReturnValue([0, 0]); + + const { container } = render( + , + ); + const cells = container.querySelectorAll('[data-lk-index]'); + + expect(cells[0]).toHaveAttribute('data-lk-highlighted', 'true'); + expect(cells[1]).toHaveAttribute('data-lk-highlighted', 'false'); + expect(cells[2]).toHaveAttribute('data-lk-highlighted', 'true'); + expect(cells[3]).toHaveAttribute('data-lk-highlighted', 'true'); + expect(cells[4]).toHaveAttribute('data-lk-highlighted', 'true'); + expect(cells[5]).toHaveAttribute('data-lk-highlighted', 'false'); + }); + + it('trims excess volumeBands values to match columnCount', () => { + mockUseMultibandTrackVolume.mockReturnValue([0, 0]); + + const { container } = render( + , + ); + const cells = container.querySelectorAll('[data-lk-index]'); + + expect(cells).toHaveLength(6); + expect(cells[0]).toHaveAttribute('data-lk-highlighted', 'true'); + expect(cells[1]).toHaveAttribute('data-lk-highlighted', 'false'); + expect(cells[2]).toHaveAttribute('data-lk-highlighted', 'true'); + expect(cells[3]).toHaveAttribute('data-lk-highlighted', 'true'); + expect(cells[4]).toHaveAttribute('data-lk-highlighted', 'true'); + expect(cells[5]).toHaveAttribute('data-lk-highlighted', 'false'); + }); + + it('pads volumeBands by duplicating the last value to match columnCount', () => { + mockUseMultibandTrackVolume.mockReturnValue([0, 0, 0]); + + const { container } = render( + , + ); + const cells = container.querySelectorAll('[data-lk-index]'); + + expect(cells).toHaveLength(9); + cells.forEach((cell) => { + expect(cell).toHaveAttribute('data-lk-highlighted', 'true'); + }); + }); + + it('ignores volumeBands when not speaking (animator coordinate still applies)', () => { + mockUseAgentAudioVisualizerGridAnimator.mockReturnValue({ x: 1, y: 1 }); + + const { container } = render( + , + ); + const cells = container.querySelectorAll('[data-lk-index]'); + + expect(cells[4]).toHaveAttribute('data-lk-highlighted', 'true'); + expect(cells[0]).toHaveAttribute('data-lk-highlighted', 'false'); + expect(cells[8]).toHaveAttribute('data-lk-highlighted', 'false'); + }); + it('preserves custom child classes for Tailwind shadow control', () => { const { container } = render( diff --git a/packages/shadcn/tests/agent-audio-visualizer-radial.test.tsx b/packages/shadcn/tests/agent-audio-visualizer-radial.test.tsx index 207cd48ac..955f01097 100644 --- a/packages/shadcn/tests/agent-audio-visualizer-radial.test.tsx +++ b/packages/shadcn/tests/agent-audio-visualizer-radial.test.tsx @@ -1,10 +1,14 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { render, screen, fireEvent } from '@testing-library/react'; import { AgentAudioVisualizerRadial } from '@/components/agents-ui/agent-audio-visualizer-radial'; +import * as LiveKitComponents from '@livekit/components-react'; // Mock hooks vi.mock('@livekit/components-react', async () => { const actual = await vi.importActual('@livekit/components-react'); - return { ...actual, useMultibandTrackVolume: vi.fn(() => []) }; + return { + ...actual, + useMultibandTrackVolume: vi.fn((_track, options) => new Array(options?.bands ?? 0).fill(0)), + }; }); vi.mock('@/hooks/agents-ui/use-agent-audio-visualizer-radial', () => ({ @@ -63,4 +67,44 @@ describe('AgentAudioVisualizerRadial', () => { expect(bar).toHaveAttribute('data-lk-highlighted'); }); }); + + it('uses volumeBands prop instead of the hook value', () => { + vi.mocked(LiveKitComponents.useMultibandTrackVolume).mockReturnValue([0, 0]); + const { container } = render( + , + ); + const bars = container.querySelectorAll('[data-lk-index]'); + expect(bars[0]).not.toHaveStyle({ height: '0px' }); + expect(bars[1]).toHaveStyle({ height: '0px' }); + }); + + it('renders volumeBands even when audioTrack is absent', () => { + const { container } = render( + , + ); + const bars = container.querySelectorAll('[data-lk-index]'); + expect(bars[0]).not.toHaveStyle({ height: '0px' }); + expect(bars[1]).toHaveStyle({ height: '0px' }); + }); + + it('trims excess volumeBands values to match barCount', () => { + const { container } = render( + , + ); + const bars = container.querySelectorAll('[data-lk-index]'); + expect(bars).toHaveLength(2); + expect(bars[0]).not.toHaveStyle({ height: '0px' }); + expect(bars[1]).toHaveStyle({ height: '0px' }); + }); + + it('pads volumeBands by duplicating the last value to match barCount', () => { + const { container } = render( + , + ); + const bars = container.querySelectorAll('[data-lk-index]'); + expect(bars).toHaveLength(3); + expect(bars[0]).toHaveStyle({ height: '0px' }); + expect(bars[1]).not.toHaveStyle({ height: '0px' }); + expect(bars[2]).not.toHaveStyle({ height: '0px' }); + }); }); diff --git a/packages/shadcn/tests/agent-audio-visualizer-wave.test.tsx b/packages/shadcn/tests/agent-audio-visualizer-wave.test.tsx index a321b59e9..67d70997a 100644 --- a/packages/shadcn/tests/agent-audio-visualizer-wave.test.tsx +++ b/packages/shadcn/tests/agent-audio-visualizer-wave.test.tsx @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { render, screen, fireEvent } from '@testing-library/react'; import { AgentAudioVisualizerWave } from '@/components/agents-ui/agent-audio-visualizer-wave'; +import { useAgentAudioVisualizerWave } from '@/hooks/agents-ui/use-agent-audio-visualizer-wave'; const mockReactShaderToy = vi.fn(); @@ -69,4 +70,12 @@ describe('AgentAudioVisualizerWave', () => { expect(shaderProps.uniforms?.uColor?.value).toEqual([18 / 255, 52 / 255, 86 / 255]); expect(shaderProps.uniforms?.uColorShift?.value).toBe(0.25); }); + + it('forwards volume to the underlying hook', () => { + render(); + + expect(useAgentAudioVisualizerWave).toHaveBeenCalledWith( + expect.objectContaining({ volume: 0.6 }), + ); + }); }); diff --git a/packages/shadcn/tests/use-agent-audio-visualizer-aura.test.ts b/packages/shadcn/tests/use-agent-audio-visualizer-aura.test.ts index bb1652689..892e66329 100644 --- a/packages/shadcn/tests/use-agent-audio-visualizer-aura.test.ts +++ b/packages/shadcn/tests/use-agent-audio-visualizer-aura.test.ts @@ -3,6 +3,22 @@ import { renderHook, waitFor } from '@testing-library/react'; import { useAgentAudioVisualizerAura } from '@/hooks/agents-ui/use-agent-audio-visualizer-aura'; import type { AgentState } from '@livekit/components-react'; import * as LiveKitComponents from '@livekit/components-react'; +import { animate } from 'motion/react'; + +// The `useMotionValueEvent`/`useMotionValue` mocks below don't propagate changes back into +// React state, so `result.current.scale` never reflects volume-driven updates. Instead, +// assert on the target value passed to the mocked `animate()` call, which is where the +// resolved volume (from the `volume` prop or `useTrackVolume`) is actually used. +function findImmediateAnimateCall(target: number) { + return vi + .mocked(animate) + .mock.calls.find( + ([, value, transition]) => + typeof value === 'number' && + (transition as { duration?: number } | undefined)?.duration === 0 && + Math.abs(value - target) < 1e-9, + ); +} // Mock the @livekit/components-react hooks vi.mock('@livekit/components-react', async () => { @@ -133,6 +149,24 @@ describe('useAgentAudioVisualizerAura', () => { }); }); + describe('volume override', () => { + it('uses the volume prop instead of track volume', () => { + vi.mocked(LiveKitComponents.useTrackVolume).mockReturnValue(0); + + renderHook(() => useAgentAudioVisualizerAura('speaking', undefined, 0.4)); + + expect(findImmediateAnimateCall(0.2 + 0.2 * 0.4)).toBeDefined(); + }); + + it('falls back to track volume when volume is not supplied', () => { + vi.mocked(LiveKitComponents.useTrackVolume).mockReturnValue(0.5); + + renderHook(() => useAgentAudioVisualizerAura('speaking')); + + expect(findImmediateAnimateCall(0.2 + 0.2 * 0.5)).toBeDefined(); + }); + }); + describe('State Transitions', () => { it('updates values when state changes', async () => { const { result, rerender } = renderHook(({ state }) => useAgentAudioVisualizerAura(state), { diff --git a/packages/shadcn/tests/use-agent-audio-visualizer-wave.test.ts b/packages/shadcn/tests/use-agent-audio-visualizer-wave.test.ts index d87de1771..c6dff0d6e 100644 --- a/packages/shadcn/tests/use-agent-audio-visualizer-wave.test.ts +++ b/packages/shadcn/tests/use-agent-audio-visualizer-wave.test.ts @@ -3,6 +3,22 @@ import { renderHook, waitFor } from '@testing-library/react'; import { useAgentAudioVisualizerWave } from '@/hooks/agents-ui/use-agent-audio-visualizer-wave'; import type { AgentState } from '@livekit/components-react'; import * as LiveKitComponents from '@livekit/components-react'; +import { animate } from 'motion/react'; + +// The `useMotionValueEvent`/`useMotionValue` mocks below don't propagate changes back into +// React state, so `result.current.amplitude`/`frequency` never reflect volume-driven updates. +// Instead, assert on the target values passed to the mocked `animate()` call, which is where +// the resolved volume (from the `volume` prop or `useTrackVolume`) is actually used. +function findImmediateAnimateCall(target: number) { + return vi + .mocked(animate) + .mock.calls.find( + ([, value, transition]) => + typeof value === 'number' && + (transition as { duration?: number } | undefined)?.duration === 0 && + Math.abs(value - target) < 1e-9, + ); +} // Mock dependencies vi.mock('@livekit/components-react', async () => { @@ -153,6 +169,26 @@ describe('useAgentAudioVisualizerWave', () => { }); }); + describe('volume override', () => { + it('uses the volume prop instead of track volume', () => { + vi.mocked(LiveKitComponents.useTrackVolume).mockReturnValue(0); + + renderHook(() => useAgentAudioVisualizerWave({ state: 'speaking', volume: 0.4 })); + + expect(findImmediateAnimateCall(0.015 + 0.4 * 0.4)).toBeDefined(); + expect(findImmediateAnimateCall(20 + 60 * 0.4)).toBeDefined(); + }); + + it('falls back to track volume when volume is not supplied', () => { + vi.mocked(LiveKitComponents.useTrackVolume).mockReturnValue(0.5); + + renderHook(() => useAgentAudioVisualizerWave({ state: 'speaking' })); + + expect(findImmediateAnimateCall(0.015 + 0.4 * 0.5)).toBeDefined(); + expect(findImmediateAnimateCall(20 + 60 * 0.5)).toBeDefined(); + }); + }); + describe('Hook Stability', () => { it('returns stable values for same state', () => { const { result, rerender } = renderHook( diff --git a/packages/shadcn/tests/utils.test.ts b/packages/shadcn/tests/utils.test.ts new file mode 100644 index 000000000..aad1b8c6e --- /dev/null +++ b/packages/shadcn/tests/utils.test.ts @@ -0,0 +1,24 @@ +import { describe, it, expect } from 'vitest'; +import { normalizeVolumeBands } from '@/lib/utils'; + +describe('normalizeVolumeBands', () => { + it('returns the array unchanged when the length already matches', () => { + expect(normalizeVolumeBands([0.1, 0.2, 0.3], 3)).toEqual([0.1, 0.2, 0.3]); + }); + + it('trims excess trailing values', () => { + expect(normalizeVolumeBands([0.1, 0.2, 0.3, 0.4], 2)).toEqual([0.1, 0.2]); + }); + + it('pads by duplicating the last value', () => { + expect(normalizeVolumeBands([0.1, 0.2], 4)).toEqual([0.1, 0.2, 0.2, 0.2]); + }); + + it('pads a single-element array by duplicating it', () => { + expect(normalizeVolumeBands([0.5], 3)).toEqual([0.5, 0.5, 0.5]); + }); + + it('pads an empty array with 0s', () => { + expect(normalizeVolumeBands([], 3)).toEqual([0, 0, 0]); + }); +});