{ "slug": "voice-agent-online-monitoring", "title": "How to Monitor a Voice Agent in Production: Dashboard, Alerts and Runbook", "description": "How to monitor a voice agent in production: per-stage latency percentiles, transcription confidence, containment paired with a resolution signal, live false-interruption rate, a handoff trigger review, and call-audio retention, each with what to chart, alert on and do.", "kind": "sub", "order": 9, "target_query": "how to monitor a voice agent in production", "secondary_queries": [ "voice ai agent monitoring", "realtime voice ai observability" ], "tags": [ "voice", "monitoring", "observability", "alerting", "latency", "agents", "production" ], "published": "2026-10-12", "updated": "2026-10-12", "words": 1305, "estimated_tokens": 1736, "premium": false, "rights": { "access": "free", "note": "Editorial guides are always free and never part of the licensed corpus.", "license": "https://changegamer.ai/license.xml", "pricing": "https://changegamer.ai/api/pricing.json", "payment": "https://changegamer.ai/api/payment.json" }, "license": "https://changegamer.ai/license.xml", "citation": "ChangeGamer (2026-10-12). How to Monitor a Voice Agent in Production: Dashboard, Alerts and Runbook. ChangeGamer. https://changegamer.ai/articles/voice-agent-online-monitoring (updated 2026-10-12).", "bibtex": "@misc{changegamer_voice_agent_online_monitoring, title = {How to Monitor a Voice Agent in Production: Dashboard, Alerts and Runbook}, publisher = {ChangeGamer}, year = {2026}, url = {https://changegamer.ai/articles/voice-agent-online-monitoring}, note = {Updated 2026-10-12}}", "canonical": "https://changegamer.ai/articles/voice-agent-online-monitoring", "markdown": "https://changegamer.ai/articles/voice-agent-online-monitoring.md", "takeaways": [ "As the author's inference, a voice agent dashboard should chart latency percentiles per pipeline stage, not only end to end, because a drifting stage can hide inside a healthy-looking total until callers notice.", "The evaluating-voice-agents reference names transcription confidence, latency per stage and overall, and containment as the online signals, so those three are the minimum live dashboard for any voice agent.", "Containment alone can look healthy while callers give up, so a voice agent dashboard should pair it with a resolution or repeat-contact signal before anyone treats a rise as an improvement.", "Every voice agent alert should be tied to a named action, and as the author's inference thresholds should be set from the agent's own baseline, since the corpus gives no safe numeric alert levels.", "The corpus does not document voice-specific alerting, so the chart, alert and action pairings in this article are the author's inference and should be rehearsed with a deliberate fault before launch." ], "outline": [ { "depth": 2, "text": "What belongs on a voice agent dashboard?", "anchor": "what-belongs-on-a-voice-agent-dashboard", "url": "https://changegamer.ai/articles/voice-agent-online-monitoring#what-belongs-on-a-voice-agent-dashboard" }, { "depth": 2, "text": "How do you alert on per-stage latency percentiles?", "anchor": "how-do-you-alert-on-per-stage-latency-percentiles", "url": "https://changegamer.ai/articles/voice-agent-online-monitoring#how-do-you-alert-on-per-stage-latency-percentiles" }, { "depth": 2, "text": "What do you do when transcription confidence drops?", "anchor": "what-do-you-do-when-transcription-confidence-drops", "url": "https://changegamer.ai/articles/voice-agent-online-monitoring#what-do-you-do-when-transcription-confidence-drops" }, { "depth": 2, "text": "How do you read containment without fooling yourself?", "anchor": "how-do-you-read-containment-without-fooling-yourself", "url": "https://changegamer.ai/articles/voice-agent-online-monitoring#how-do-you-read-containment-without-fooling-yourself" }, { "depth": 2, "text": "How do you watch false interruptions on live calls?", "anchor": "how-do-you-watch-false-interruptions-on-live-calls", "url": "https://changegamer.ai/articles/voice-agent-online-monitoring#how-do-you-watch-false-interruptions-on-live-calls" }, { "depth": 2, "text": "How do you review the handoff trigger log?", "anchor": "how-do-you-review-the-handoff-trigger-log", "url": "https://changegamer.ai/articles/voice-agent-online-monitoring#how-do-you-review-the-handoff-trigger-log" }, { "depth": 2, "text": "How long should call audio be kept and how should it be redacted?", "anchor": "how-long-should-call-audio-be-kept-and-how-should-it-be-redacted", "url": "https://changegamer.ai/articles/voice-agent-online-monitoring#how-long-should-call-audio-be-kept-and-how-should-it-be-redacted" }, { "depth": 2, "text": "What does ChangeGamer know first-hand about monitoring a voice agent?", "anchor": "what-does-changegamer-know-first-hand-about-monitoring-a-voice-agent", "url": "https://changegamer.ai/articles/voice-agent-online-monitoring#what-does-changegamer-know-first-hand-about-monitoring-a-voice-agent" } ], "faq": [ { "question": "What should you monitor on a voice agent in production?", "answer": "Monitor latency percentiles per pipeline stage, transcription confidence, containment with a resolution signal, false-interruption rate and handoff triggers. The evaluating-voice-agents reference names the first three as online signals; the rest are the author's inference." }, { "question": "What latency alert threshold should a voice agent use?", "answer": "Alert on drift from your own measured baseline per stage, compared with the budget you set. The corpus cites a sub-600 ms end-to-end target but no per-stage alert level, so any fixed number in a vendor blog should be treated as a starting guess." }, { "question": "Is containment rate a good success metric for a voice agent?", "answer": "Not alone. The customer-support reference cited in the pillar cautions that a contained call includes callers who gave up, so pair containment with a repeat-contact or sampled resolution review before reading a rise as an improvement." }, { "question": "Do you need to record call audio to monitor a voice agent?", "answer": "As the author's inference, no: per-stage timings, transcripts with confidence and trigger logs support most monitoring without audio. Keep audio only for sampled or flagged calls, under a stated retention period and with redaction applied." } ], "body": "A voice agent needs a live dashboard, a short list of alerts and a runbook that says what to do when each one fires; as inference, the failures are audible to callers before a text error log shows anything. This article is the operator's version of the monitoring half of the [voice AI agent in production](/articles/voice-ai-agent-in-production) pillar. Metric definitions live in the [evaluating voice agents](/resources/evaluating-voice-agents) reference and are not restated here; the question here is what to put on a screen and what to do about it.\n\nLabels: \"corpus\" is a claim from a named reference, \"inference\" is the author's reasoning. The corpus names the signals but gives no alert levels, so every threshold below is \"set from your own baseline\".\n\n## What belongs on a voice agent dashboard?\n\nA voice agent dashboard needs five panels: latency by stage, transcription confidence, containment next to a resolution signal, false interruptions, and handoff triggers. The first, second and third panels come from the online signals the evaluating-voice-agents reference lists (corpus); the panel layout is inference.\n\n| Panel | Chart | Alert on | Action that follows |\n|---|---|---|---|\n| Stage latency | P50 and P95 per stage | Drift from baseline or budget | Find the stage, then the cause |\n| Transcription confidence | Distribution over calls | Shift in the distribution | Sample low-confidence calls |\n| Containment and resolution | Both on one axis | They diverge | Review sampled ended calls |\n| False interruptions | Rate per day | A step change | Revisit turn detection |\n| Handoff triggers | Count by category | A new or surging category | Read the logged calls |\n\nTie each alert to a named person and a first step, or delete it. An alert nobody can act on trains the team to ignore the next one.\n\n## How do you alert on per-stage latency percentiles?\n\nAlert when any stage drifts from its own baseline or when the total breaches the budget; the split and the timestamping method are in the [voice agent latency budget](/articles/voice-agent-latency-budget), and the per-stage alerting rule is inference.\n\n- **Chart:** each stage and the total, using that article's percentiles.\n- **Alert on:** a stage whose P95 moves away from its own recent baseline even if the total still passes, plus any sustained breach of the total.\n- **Action:** open a sampled slow call, read its trace to find the stage, then check the usual suspects: a tool call that stalled, a prompt that grew, or a vendor change. The [agent-observability reference](/resources/agent-observability) describes the trace-and-span model and says to attach exceptions to the span that failed, which is how a slow stage becomes findable.\n\nAs inference, charting only the total can hide a drifting stage, since stage changes may offset each other. For generic drift mechanics see [online agent metrics and drift monitoring](/articles/online-agent-metrics-and-drift-monitoring).\n\n## What do you do when transcription confidence drops?\n\nTreat a shift in the transcription confidence distribution as an early warning that the audio, not the model, has changed, and respond by sampling the low-confidence calls. The reference lists transcription confidence as an online signal (corpus); what a drop means is inference.\n\n- **Chart:** the distribution of per-call confidence, split by whatever you can observe cheaply, such as telephony route or region.\n- **Alert on:** a change in the distribution's shape, not a single low call.\n- **Action:** listen to a sample from the affected slice. A route-specific drop points at the audio path; a drop everywhere after a date points at a vendor or model change.\n\nTranscription confidence is the score your speech-to-text provider exposes; the corpus does not define it, so compare it only within one vendor and model version (inference). A mis-heard value that reaches a tool call is more costly than a mis-heard filler word, so also chart how often critical values such as order numbers were confirmed back, a design covered in [voice agent guardrails for spoken output](/articles/guardrails-for-spoken-output-in-voice-agents).\n\n## How do you read containment without fooling yourself?\n\nRead containment only next to a resolution signal, because a contained call includes callers who quit. The customer-support reference, as quoted in the pillar, cautions that a deflection count cannot separate resolution from abandonment (corpus); the pairing is the remedy.\n\n- **Chart:** containment and a repeat-contact rate for the same caller or topic on one shared time axis. If you cannot link repeat calls, chart a weekly sample of ended calls graded for whether the issue was resolved.\n- **Alert on:** containment rising while repeat contact also rises, or while sampled resolution falls.\n- **Action:** read the sampled calls that ended without a handoff. Check whether callers hung up mid-task, and whether a refusal or loop pushed them out.\n\nThe grade of a sampled call is a judgment, so write the rubric down once and reuse it, or the trend measures the grader. Judge-style rubrics are discussed in [agent observability and the production evaluation playbook](/articles/agent-observability-and-evaluation), but writing the rubric for your own sampled calls is a step that playbook does not spell out.\n\n## How do you watch false interruptions on live calls?\n\nChart false-interruption rate daily and alert on a step change, because a sudden jump usually follows a configuration, vendor or audio-path change. The reference names false-interruption rate and barge-in accuracy among end-to-end measures (corpus); the live version is inference.\n\n- **Chart:** the rate per day, plus count of agent turns cut off early in the utterance if your logs can show it (define the window yourself, inference).\n- **Alert on:** a step change aligned with a deploy date, not slow noise.\n- **Action:** replay affected calls through the test method in [how to test turn detection and barge-in](/articles/voice-turn-detection-and-barge-in-testing), then compare the failing audio with your offline set.\n\nLive data carries no ground truth on what the caller meant, so a person has to label a sample before the number is trusted. A rising rate with no deploy suggests the callers or their devices changed; add those clips to the offline set.\n\n## How do you review the handoff trigger log?\n\nReview the handoff trigger log weekly by category, because a new or growing category reveals a gap in the agent faster than any aggregate. Log contents and transfer mechanics are in [voice agent human handoff](/articles/voice-agent-human-handoff); only the review rules follow.\n\n1. **Chart** handoffs by trigger category.\n2. **Alert on** a category appearing for the first time or taking a growing share, and on any handoff with a missing payload field.\n3. **Act** by reading a handful of calls in that category and deciding whether to fix the agent, change the trigger, or accept it.\n\nTreat a falling handoff count with suspicion until the resolution panel agrees. Fewer transfers can mean the agent is better, or that callers can no longer reach a person.\n\n## How long should call audio be kept and how should it be redacted?\n\nKeep call audio only for sampled or flagged calls, for a retention period you state in writing, with redaction applied before anyone reviews it. That is inference; the pillar says to decide retention before launch, and the redaction method is in [redacting sensitive data from agent traces](/articles/redacting-sensitive-data-from-agent-traces).\n\n## What does ChangeGamer know first-hand about monitoring a voice agent?\n\nNothing about operating one: as of October 2026 ChangeGamer runs no voice agent and has no call data, so no figure above is measured. The one verifiable fact is about its own build: `node scripts/verify-invariants.mjs`, run in `changegamer/` after `npm run build`, prints \"All invariants hold.\" when its checks pass, and an alert that can be checked as mechanically as that one is the model to copy: it passes or fails with no judgment call. For the full decision guide, return to the [voice agents cluster](/articles/voice-ai-agent-in-production), and for the realtime vendor landscape see the [voice and realtime agents](/resources/voice-realtime-agents) reference.", "cluster": { "id": "voice-agents-in-production", "title": "Voice agents in production", "description": "How to build and run a live voice AI agent — architecture choice, turn detection and barge-in, latency budget, tool calls in a voice loop, transport, cost, human handoff, guardrails, monitoring and launch — covering live caller conversations only, not offline dubbing/localization (ai-dubbing-localization-and-subtitling) and not the WER/MOS/VoiceBench scoring methodology owned by the evaluating-voice-agents resource.", "status": "complete", "pillar": { "slug": "voice-ai-agent-in-production", "title": "Voice AI Agent in Production: Architecture, Latency, Handoff and Launch", "description": "How to build and run a voice AI agent in production: cascaded vs speech-to-speech architecture, turn detection and barge-in, the latency budget, tool calls in a voice loop, transport, cost, human handoff and launch sequencing.", "kind": "pillar", "order": 0, "html": "https://changegamer.ai/articles/voice-ai-agent-in-production", "markdown": "https://changegamer.ai/articles/voice-ai-agent-in-production.md", "json": "https://changegamer.ai/api/articles/voice-ai-agent-in-production.json" }, "articles": [ { "slug": "voice-agent-latency-budget", "title": "Voice Agent Latency Budget: Where the 600 ms Goes and How to Hold It", "description": "How to split a 600 ms voice agent latency budget across speech-to-text, the language model, speech synthesis and tool calls, how to measure each stage, and which levers hold the budget.", "kind": "sub", "order": 1, "html": "https://changegamer.ai/articles/voice-agent-latency-budget", "markdown": "https://changegamer.ai/articles/voice-agent-latency-budget.md", "json": "https://changegamer.ai/api/articles/voice-agent-latency-budget.json" }, { "slug": "voice-turn-detection-and-barge-in-testing", "title": "How to Test Voice Agent Turn Detection and Barge-In", "description": "How to test voice agent turn detection and barge-in: build a labelled clip set, count barge-in accuracy and false-interruption rate, separate cut-off from talk-over failures, and split offline gates from online monitors.", "kind": "sub", "order": 2, "html": "https://changegamer.ai/articles/voice-turn-detection-and-barge-in-testing", "markdown": "https://changegamer.ai/articles/voice-turn-detection-and-barge-in-testing.md", "json": "https://changegamer.ai/api/articles/voice-turn-detection-and-barge-in-testing.json" }, { "slug": "voice-agent-architecture-cascaded-vs-speech-to-speech", "title": "What to Log and Filter in a Voice Agent, by Architecture", "description": "What to log and filter in a voice agent depending on architecture: where a cascaded pipeline gives you text seams for logging, redaction and output filters, what a native speech-to-speech model leaves you to do from outside, and what a model swap forces you to re-test.", "kind": "sub", "order": 3, "html": "https://changegamer.ai/articles/voice-agent-architecture-cascaded-vs-speech-to-speech", "markdown": "https://changegamer.ai/articles/voice-agent-architecture-cascaded-vs-speech-to-speech.md", "json": "https://changegamer.ai/api/articles/voice-agent-architecture-cascaded-vs-speech-to-speech.json" }, { "slug": "tools-in-a-voice-agent-loop", "title": "Function Calling in a Voice Agent: What a Tool May Be in a Spoken Loop", "description": "Function calling in a voice agent: which vendors and frameworks support tools as of the corpus's 2026-07-09 verification, how to classify tools as lookup, action or handoff, what barge-in does to a side effect, and a checklist for admitting a tool.", "kind": "sub", "order": 4, "html": "https://changegamer.ai/articles/tools-in-a-voice-agent-loop", "markdown": "https://changegamer.ai/articles/tools-in-a-voice-agent-loop.md", "json": "https://changegamer.ai/api/articles/tools-in-a-voice-agent-loop.json" }, { "slug": "voice-agent-transport-webrtc-vs-websocket-vs-sip", "title": "Voice Agent Transport: WebRTC vs WebSocket, and Where SIP Fits", "description": "WebRTC vs WebSocket for voice AI agents: which transport each vendor documents as of the corpus's 2026-07-09 verification, how cascaded stacks use WebSocket components, a decision tree by client type, and a per-transport test checklist.", "kind": "sub", "order": 5, "html": "https://changegamer.ai/articles/voice-agent-transport-webrtc-vs-websocket-vs-sip", "markdown": "https://changegamer.ai/articles/voice-agent-transport-webrtc-vs-websocket-vs-sip.md", "json": "https://changegamer.ai/api/articles/voice-agent-transport-webrtc-vs-websocket-vs-sip.json" }, { "slug": "voice-agent-human-handoff", "title": "Voice Agent Human Handoff: What Happens on the Call When the Agent Lets Go", "description": "Voice agent human handoff mechanics: the handoff as a tool that ends the agent's session, barge-in during the transfer announcement, what the caller hears on hold and on failure, a context payload schema, and a real-call test checklist.", "kind": "sub", "order": 6, "html": "https://changegamer.ai/articles/voice-agent-human-handoff", "markdown": "https://changegamer.ai/articles/voice-agent-human-handoff.md", "json": "https://changegamer.ai/api/articles/voice-agent-human-handoff.json" }, { "slug": "voice-agent-cost-per-minute", "title": "How to Compute Your Voice Agent Cost per Minute", "description": "How to compute a voice agent cost per minute from your own pilot: symbolic formulas for native speech-to-speech and cascaded STT, LLM and TTS, the variables that move the result, idle audio, retries, tool-call turns, and a sensitivity check.", "kind": "sub", "order": 7, "html": "https://changegamer.ai/articles/voice-agent-cost-per-minute", "markdown": "https://changegamer.ai/articles/voice-agent-cost-per-minute.md", "json": "https://changegamer.ai/api/articles/voice-agent-cost-per-minute.json" }, { "slug": "guardrails-for-spoken-output-in-voice-agents", "title": "Voice Agent Guardrails: Checking What the Agent Says Before the Caller Hears It", "description": "Voice agent guardrails for spoken output: sentence-level gating before TTS, fail-closed judges, spoken numbers a text PII filter misses, readback before state changes, transcription-driven injection, refusal phrasing for the ear, and an adversarial test list.", "kind": "sub", "order": 8, "html": "https://changegamer.ai/articles/guardrails-for-spoken-output-in-voice-agents", "markdown": "https://changegamer.ai/articles/guardrails-for-spoken-output-in-voice-agents.md", "json": "https://changegamer.ai/api/articles/guardrails-for-spoken-output-in-voice-agents.json" }, { "slug": "voice-agent-online-monitoring", "title": "How to Monitor a Voice Agent in Production: Dashboard, Alerts and Runbook", "description": "How to monitor a voice agent in production: per-stage latency percentiles, transcription confidence, containment paired with a resolution signal, live false-interruption rate, a handoff trigger review, and call-audio retention, each with what to chart, alert on and do.", "kind": "sub", "order": 9, "html": "https://changegamer.ai/articles/voice-agent-online-monitoring", "markdown": "https://changegamer.ai/articles/voice-agent-online-monitoring.md", "json": "https://changegamer.ai/api/articles/voice-agent-online-monitoring.json" }, { "slug": "voice-agent-vendor-change-management", "title": "How to Handle Voice AI Vendor Model Changes Without Regressing Callers", "description": "How to handle voice AI vendor model changes: classify the change, record the model id on every trace, replay an offline set with hard calls on old and new, canary before cutover, keep a rollback, and run a deprecation-readiness routine.", "kind": "sub", "order": 10, "html": "https://changegamer.ai/articles/voice-agent-vendor-change-management", "markdown": "https://changegamer.ai/articles/voice-agent-vendor-change-management.md", "json": "https://changegamer.ai/api/articles/voice-agent-vendor-change-management.json" }, { "slug": "voice-agent-launch-checklist", "title": "Voice Agent Launch Checklist: The Go/No-Go Gates Before Real Callers", "description": "A voice agent launch checklist as yes/no gates: scope, offline replay, per-stage latency, barge-in, tools, transport, handoff, cost, guardrails, monitoring and rollback, each with a pass signal and the article that owns the method.", "kind": "sub", "order": 11, "html": "https://changegamer.ai/articles/voice-agent-launch-checklist", "markdown": "https://changegamer.ai/articles/voice-agent-launch-checklist.md", "json": "https://changegamer.ai/api/articles/voice-agent-launch-checklist.json" }, { "slug": "voice-agent-multilingual-callers", "title": "How Does a Voice Agent Handle Callers Who Speak Another Language?", "description": "How a live voice agent handles callers who speak another language: detecting the language, choosing speech components per language, slicing evaluation by language and handing off when the language is unsupported, with the thin corpus evidence stated plainly.", "kind": "sub", "order": 12, "html": "https://changegamer.ai/articles/voice-agent-multilingual-callers", "markdown": "https://changegamer.ai/articles/voice-agent-multilingual-callers.md", "json": "https://changegamer.ai/api/articles/voice-agent-multilingual-callers.json" } ] }, "navigation": { "pillar": { "slug": "voice-ai-agent-in-production", "title": "Voice AI Agent in Production: Architecture, Latency, Handoff and Launch", "description": "How to build and run a voice AI agent in production: cascaded vs speech-to-speech architecture, turn detection and barge-in, the latency budget, tool calls in a voice loop, transport, cost, human handoff and launch sequencing.", "kind": "pillar", "order": 0, "html": "https://changegamer.ai/articles/voice-ai-agent-in-production", "markdown": "https://changegamer.ai/articles/voice-ai-agent-in-production.md", "json": "https://changegamer.ai/api/articles/voice-ai-agent-in-production.json" }, "previous": { "slug": "guardrails-for-spoken-output-in-voice-agents", "title": "Voice Agent Guardrails: Checking What the Agent Says Before the Caller Hears It", "description": "Voice agent guardrails for spoken output: sentence-level gating before TTS, fail-closed judges, spoken numbers a text PII filter misses, readback before state changes, transcription-driven injection, refusal phrasing for the ear, and an adversarial test list.", "kind": "sub", "order": 8, "html": "https://changegamer.ai/articles/guardrails-for-spoken-output-in-voice-agents", "markdown": "https://changegamer.ai/articles/guardrails-for-spoken-output-in-voice-agents.md", "json": "https://changegamer.ai/api/articles/guardrails-for-spoken-output-in-voice-agents.json" }, "next": { "slug": "voice-agent-vendor-change-management", "title": "How to Handle Voice AI Vendor Model Changes Without Regressing Callers", "description": "How to handle voice AI vendor model changes: classify the change, record the model id on every trace, replay an offline set with hard calls on old and new, canary before cutover, keep a rollback, and run a deprecation-readiness routine.", "kind": "sub", "order": 10, "html": "https://changegamer.ai/articles/voice-agent-vendor-change-management", "markdown": "https://changegamer.ai/articles/voice-agent-vendor-change-management.md", "json": "https://changegamer.ai/api/articles/voice-agent-vendor-change-management.json" } }, "resources": [ { "slug": "evaluating-voice-agents", "html": "https://changegamer.ai/resources/evaluating-voice-agents", "markdown": "https://changegamer.ai/resources/evaluating-voice-agents.md", "json": "https://changegamer.ai/api/resources/evaluating-voice-agents.json" }, { "slug": "agent-observability", "html": "https://changegamer.ai/resources/agent-observability", "markdown": "https://changegamer.ai/resources/agent-observability.md", "json": "https://changegamer.ai/api/resources/agent-observability.json" }, { "slug": "voice-realtime-agents", "html": "https://changegamer.ai/resources/voice-realtime-agents", "markdown": "https://changegamer.ai/resources/voice-realtime-agents.md", "json": "https://changegamer.ai/api/resources/voice-realtime-agents.json" } ] }