{ "slug": "redacting-sensitive-data-from-agent-traces", "title": "How to Redact PII From AI Agent Traces: Placement, Testing and Cleanup", "description": "How to redact PII from AI agent traces in practice: where the redactor sits in the pipeline, what to do per span field, how to test it with seeded fake PII, and how to clean up after a leak.", "kind": "sub", "order": 11, "target_query": "how to redact PII from AI agent traces", "secondary_queries": [ "llm tracing", "ai agent observability", "PII redaction before AI agent model context" ], "tags": [ "agents", "observability", "tracing", "pii", "redaction", "privacy", "secrets" ], "published": "2026-10-06", "updated": "2026-10-06", "words": 1413, "estimated_tokens": 1879, "premium": false, "rights": { "access": "free", "note": "Editorial guides are always free and never part of the licensed corpus.", "license": "https://changegamer.ai/license.xml", "pricing": "https://changegamer.ai/api/pricing.json", "payment": "https://changegamer.ai/api/payment.json" }, "license": "https://changegamer.ai/license.xml", "citation": "ChangeGamer (2026-10-06). How to Redact PII From AI Agent Traces: Placement, Testing and Cleanup. ChangeGamer. https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces (updated 2026-10-06).", "bibtex": "@misc{changegamer_redacting_sensitive_data_from_agent_traces, title = {How to Redact PII From AI Agent Traces: Placement, Testing and Cleanup}, publisher = {ChangeGamer}, year = {2026}, url = {https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces}, note = {Updated 2026-10-06}}", "canonical": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces", "markdown": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces.md", "takeaways": [ "To redact PII from AI agent traces, put the redactor in the exporter path so spans are scrubbed before they leave the process, and so every downstream sink receives only sanitized data.", "Redaction policy should be set per span field, because prompt bodies, tool arguments, tool outputs and attributes carry different risks and need different treatment, such as drop, mask or placeholder.", "A trace redactor should be verified, not assumed: run fixtures with seeded fake PII through the agent, then scan the exported spans for those exact values and fail the build if any appear.", "Credentials are a separate class from PII, and the agentic security checklist advises redacting known secret formats from anything flowing into model context, not only from traces.", "The reference corpus contains no detection accuracy or latency figures for any redaction tool as of October 2026, so every pipeline-design claim here beyond the tool descriptions is reasoned inference." ], "outline": [ { "depth": 2, "text": "What is already covered elsewhere?", "anchor": "what-is-already-covered-elsewhere", "url": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces#what-is-already-covered-elsewhere" }, { "depth": 2, "text": "Where in the pipeline should the redactor sit?", "anchor": "where-in-the-pipeline-should-the-redactor-sit", "url": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces#where-in-the-pipeline-should-the-redactor-sit" }, { "depth": 2, "text": "What policy should each span field get?", "anchor": "what-policy-should-each-span-field-get", "url": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces#what-policy-should-each-span-field-get" }, { "depth": 3, "text": "Which operator should replace a detected value?", "anchor": "which-operator-should-replace-a-detected-value", "url": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces#which-operator-should-replace-a-detected-value" }, { "depth": 2, "text": "Why are secrets a separate class?", "anchor": "why-are-secrets-a-separate-class", "url": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces#why-are-secrets-a-separate-class" }, { "depth": 2, "text": "How do you verify the redactor works?", "anchor": "how-do-you-verify-the-redactor-works", "url": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces#how-do-you-verify-the-redactor-works" }, { "depth": 2, "text": "What if PII already reached the trace store?", "anchor": "what-if-pii-already-reached-the-trace-store", "url": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces#what-if-pii-already-reached-the-trace-store" }, { "depth": 2, "text": "Does redacting conflict with keeping an audit trail?", "anchor": "does-redacting-conflict-with-keeping-an-audit-trail", "url": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces#does-redacting-conflict-with-keeping-an-audit-trail" }, { "depth": 2, "text": "What does ChangeGamer do here?", "anchor": "what-does-changegamer-do-here", "url": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces#what-does-changegamer-do-here" } ], "faq": [ { "question": "How do I redact PII from AI agent traces?", "answer": "Redact PII from AI agent traces by running a detector and anonymizer inside the exporter path, before spans leave your process, with a separate policy for each span field. Then prove it works by scanning exported spans for seeded fake PII values on every build." }, { "question": "Does the OpenTelemetry GenAI convention capture prompts by default?", "answer": "No. The OpenTelemetry GenAI conventions leave model input and output text off by default, as optional span events for PII safety, as of the agent-observability corpus entry updated July 2026. Turning them on means taking over the redaction job yourself." }, { "question": "What should I do if PII has already reached my trace store?", "answer": "Stop the leak at the source first by fixing and deploying the redactor, then purge or re-process the affected spans in the store, and rotate any credentials that appeared. Which deletion methods your backend offers must be checked in its own documentation." }, { "question": "Is credential redaction the same as PII redaction?", "answer": "No. Credentials are a separate class because a leaked key grants access, whereas leaked PII exposes a person. The agentic security checklist recommends redacting known secret formats such as common key prefixes, bearer tokens and PEM blocks from anything flowing into model context." } ], "body": "To redact PII from AI agent traces, run the detector inside your exporter so spans are scrubbed before they leave the process, apply a different rule to each span field, and test the result by scanning exported spans for fake values you seeded yourself. The corpus holds no detection accuracy or latency numbers as of October 2026, so the pipeline design below is reasoned inference built on the tool descriptions it does contain. Context for the topic is in [the pillar on observability and evaluation](/articles/agent-observability-and-evaluation).\n\n## What is already covered elsewhere?\n\nThree sibling articles make the case for redacting, and this one deliberately does not repeat it. [Agent observability for reliability](/articles/agent-observability-for-reliability) explains why traces need redaction at all. [Sampling and retention of agent traces](/articles/sampling-and-retention-of-agent-traces) sets the rule that redaction happens before the retention clock starts. [Data privacy and PII for AI agents](/articles/data-privacy-and-pii-for-ai-agents) covers the exposure surface and legal framing, and nothing here is legal advice. What remains is the mechanics: where the code sits, what it does to each field, how you know it works, and what to do when it fails.\n\n## Where in the pipeline should the redactor sit?\n\nThe redactor belongs in the exporter layer, after spans are created and before they are buffered or sent anywhere. The data-privacy-for-agents resource says to scrub in the exporter path before export, so all downstream sinks (LangSmith, Datadog, S3) receive only sanitized data.\n\nThe ordering below is inference, not a published standard:\n\n1. The agent creates a span with raw attributes and events.\n2. A span processor or exporter wrapper runs detection and anonymization on the span.\n3. Only the sanitized span enters the export buffer.\n4. The buffer exports to one or more backends.\n\nPutting the step before the buffer matters because a buffer, a retry queue or a local file spool is itself a store. If raw content sits in any of them, you have a leak surface that the backend's own controls will not reach. A crashed process can also leave raw spans on disk, which ties into [crash-safe run-start records](/articles/crash-safe-run-start-records-for-agent-traces).\n\nA related but different control is redaction before the model sees data. The same resource lists \"redact PII from all content before it enters the model context window\" as its own checklist item, so the trace redactor is the second gate and not a substitute for the first. If you only scrub at export, the model still received the raw value.\n\n## What policy should each span field get?\n\nEach span field needs its own rule, because the four places PII hides in a trace behave differently. The table is a reasoned starting point, not a corpus rule.\n\n| Field | Typical content | Suggested default |\n|---|---|---|\n| Prompt and completion bodies | Free text, user messages, retrieved excerpts | Off, or detect and replace with placeholders |\n| Tool arguments | Structured parameters, search queries | Allow-list known-safe keys, redact the rest |\n| Tool outputs | Records, documents, API responses | Redact, or keep only status and size |\n| Attributes | IDs, model names, token counts | Keep structured IDs, scan free-text values |\n\nThe corpus supports three anchors for this table. First, the OpenTelemetry GenAI conventions make model input and output text an optional span event, off by default for PII safety. Second, the agent-observability resource lists tool-call inputs and outputs among the signals to capture, with the instruction to redact PII before logging. Third, the privacy resource notes that structured fields like `user_id` are acceptable while raw prompt strings containing names, SSNs or health data are not.\n\nTool outputs deserve extra care because the privacy resource states that retrieved and tool-output content can itself carry names or financial and health data, not only what the user typed.\n\n### Which operator should replace a detected value?\n\nChoose the operator by what the trace is for. The Presidio Anonymizer, described in the corpus, applies four configurable operators: replace, mask, redact and encrypt. The resource also shows placeholders such as `` and ``.\n\n- **Placeholders** keep the sentence structure readable for debugging and evaluation, and a numbered placeholder lets you see that two mentions were the same entity.\n- **Redact** removes the value entirely and is the safest default for fields you never need to read.\n- **Mask** keeps a partial shape and suits identifiers you only need to recognise.\n- **Encrypt** keeps the original recoverable, which makes the trace store a holder of personal data again. Treat it as a deliberate exception.\n\n## Why are secrets a separate class?\n\nSecrets need their own rule because a leaked credential is an access problem and a leaked name is a privacy problem, and the fixes differ. The agentic-security-checklist resource states that a secret entering a prompt, a retrieved document, a tool result or a log line the model can read must be treated as disclosed.\n\nIts defences include redacting known secret formats from anything flowing into model context, covering prefix detection for common key shapes, bearer tokens and PEM blocks. Format-based matching is a different technique from the entity recognition used for names, so run it as a separate step in the same processor. Because secrets are often disclosed before they reach any trace, the checklist's first defence also applies: resolve credentials server-side at tool-execution time so the model never holds the key.\n\n## How do you verify the redactor works?\n\nVerify a redactor by seeding fake PII and secrets into test runs, then scanning the exported spans for those exact strings. This is reasoned inference, since the corpus describes the tools but gives no testing method.\n\n1. Build a fixture set of fake but realistic values: a name, an email, a phone number, a card-shaped number and a dummy key with a real prefix shape.\n2. Place them in every field class from the table, including nested tool arguments and tool outputs.\n3. Run the agent against a local or test backend.\n4. Fetch what was actually exported, not what the processor claims to have emitted.\n5. Fail the build if any seeded value appears.\n\nA minimal scan over exported spans looks like this:\n\n```bash\n# spans.jsonl: spans as received by the test backend, one JSON object per line\n# seeds.txt: the fake values you injected, one per line\nif grep -F -f <(grep -v '^$' seeds.txt) spans.jsonl; then\n echo \"FAIL: seeded PII found in exported spans\" >&2\n exit 1\nfi\necho \"OK: no seeded values found\"\n```\n\nCanary values add a second signal in production. Use a recognisable fake identity in a synthetic run on a schedule and search the real store for it. A hit means the redactor regressed after a deploy. Remember a clean scan only proves the seeded patterns are caught, not that all real PII is, so treat detection coverage as unmeasured until you measure it on your own data.\n\n## What if PII already reached the trace store?\n\nTreat it as an incident: stop the source, remove the data, then re-verify. The order matters because purging before the fix just refills the store.\n\n1. Deploy the corrected redactor and confirm it with the scan above.\n2. Identify the affected window using the deploy history and the canary hits.\n3. Delete or re-process affected spans using whatever your backend supports. The corpus states no deletion capability for any listed backend, so check its documentation.\n4. Check copies: exports, backups, evaluation datasets built from sampled traces, and any other sink.\n5. Rotate any credential that appeared, since the checklist says to treat it as disclosed.\n\nSampled datasets are easy to forget. The agent-observability resource notes that stored spans feed evaluation datasets, so a leak can travel there.\n\n## Does redacting conflict with keeping an audit trail?\n\nRedaction and auditability pull in opposite directions, and you resolve it by separating the stores rather than weakening either. An audit record needs enough to reconstruct who did what, while a debugging trace should hold as little personal content as possible. [Audit trails for AI agents](/articles/audit-trails-for-ai-agents) covers what an audit record needs to hold. A reasonable inference is to keep identifiers and decisions in the audit store and let the trace carry placeholders.\n\n## What does ChangeGamer do here?\n\nChangeGamer runs no trace store, and no agent content passes through a redaction pipeline on this site. Nothing in this article comes from operating one. The tool descriptions come from the corpus, and the pipeline design, the field table and the test method are reasoning from them.", "cluster": { "id": "agent-observability-evaluation", "title": "Agent observability and evaluation", "description": "How to observe and evaluate an AI agent already live in production — tracing spans for tool calls and retrieval steps, judge-based screening versus human acceptance of live output, online metrics, cost telemetry, and incident postmortems that feed evaluation fixtures — not the pre-ship, CI-gating evaluation methodology already owned by evaluating-ai-agents-in-ci, and not the trace/span field mechanics already owned by agent-observability-for-reliability, both in the completed agent-reliability cluster.", "status": "complete", "pillar": { "slug": "agent-observability-and-evaluation", "title": "AI Agent Observability and the Production Evaluation Playbook", "description": "AI agent observability and evaluation once an agent is already live: tracing spans, retrieval-attribution logs, judge-based screening vs. human acceptance, online metrics, cost telemetry, and incident postmortems that feed evaluation fixtures.", "kind": "pillar", "order": 0, "html": "https://changegamer.ai/articles/agent-observability-and-evaluation", "markdown": "https://changegamer.ai/articles/agent-observability-and-evaluation.md", "json": "https://changegamer.ai/api/articles/agent-observability-and-evaluation.json" }, "articles": [ { "slug": "multi-agent-trace-propagation", "title": "Distributed Tracing for Multi-Agent AI Systems", "description": "How trace ID propagation actually works across a multi-agent handoff, why handoff and delegation need different trace shapes, and why an orchestrator-level trace can hide a failed leg of a fan-out.", "kind": "sub", "order": 1, "html": "https://changegamer.ai/articles/multi-agent-trace-propagation", "markdown": "https://changegamer.ai/articles/multi-agent-trace-propagation.md", "json": "https://changegamer.ai/api/articles/multi-agent-trace-propagation.json" }, { "slug": "llm-as-judge-screening-in-production", "title": "Designing an LLM-as-Judge Pipeline for Production AI Agents", "description": "An operator playbook for screening live AI agent output with an LLM judge: a confidence/stakes routing architecture to a human queue, continuous live-traffic rubric design, and per-bias mitigations for position, verbosity, and self-preference.", "kind": "sub", "order": 2, "html": "https://changegamer.ai/articles/llm-as-judge-screening-in-production", "markdown": "https://changegamer.ai/articles/llm-as-judge-screening-in-production.md", "json": "https://changegamer.ai/api/articles/llm-as-judge-screening-in-production.json" }, { "slug": "agent-cost-telemetry-in-production", "title": "How to Track AI Agent Costs in Production", "description": "How to track AI agent costs in production: version your price table, separate cached from uncached tokens, keep failed runs in the books, attribute shared costs, reconcile against the invoice, and alert on rate of spend.", "kind": "sub", "order": 3, "html": "https://changegamer.ai/articles/agent-cost-telemetry-in-production", "markdown": "https://changegamer.ai/articles/agent-cost-telemetry-in-production.md", "json": "https://changegamer.ai/api/articles/agent-cost-telemetry-in-production.json" }, { "slug": "incident-to-eval-fixture-loop", "title": "How to Turn an AI Agent Incident into an Evaluation Test Case", "description": "How to turn an AI agent incident into an evaluation test case: freeze the trace, redact it, minimize it to the failing decision, label the expected outcome with a trusted oracle, prove it fails then passes, and retire it later.", "kind": "sub", "order": 4, "html": "https://changegamer.ai/articles/incident-to-eval-fixture-loop", "markdown": "https://changegamer.ai/articles/incident-to-eval-fixture-loop.md", "json": "https://changegamer.ai/api/articles/incident-to-eval-fixture-loop.json" }, { "slug": "retrieval-attribution-logging-in-production", "title": "How to Log RAG Retrieval in Production for Debugging Agent Answers", "description": "How to log RAG retrieval in production so a wrong agent answer can be debugged: per-retrieval fields, what reached the context window versus what the model cited, privacy by hashes and IDs, and retention and sampling choices.", "kind": "sub", "order": 5, "html": "https://changegamer.ai/articles/retrieval-attribution-logging-in-production", "markdown": "https://changegamer.ai/articles/retrieval-attribution-logging-in-production.md", "json": "https://changegamer.ai/api/articles/retrieval-attribution-logging-in-production.json" }, { "slug": "online-feedback-signals-for-ai-agents", "title": "How to Measure AI Agent Quality from Live User Feedback", "description": "How to measure AI agent quality from live user feedback: why explicit ratings are sparse and biased, how re-asks, abandonment and escalations mislead, and how to join each signal to a trace and route it to review.", "kind": "sub", "order": 6, "html": "https://changegamer.ai/articles/online-feedback-signals-for-ai-agents", "markdown": "https://changegamer.ai/articles/online-feedback-signals-for-ai-agents.md", "json": "https://changegamer.ai/api/articles/online-feedback-signals-for-ai-agents.json" }, { "slug": "online-agent-metrics-and-drift-monitoring", "title": "How to Detect Quality Drift in a Production AI Agent", "description": "How to detect quality drift in a production AI agent: baseline aggregate signals, alert on a diff against the baseline, and separate a prompt, model or tool-version change from a shift in traffic mix.", "kind": "sub", "order": 7, "html": "https://changegamer.ai/articles/online-agent-metrics-and-drift-monitoring", "markdown": "https://changegamer.ai/articles/online-agent-metrics-and-drift-monitoring.md", "json": "https://changegamer.ai/api/articles/online-agent-metrics-and-drift-monitoring.json" }, { "slug": "sampling-and-retention-of-agent-traces", "title": "How to Sample and Retain Production AI Agent Traces", "description": "How to sample and retain production AI agent traces: head vs tail sampling, keep-all-errors plus a random baseline, whole-trace decisions for multi-agent runs, retention tiers and redaction before the clock starts.", "kind": "sub", "order": 8, "html": "https://changegamer.ai/articles/sampling-and-retention-of-agent-traces", "markdown": "https://changegamer.ai/articles/sampling-and-retention-of-agent-traces.md", "json": "https://changegamer.ai/api/articles/sampling-and-retention-of-agent-traces.json" }, { "slug": "crash-safe-run-start-records-for-agent-traces", "title": "How to Keep Trace Data When an AI Agent Crashes Mid-Run", "description": "How to keep trace data when an AI agent crashes mid-run: write a small run-start record outside the trace buffer, detect orphans, count them as unknown outcomes, and force-decide on shutdown.", "kind": "sub", "order": 9, "html": "https://changegamer.ai/articles/crash-safe-run-start-records-for-agent-traces", "markdown": "https://changegamer.ai/articles/crash-safe-run-start-records-for-agent-traces.md", "json": "https://changegamer.ai/api/articles/crash-safe-run-start-records-for-agent-traces.json" }, { "slug": "choosing-an-llm-observability-backend", "title": "How to Choose an LLM Observability Platform for AI Agents", "description": "How to choose an LLM observability platform: decide on OTel-native ingestion, self-host versus cloud, export portability, redaction hooks and retention support before comparing vendors.", "kind": "sub", "order": 10, "html": "https://changegamer.ai/articles/choosing-an-llm-observability-backend", "markdown": "https://changegamer.ai/articles/choosing-an-llm-observability-backend.md", "json": "https://changegamer.ai/api/articles/choosing-an-llm-observability-backend.json" }, { "slug": "redacting-sensitive-data-from-agent-traces", "title": "How to Redact PII From AI Agent Traces: Placement, Testing and Cleanup", "description": "How to redact PII from AI agent traces in practice: where the redactor sits in the pipeline, what to do per span field, how to test it with seeded fake PII, and how to clean up after a leak.", "kind": "sub", "order": 11, "html": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces", "markdown": "https://changegamer.ai/articles/redacting-sensitive-data-from-agent-traces.md", "json": "https://changegamer.ai/api/articles/redacting-sensitive-data-from-agent-traces.json" } ] }, "navigation": { "pillar": { "slug": "agent-observability-and-evaluation", "title": "AI Agent Observability and the Production Evaluation Playbook", "description": "AI agent observability and evaluation once an agent is already live: tracing spans, retrieval-attribution logs, judge-based screening vs. human acceptance, online metrics, cost telemetry, and incident postmortems that feed evaluation fixtures.", "kind": "pillar", "order": 0, "html": "https://changegamer.ai/articles/agent-observability-and-evaluation", "markdown": "https://changegamer.ai/articles/agent-observability-and-evaluation.md", "json": "https://changegamer.ai/api/articles/agent-observability-and-evaluation.json" }, "previous": { "slug": "choosing-an-llm-observability-backend", "title": "How to Choose an LLM Observability Platform for AI Agents", "description": "How to choose an LLM observability platform: decide on OTel-native ingestion, self-host versus cloud, export portability, redaction hooks and retention support before comparing vendors.", "kind": "sub", "order": 10, "html": "https://changegamer.ai/articles/choosing-an-llm-observability-backend", "markdown": "https://changegamer.ai/articles/choosing-an-llm-observability-backend.md", "json": "https://changegamer.ai/api/articles/choosing-an-llm-observability-backend.json" }, "next": null }, "resources": [ { "slug": "agent-observability", "html": "https://changegamer.ai/resources/agent-observability", "markdown": "https://changegamer.ai/resources/agent-observability.md", "json": "https://changegamer.ai/api/resources/agent-observability.json" }, { "slug": "data-privacy-for-agents", "html": "https://changegamer.ai/resources/data-privacy-for-agents", "markdown": "https://changegamer.ai/resources/data-privacy-for-agents.md", "json": "https://changegamer.ai/api/resources/data-privacy-for-agents.json" }, { "slug": "agentic-security-checklist", "html": "https://changegamer.ai/resources/agentic-security-checklist", "markdown": "https://changegamer.ai/resources/agentic-security-checklist.md", "json": "https://changegamer.ai/api/resources/agentic-security-checklist.json" } ] }