{ "slug": "evaluating-rag-systems", "title": "How to Evaluate a RAG System (Retrieval Metrics, Generation Metrics, CI Gates)", "description": "The evaluation harness that keeps RAG changeable: golden-set construction, retrieval metrics separated from generation metrics, LLM-as-judge screening with human acceptance, CI regression gates, and the logged-query flywheel.", "kind": "sub", "order": 6, "target_query": "how to evaluate a RAG system", "secondary_queries": [ "RAG evaluation metrics recall MRR faithfulness", "golden dataset for RAG testing", "LLM as judge evaluation pitfalls", "RAG regression testing in CI" ], "tags": [ "rag", "evaluation", "testing", "quality" ], "published": "2026-08-23", "updated": "2026-08-23", "words": 908, "estimated_tokens": 1208, "premium": false, "rights": { "access": "free", "note": "Editorial guides are always free and never part of the licensed corpus.", "license": "https://changegamer.ai/license.xml", "pricing": "https://changegamer.ai/api/pricing.json", "payment": "https://changegamer.ai/api/payment.json" }, "license": "https://changegamer.ai/license.xml", "citation": "ChangeGamer (2026-08-23). How to Evaluate a RAG System (Retrieval Metrics, Generation Metrics, CI Gates). ChangeGamer. https://changegamer.ai/articles/evaluating-rag-systems (updated 2026-08-23).", "bibtex": "@misc{changegamer_evaluating_rag_systems, title = {How to Evaluate a RAG System (Retrieval Metrics, Generation Metrics, CI Gates)}, publisher = {ChangeGamer}, year = {2026}, url = {https://changegamer.ai/articles/evaluating-rag-systems}, note = {Updated 2026-08-23}}", "canonical": "https://changegamer.ai/articles/evaluating-rag-systems", "markdown": "https://changegamer.ai/articles/evaluating-rag-systems.md", "takeaways": [ "A RAG system without evaluation cannot be safely changed — every knob (chunking, embeddings, fusion, reranking, prompts) moves quality in ways nobody can eyeball. The harness converts those knobs from acts of faith into measured engineering decisions.", "Build a version-controlled golden set: real questions annotated with the passages that must be retrieved and reference answers. Cover exact-identifier lookups, paraphrases, multi-hop questions, empty-result cases, and permission-negative cases that must return nothing for some users.", "Score retrieval separately from generation. Recall@k, MRR and nDCG are deterministic, fast and model-free — ideal CI gates. Faithfulness and answer relevance are noisier generation-side measures; use judges for regression screening and humans for acceptance.", "Wire the suite into CI so any change to chunking, models, prompts or retrieval parameters runs it automatically, and treat threshold breaches as build failures. Then close the loop with production query logs: sample failures, cluster them into intents, promote the instructive ones into the golden set." ], "outline": [ { "depth": 2, "text": "What goes into a golden set?", "anchor": "what-goes-into-a-golden-set", "url": "https://changegamer.ai/articles/evaluating-rag-systems#what-goes-into-a-golden-set" }, { "depth": 2, "text": "Which metrics gate which layer?", "anchor": "which-metrics-gate-which-layer", "url": "https://changegamer.ai/articles/evaluating-rag-systems#which-metrics-gate-which-layer" }, { "depth": 2, "text": "What makes judge-based evaluation drift?", "anchor": "what-makes-judge-based-evaluation-drift", "url": "https://changegamer.ai/articles/evaluating-rag-systems#what-makes-judge-based-evaluation-drift" }, { "depth": 2, "text": "How do you keep the golden set honest?", "anchor": "how-do-you-keep-the-golden-set-honest", "url": "https://changegamer.ai/articles/evaluating-rag-systems#how-do-you-keep-the-golden-set-honest" }, { "depth": 2, "text": "How does evaluation wire into delivery?", "anchor": "how-does-evaluation-wire-into-delivery", "url": "https://changegamer.ai/articles/evaluating-rag-systems#how-does-evaluation-wire-into-delivery" }, { "depth": 2, "text": "How does the harness improve after launch?", "anchor": "how-does-the-harness-improve-after-launch", "url": "https://changegamer.ai/articles/evaluating-rag-systems#how-does-the-harness-improve-after-launch" } ], "faq": [ { "question": "What is a golden set for RAG evaluation?", "answer": "A curated, version-controlled set of representative questions with annotations: which passages must be retrieved for each question, plus reference answers. Quality beats quantity — thirty well-chosen cases spanning your real query distribution catch more regressions than thousands assembled carelessly. It should include negative cases (questions the system must decline or return empty) and permission cases (queries that must yield nothing for unauthorized users)." }, { "question": "Which metrics measure retrieval quality?", "answer": "Recall@k asks whether a needed passage made the top-k candidates; MRR and nDCG score how high it ranked. All three are deterministic, fast, and independent of any language model, which makes them ideal for continuous integration gates. Most \"answers got worse\" reports resolve here: generation did not degrade, retrieval stopped surfacing the evidence the generator needed." }, { "question": "Can an LLM judge evaluate RAG answers reliably?", "answer": "Reliably enough for regression screening when calibrated: give the judge the retrieved context and ask whether the answer is faithful to it, plus whether it addresses the question. Judge scores are noisy at fine granularity, drift with prompt changes, and inherit model biases — so treat them as cheap screening, validate them periodically against human judgments, and reserve humans for acceptance decisions and the disagreement cases." }, { "question": "How often should a RAG evaluation run?", "answer": "On every proposed change in CI — chunking, embedding model, fusion weights, reranker depth, prompts — because each of those knobs shifts answer quality invisibly. Additionally run a scheduled full pass against production data weekly or monthly to catch environmental drift (corpus growth, query-mix shifts) that per-change runs cannot see. A red gate blocks merge exactly like a failing unit test." } ], "body": "\nEverything else in this cluster ([ingestion](/articles/rag-ingestion-pipeline), [chunking](/articles/chunking-documents-for-rag), [embeddings](/articles/choosing-embedding-models), [fusion](/articles/hybrid-retrieval-fusion), [reranking](/articles/reranking-retrieved-results)) produces knobs that move answer quality invisibly. Evaluation is the instrument panel that makes those knobs safe to turn — and per the [pillar guide](/articles/rag-in-production), it is what separates maintainable systems from demos that decay.\n\n## What goes into a golden set?\n\nFive question classes, drawn from real traffic rather than imagination:\n\n1. **Exact-identifier lookups** — codes, names, part numbers, citations\n2. **Paraphrases** — questions whose wording differs from the source documents\n3. **Multi-hop questions** — answers requiring evidence from more than one passage\n4. **Empty-result cases** — questions the corpus genuinely cannot answer, where the correct behavior is a confident \"not found\"\n5. **Permission-negative cases** — questions answerable only from content a given principal must not see\n\nAnnotate each with its required passages and, where practical, reference answers. Version the set in git next to the code it tests; evolve it through review, not by silent edits. Thirty disciplined cases outperform three thousand sloppy ones because every case earns its place by representing something real.\n\n## Which metrics gate which layer?\n\nKeep the layers strictly separate:\n\n| Layer | Metric | Property |\n|---|---|---|\n| Retrieval | recall@k | did needed passage enter top-k |\n| Retrieval | MRR / nDCG | how high did it rank |\n| Generation | faithfulness | is the answer entailed by retrieved context |\n| Generation | relevance | does it address the asked question |\n\nRetrieval metrics are deterministic, fast, and model-independent — they belong in every pull request's checks. Generation metrics need either rubric-scored human review or LLM-as-judge patterns; judges are cheap and noisy, humans are ground truth and expensive, so screen with judges continuously and calibrate them against humans on a sample, reserving people for acceptance and disagreements.\n\n## What makes judge-based evaluation drift?\n\nJudges are language models, so they inherit every instability of language models:\n\n- **Prompt sensitivity**: rewording the rubric shifts scores systematically, which reads as a quality change when it is a measurement change\n- **Model updates**: swapping or upgrading the judge silently re-bases your entire metric history; pin the judge model and version it like any other dependency\n- **Position and verbosity bias**: judges favor longer answers and sometimes earlier options; randomize presentation order where applicable and cap answer lengths before judging\n- **Self-preference**: a judge from the same family as the generator grades that generator generously\n\nThe countermeasures are procedural: calibrate against human labels on a sample each cycle, track judge-versus-human agreement as its own metric, and route disagreement cases to humans rather than averaging them away.\n\n## How do you keep the golden set honest?\n\nGolden sets rot in three ways: they leak into prompts (someone pastes test questions into few-shot examples), they overfit through repeated tuning against them, and they stop representing traffic as the product evolves. Countermeasures: hold out a rotation slice never used during development; refresh question phrasings periodically while keeping annotations stable; review additions through the same pull-request discipline as code; and maintain a strict no-training-on-the-golden-set rule for prompts, retrievers and routers alike.\n\n## How does evaluation wire into delivery?\n\nTwo loops with different clocks:\n\n- **Per-change loop (CI)**: on every PR touching chunking, models, prompts or retrieval parameters, rebuild a sample index, run the offline suite, fail the build beyond thresholds. This is the regression gate that makes bold changes boring to ship\n- **Scheduled loop**: a periodic full pass over current production data catches environmental drift — corpus growth skewing rankings, connector changes shifting text distributions — that per-change runs on stable samples cannot see\n\nTreat gate thresholds as reviewed configuration with history, not folklore. When a metric improves, tighten thoughtfully; when it regresses, the fix is reverting or fixing the change, never quietly lowering the bar ([the deploy gates article pattern](/articles/rag-in-production) applies here too).\n\n## How does the harness improve after launch?\n\nThrough the logged-query flywheel. Sample production queries on a schedule, cluster them by intent, and inspect the failures — questions with no clicks, negative feedback, or escalations. Promote instructive failures into the golden set with annotations, so the suite grows toward the system's real weak spots rather than its imagined ones. The same logs surface coverage gaps: an intent cluster with consistently poor retrieval is next quarter's ingestion or chunking work, discovered by measurement instead of complaint. Observability feeds the flywheel — [agent observability](/resources/agent-observability) covers the instrumentation side; the evaluation side closes it by turning telemetry back into tests.\n\nThe result compounds: every incident becomes a fixture, every fixture sharpens every future decision, and the team stops experiencing RAG as fragile — because nothing changed without the instrument panel noticing first.\n\nA closing calibration on effort. The harness described here costs real time to build — typically days for the first golden set and gate, not weeks — and pays it back at the first significant change it screens: one prevented bad deploy of a chunking migration, one caught prompt regression, one embedding switch that shadow-testing rejected, repays the build several times over. Teams skip evaluation because everything works at launch. That is precisely when the harness is cheapest to build and the absence of it most expensive later, since every knob added afterward multiplies the number of silent ways the system can regress.\n\nStart small if starting is hard: ten golden questions, one retrieval metric, one CI step. The harness's value compounds from existing, not from being complete on day one.\n", "cluster": { "id": "rag-in-production", "title": "RAG in production", "description": "How to run retrieval-augmented generation as a real system — ingestion and chunking, embedding choice and reindexing, hybrid search, reranking, evaluation, freshness, access control, cost, latency, and when RAG is the wrong answer.", "status": "complete", "pillar": { "slug": "rag-in-production", "title": "Agentic RAG in Production: The Complete Operator Guide", "description": "The operator playbook for agentic RAG in production: ingestion, chunking, hybrid retrieval, reranking, evaluation, freshness and cost.", "kind": "pillar", "order": 0, "html": "https://changegamer.ai/articles/rag-in-production", "markdown": "https://changegamer.ai/articles/rag-in-production.md", "json": "https://changegamer.ai/api/articles/rag-in-production.json" }, "articles": [ { "slug": "rag-ingestion-pipeline", "title": "How to Build a RAG Ingestion Pipeline That Survives Production", "description": "The six properties that separate a production RAG ingestion pipeline from a demo script: tested extraction, idempotent writes, incremental updates, deletion propagation, durable execution, and metadata captured at ingest time.", "kind": "sub", "order": 1, "html": "https://changegamer.ai/articles/rag-ingestion-pipeline", "markdown": "https://changegamer.ai/articles/rag-ingestion-pipeline.md", "json": "https://changegamer.ai/api/articles/rag-ingestion-pipeline.json" }, { "slug": "chunking-documents-for-rag", "title": "How to Chunk Documents for RAG (Strategy Beats Size)", "description": "Chunking decisions that actually move retrieval quality: structural boundaries before fixed windows, parent-document expansion, special handling for tables and code, overlap trade-offs, and tuning against recall@k instead of blog defaults.", "kind": "sub", "order": 2, "html": "https://changegamer.ai/articles/chunking-documents-for-rag", "markdown": "https://changegamer.ai/articles/chunking-documents-for-rag.md", "json": "https://changegamer.ai/api/articles/chunking-documents-for-rag.json" }, { "slug": "choosing-embedding-models", "title": "How to Choose an Embedding Model for RAG (and Version It Like a Schema)", "description": "Embedding selection as an operations problem: the criteria that dominate total cost of ownership, the never-mix-spaces invariant, reindex migrations with dual indexes, and where quantization fits once cost shows up.", "kind": "sub", "order": 3, "html": "https://changegamer.ai/articles/choosing-embedding-models", "markdown": "https://changegamer.ai/articles/choosing-embedding-models.md", "json": "https://changegamer.ai/api/articles/choosing-embedding-models.json" }, { "slug": "hybrid-retrieval-fusion", "title": "How to Combine Keyword and Vector Search in RAG", "description": "Why hybrid retrieval is the production default rather than an upgrade: complementary failure modes of lexical and dense search, reciprocal rank fusion versus weighted scoring, parameter choices, and how filtering interacts with fusion.", "kind": "sub", "order": 4, "html": "https://changegamer.ai/articles/hybrid-retrieval-fusion", "markdown": "https://changegamer.ai/articles/hybrid-retrieval-fusion.md", "json": "https://changegamer.ai/api/articles/hybrid-retrieval-fusion.json" }, { "slug": "reranking-retrieved-results", "title": "When and How to Rerank Retrieved Documents in RAG", "description": "Reranking as a budget decision: why first-stage ranking misorders good evidence, when cross-encoder reranking pays for itself, how to pick candidate depth at the knee, gating by query difficulty, and deduplicating after fusion.", "kind": "sub", "order": 5, "html": "https://changegamer.ai/articles/reranking-retrieved-results", "markdown": "https://changegamer.ai/articles/reranking-retrieved-results.md", "json": "https://changegamer.ai/api/articles/reranking-retrieved-results.json" }, { "slug": "evaluating-rag-systems", "title": "How to Evaluate a RAG System (Retrieval Metrics, Generation Metrics, CI Gates)", "description": "The evaluation harness that keeps RAG changeable: golden-set construction, retrieval metrics separated from generation metrics, LLM-as-judge screening with human acceptance, CI regression gates, and the logged-query flywheel.", "kind": "sub", "order": 6, "html": "https://changegamer.ai/articles/evaluating-rag-systems", "markdown": "https://changegamer.ai/articles/evaluating-rag-systems.md", "json": "https://changegamer.ai/api/articles/evaluating-rag-systems.json" }, { "slug": "rag-index-freshness", "title": "How to Keep a RAG Index Fresh (Staleness Bounds, Not Vibes)", "description": "Freshness as an engineered property: per-source staleness contracts, document versioning and tombstones, effective-date filtering, deletion propagation with reconciliation backstops, and the sync-lag metrics that predict stale answers before users report them.", "kind": "sub", "order": 7, "html": "https://changegamer.ai/articles/rag-index-freshness", "markdown": "https://changegamer.ai/articles/rag-index-freshness.md", "json": "https://changegamer.ai/api/articles/rag-index-freshness.json" }, { "slug": "multi-tenant-rag-permissions", "title": "Permission-Aware Retrieval: Multi-Tenant RAG Without Leaks", "description": "How to enforce access control inside retrieval for multi-tenant RAG: authorization as mandatory pre-filters derived from the authenticated principal, tenant isolation mechanics, permission-negative testing, and the post-filter trap that produces empty answers.", "kind": "sub", "order": 8, "html": "https://changegamer.ai/articles/multi-tenant-rag-permissions", "markdown": "https://changegamer.ai/articles/multi-tenant-rag-permissions.md", "json": "https://changegamer.ai/api/articles/multi-tenant-rag-permissions.json" }, { "slug": "agentic-retrieval-patterns", "title": "Retrieval as a Tool: Agentic RAG Patterns That Survive Production", "description": "When AI agents consume retrieval as a tool rather than a pipeline stage: narrow tool contracts, hard budgets on iterative retrieval, query decomposition, provenance-carrying results, and keeping trust boundaries inside the retrieval path.", "kind": "sub", "order": 9, "html": "https://changegamer.ai/articles/agentic-retrieval-patterns", "markdown": "https://changegamer.ai/articles/agentic-retrieval-patterns.md", "json": "https://changegamer.ai/api/articles/agentic-retrieval-patterns.json" }, { "slug": "rag-cost-and-latency", "title": "How to Cut RAG Cost and Latency Without Cutting Quality", "description": "RAG cost and latency as engineered budgets: where the money actually goes, caching layers and their hit-rate economics, routing queries to right-sized models, bounding retrieval fan-out, the hidden lines (reindex migrations, eval compute), and p95 discipline.", "kind": "sub", "order": 10, "html": "https://changegamer.ai/articles/rag-cost-and-latency", "markdown": "https://changegamer.ai/articles/rag-cost-and-latency.md", "json": "https://changegamer.ai/api/articles/rag-cost-and-latency.json" }, { "slug": "rag-failure-modes-runbook", "title": "Common RAG Failure Modes and How to Fix Them", "description": "An operator runbook for the four RAG failure classes with no dedicated deep-dive elsewhere: retrieval miss, context overload, injection via content at ingestion time, and silent quality degradation — symptom, first diagnostic, and fix for each.", "kind": "sub", "order": 11, "html": "https://changegamer.ai/articles/rag-failure-modes-runbook", "markdown": "https://changegamer.ai/articles/rag-failure-modes-runbook.md", "json": "https://changegamer.ai/api/articles/rag-failure-modes-runbook.json" }, { "slug": "graphrag-vs-vector-rag", "title": "GraphRAG vs Vector RAG: When to Use a Knowledge Graph Instead", "description": "A decision framework for choosing graph-structured retrieval over standard vector RAG: which query types GraphRAG actually wins, what building a knowledge graph costs, named implementations, and hybrid vector-plus-graph patterns.", "kind": "sub", "order": 12, "html": "https://changegamer.ai/articles/graphrag-vs-vector-rag", "markdown": "https://changegamer.ai/articles/graphrag-vs-vector-rag.md", "json": "https://changegamer.ai/api/articles/graphrag-vs-vector-rag.json" }, { "slug": "when-rag-is-the-wrong-answer", "title": "When Is RAG the Wrong Answer? A Decision Guide", "description": "A worked decision guide for the four real alternatives to retrieval-augmented generation: including knowledge directly, querying structured data with text-to-SQL, fine-tuning for behavior change, and graph-based retrieval for entity relationships.", "kind": "sub", "order": 13, "html": "https://changegamer.ai/articles/when-rag-is-the-wrong-answer", "markdown": "https://changegamer.ai/articles/when-rag-is-the-wrong-answer.md", "json": "https://changegamer.ai/api/articles/when-rag-is-the-wrong-answer.json" } ] }, "navigation": { "pillar": { "slug": "rag-in-production", "title": "Agentic RAG in Production: The Complete Operator Guide", "description": "The operator playbook for agentic RAG in production: ingestion, chunking, hybrid retrieval, reranking, evaluation, freshness and cost.", "kind": "pillar", "order": 0, "html": "https://changegamer.ai/articles/rag-in-production", "markdown": "https://changegamer.ai/articles/rag-in-production.md", "json": "https://changegamer.ai/api/articles/rag-in-production.json" }, "previous": { "slug": "reranking-retrieved-results", "title": "When and How to Rerank Retrieved Documents in RAG", "description": "Reranking as a budget decision: why first-stage ranking misorders good evidence, when cross-encoder reranking pays for itself, how to pick candidate depth at the knee, gating by query difficulty, and deduplicating after fusion.", "kind": "sub", "order": 5, "html": "https://changegamer.ai/articles/reranking-retrieved-results", "markdown": "https://changegamer.ai/articles/reranking-retrieved-results.md", "json": "https://changegamer.ai/api/articles/reranking-retrieved-results.json" }, "next": { "slug": "rag-index-freshness", "title": "How to Keep a RAG Index Fresh (Staleness Bounds, Not Vibes)", "description": "Freshness as an engineered property: per-source staleness contracts, document versioning and tombstones, effective-date filtering, deletion propagation with reconciliation backstops, and the sync-lag metrics that predict stale answers before users report them.", "kind": "sub", "order": 7, "html": "https://changegamer.ai/articles/rag-index-freshness", "markdown": "https://changegamer.ai/articles/rag-index-freshness.md", "json": "https://changegamer.ai/api/articles/rag-index-freshness.json" } }, "resources": [ { "slug": "evaluating-ai-agents", "html": "https://changegamer.ai/resources/evaluating-ai-agents", "markdown": "https://changegamer.ai/resources/evaluating-ai-agents.md", "json": "https://changegamer.ai/api/resources/evaluating-ai-agents.json" }, { "slug": "testing-ai-agents", "html": "https://changegamer.ai/resources/testing-ai-agents", "markdown": "https://changegamer.ai/resources/testing-ai-agents.md", "json": "https://changegamer.ai/api/resources/testing-ai-agents.json" }, { "slug": "rag-retrieval-for-agents", "html": "https://changegamer.ai/resources/rag-retrieval-for-agents", "markdown": "https://changegamer.ai/resources/rag-retrieval-for-agents.md", "json": "https://changegamer.ai/api/resources/rag-retrieval-for-agents.json" }, { "slug": "agent-observability", "html": "https://changegamer.ai/resources/agent-observability", "markdown": "https://changegamer.ai/resources/agent-observability.md", "json": "https://changegamer.ai/api/resources/agent-observability.json" }, { "slug": "embeddings-vector-search", "html": "https://changegamer.ai/resources/embeddings-vector-search", "markdown": "https://changegamer.ai/resources/embeddings-vector-search.md", "json": "https://changegamer.ai/api/resources/embeddings-vector-search.json" } ] }