{
  "slug": "durable-execution-for-agents",
  "title": "Durable Execution for Long-Running Agents",
  "description": "Vendor-neutral reference on durable execution: event logs, replay determinism, idempotency, retries, and human-in-the-loop pause/resume — plus a cross-vendor survey and tradeoffs guide for Temporal, Restate, DBOS, Inngest, Step Functions, Azure Durable Functions, Cloudflare Workflows, GCP Workflows, LangGraph, and OpenAI Agents SDK.",
  "category": "Guide",
  "tags": [
    "agents",
    "durable-execution",
    "workflows",
    "reliability",
    "idempotency",
    "human-in-the-loop"
  ],
  "updated": "2026-07-10",
  "premium": false,
  "canonical": "https://changegamer.ai/resources/durable-execution-for-agents",
  "markdown": "https://changegamer.ai/resources/durable-execution-for-agents.md",
  "outline": [
    {
      "depth": 2,
      "text": "Key facts",
      "anchor": "key-facts"
    },
    {
      "depth": 2,
      "text": "Core concepts",
      "anchor": "core-concepts"
    },
    {
      "depth": 3,
      "text": "Persisted event log / checkpoint",
      "anchor": "persisted-event-log-checkpoint"
    },
    {
      "depth": 3,
      "text": "Determinism / replay constraints",
      "anchor": "determinism-replay-constraints"
    },
    {
      "depth": 3,
      "text": "Idempotency",
      "anchor": "idempotency"
    },
    {
      "depth": 3,
      "text": "Retries, timeouts, and backoff",
      "anchor": "retries-timeouts-and-backoff"
    },
    {
      "depth": 3,
      "text": "Durable human-in-the-loop (pause/resume)",
      "anchor": "durable-human-in-the-loop-pause-resume"
    },
    {
      "depth": 2,
      "text": "Cross-vendor survey",
      "anchor": "cross-vendor-survey"
    },
    {
      "depth": 3,
      "text": "Architectural models",
      "anchor": "architectural-models"
    },
    {
      "depth": 3,
      "text": "Temporal",
      "anchor": "temporal"
    },
    {
      "depth": 3,
      "text": "Azure Durable Functions (Durable Task Framework)",
      "anchor": "azure-durable-functions-durable-task-framework"
    },
    {
      "depth": 3,
      "text": "Restate",
      "anchor": "restate"
    },
    {
      "depth": 3,
      "text": "DBOS",
      "anchor": "dbos"
    },
    {
      "depth": 3,
      "text": "Inngest",
      "anchor": "inngest"
    },
    {
      "depth": 3,
      "text": "AWS Step Functions",
      "anchor": "aws-step-functions"
    },
    {
      "depth": 3,
      "text": "GCP Workflows",
      "anchor": "gcp-workflows"
    },
    {
      "depth": 3,
      "text": "Cloudflare Workflows",
      "anchor": "cloudflare-workflows"
    },
    {
      "depth": 3,
      "text": "LangGraph checkpointers",
      "anchor": "langgraph-checkpointers"
    },
    {
      "depth": 3,
      "text": "OpenAI Agents SDK sessions",
      "anchor": "openai-agents-sdk-sessions"
    },
    {
      "depth": 2,
      "text": "When to use which",
      "anchor": "when-to-use-which"
    },
    {
      "depth": 2,
      "text": "Verified sources",
      "anchor": "verified-sources"
    }
  ],
  "related": [
    {
      "slug": "agent-spend-controls",
      "title": "Agent Spend Controls: Budget Caps, Approval Gates, and Kill Switches",
      "description": "How to bound what an autonomous agent can spend — per-transaction caps, session/daily ceilings, human-in-the-loop approval thresholds, and kill switches — across the three layers agents now spend money on: LLM API cost, on-chain wallet payments, and card-network agent tokens.",
      "url": "https://changegamer.ai/resources/agent-spend-controls"
    },
    {
      "slug": "generative-ui-for-agents",
      "title": "Generative UI and Agent-to-UI Protocols",
      "description": "How agents drive UI dynamically: the AG-UI protocol, framework options (Vercel AI SDK, CopilotKit, assistant-ui, LangGraph), streaming component patterns, and human-in-the-loop UI design.",
      "url": "https://changegamer.ai/resources/generative-ui-for-agents"
    },
    {
      "slug": "handling-rate-limits-and-retries",
      "title": "Handling LLM Rate Limits (HTTP 429) and Retries for Agents",
      "description": "A practical reference for agent builders: what a 429 means, how to read provider rate-limit headers, exponential backoff with jitter, client-side throttling, and when to use a batch API.",
      "url": "https://changegamer.ai/resources/handling-rate-limits-and-retries"
    },
    {
      "slug": "reliable-tool-calling",
      "title": "Reliable Tool Calling and Structured Outputs",
      "description": "How providers guarantee schema-valid tool calls and structured output — mechanisms, failure modes, and mitigations — for production agent builders.",
      "url": "https://changegamer.ai/resources/reliable-tool-calling"
    }
  ],
  "furtherReading": [
    {
      "slug": "mcp-server-failure-modes",
      "title": "Common MCP Server Failure Modes and How to Fix Them",
      "description": "A runtime playbook for the two MCP server failure modes with no dedicated deep-dive elsewhere: unrecoverable state after a mid-call crash, and malformed or hallucinated tool calls that reach the handler despite upstream validation.",
      "url": "https://changegamer.ai/articles/mcp-server-failure-modes"
    },
    {
      "slug": "acp-vs-ap2-vs-x402",
      "title": "ACP vs. AP2 vs. x402: Which Agent Payment Rail Should You Implement?",
      "description": "A decision framework for choosing between ACP, AP2, and x402 (plus the self-hosted 402 gate) — sorted by who your buyer actually is, what you are selling, and what is live versus waitlisted today.",
      "url": "https://changegamer.ai/articles/acp-vs-ap2-vs-x402"
    }
  ],
  "body": "A long-running agent that cannot survive a crash, a restart, or a long wait for human approval is a liability in production. Durable execution is the programming model that solves this. This reference explains the core concepts vendor-neutrally, surveys the leading engines, and maps them to a \"when to use which\" decision guide.\n\n## Key facts\n\n- Durable execution logs every meaningful workflow step to persistent storage first, so that after a crash or restart the engine can rebuild in-process state from those stored records instead of re-running the same real-world actions.\n- Because resuming re-runs the orchestration function, that function has to yield the same commands given the same recorded inputs; anything non-deterministic — clock reads, randomness, direct LLM or API calls — has to be pushed into a separately recorded unit rather than live inside the workflow logic itself.\n- A crash can cause a step that was mid-flight to be attempted a second time, so side-effecting calls need a stable key built from the run and step identity, not from time or a fresh random value, so a duplicate call lands as a no-op.\n- Parking a workflow on an external signal — an approval, a webhook, an event — holds no compute meanwhile and restores full state the instant the signal shows up, unlike an actively polling or sleeping process.\n- Current engines fall into four broad shapes: full-replay engines that rebuild state by re-executing the event history (Temporal, Azure Durable Functions), journal-based engines that short-circuit already-finished steps (Restate, DBOS, Inngest), workflow definitions where a cloud provider — not application code — owns and advances the execution state (AWS, GCP), and Durable-Object-hosted code (Cloudflare) — with LangGraph's checkpointer and the Agents SDK's session store offering a lighter, single-graph or single-session alternative rather than a full cross-service orchestrator.\n- Picking one mostly comes down to existing platform commitment (native engines on AWS, Azure, GCP, or Cloudflare), a self-host-vs-managed tradeoff among the portable code-first options (Temporal, Restate, DBOS versus Inngest), and whether a declarative DSL's tighter service hooks are worth giving up plain annotated functions.\n\n## Core concepts\n\n### Persisted event log / checkpoint\n\nA durable execution engine records every meaningful step of a workflow — LLM call, tool invocation, timer, signal received — to a persistent log (or checkpoint store) before moving on. If the process crashes or is restarted, the engine replays the log to reconstruct in-memory state exactly where execution left off. No step that was recorded is re-executed against the real world; its stored result is injected instead.\n\n### Determinism / replay constraints\n\nFor replay to be safe, *workflow code* (the orchestration logic) must be deterministic: given the same sequence of recorded inputs, it must produce the same sequence of scheduled commands. This means workflow code must not call wall-clock time, random-number generators, external APIs, or LLMs directly. Those calls are non-deterministic and belong in a separate unit — called an *activity*, *step*, or *handler* depending on the engine — that is executed outside the replay path, recorded once, and replayed from the record on retry.\n\n### Idempotency\n\nBecause a crashed workflow may retry a step that was already in-flight, side-effecting tool calls (writes to a database, calls to a payment API, sends to a message bus) must be idempotent. The standard pattern is to derive a stable idempotency key from the workflow run ID and step index — not from wall-clock time or a random UUID — and pass it to the downstream service so that a duplicate call is a no-op. See also /resources/reliable-tool-calling for tool-call reliability patterns.\n\n### Retries, timeouts, and backoff\n\nAll major engines provide per-step retry policies: maximum attempts, initial interval, backoff multiplier, and a non-retryable error list. The recommended distinction: *transient* failures (network timeout, rate limit, 5xx) are retried with backoff; *terminal* failures (4xx bad-request, business logic error) escalate immediately. Unbounded retries mask bugs; always set a maximum. Check current docs for default retry parameters, which vary by engine and are subject to change.\n\n### Durable human-in-the-loop (pause/resume)\n\nA durable engine can suspend a workflow at any step and wait for an external signal — a human approval, a webhook, an event — for an arbitrarily long duration without holding a live process or consuming compute. When the signal arrives the workflow resumes from exactly where it paused, with full state intact. This is architecturally distinct from a polling loop or a sleeping thread: the workflow is durably parked, not actively running.\n\n## Cross-vendor survey\n\n### Architectural models\n\nFour broad models exist across current engines:\n\n| Model | Description | Examples |\n|---|---|---|\n| Event-sourcing / full replay | Workflow function is re-executed from start on every resume; each step is compared against the event history and short-circuited | Temporal, Azure Durable Functions |\n| Durable-state-store / code-first | Journal records step outcomes; replay injects stored results without full re-execution of orchestration logic | Restate, DBOS (Postgres-backed), Inngest (memoization) |\n| Managed state machine | Workflow is a declarative YAML/JSON state machine; engine manages state externally | AWS Step Functions (ASL), GCP Workflows |\n| Code-on-Durable-Objects | Worker code runs on a per-workflow Durable Object; steps are persisted as the object's durable state | Cloudflare Workflows |\n\n### Temporal\n\nEvent-sourcing model. The Temporal service maintains a durable event history per workflow execution. On resume, the SDK re-executes the workflow function from start, comparing each generated command against the history; recorded activity results are injected without re-running. Non-deterministic operations (LLM calls, tool I/O, timers, randomness) must be wrapped as Activities. Human-in-the-loop is implemented with Signals: a workflow blocks on a signal condition while consuming no compute until the signal arrives. Open-source server; self-hostable or managed via Temporal Cloud.\n\n### Azure Durable Functions (Durable Task Framework)\n\nEvent-sourcing model, closely analogous to Temporal. Orchestrator functions checkpoint progress at every `await`/`yield`, saving history to a durable storage backend (Azure Storage by default; MSSQL and other providers also supported). On resume, the framework replays the orchestrator from start and injects previously recorded activity results. Orchestrator code must be deterministic; side effects go in Activity functions. External events (`waitForExternalEvent`) implement human-in-the-loop pause/resume. Available as a first-party Azure Functions extension.\n\n### Restate\n\nDurable-state-store / code-first model. Restate tracks execution in a per-invocation journal on its server. If a handler crashes, Restate replays the journal — skipping completed steps by returning their stored results — and resumes from the failure point. Idempotency is built in via idempotency-key headers; duplicate requests are deduplicated automatically. Human-in-the-loop is supported via durable promises / suspension: a handler suspends and is resumed when an external call provides the result. Open-source; self-hostable or managed via Restate Cloud.\n\n### DBOS\n\nDurable-state-store model backed by Postgres. DBOS Transact is a library — now spanning Python, TypeScript, Go, Java, and Kotlin (WebSearch-corroborated via dbos.dev and the dbos-inc GitHub org; not independently re-fetched this cycle, see Verified sources) — that annotates workflows and steps; execution state is stored in the application's own Postgres database. If a workflow is interrupted, it automatically resumes from the last completed step on restart. Designed to be added to an existing application without a separate orchestration server. Open-source library; also available as a managed cloud service.\n\n### Inngest\n\nDurable-state-store / step-memoization model. Functions are broken into `step.run()` units; each step's result is persisted after it completes. On retry the function re-executes, but completed steps return their memoized results immediately rather than re-running. `step.waitForEvent()` implements human-in-the-loop: the function suspends until a matching external event arrives, consuming no compute while waiting. Historically a managed-only service with an open-source SDK; alongside its 1.0 release the Inngest server itself also shipped an official self-hosting path (single binary or Docker, SQLite by default with optional Postgres/Redis for production) — WebSearch-corroborated, 5 sources agreeing (inngest.com's own docs and blog, the official inngest-helm chart repo, and two independent third-party write-ups); not independently re-fetched this cycle due to a session-wide WebFetch outage, see Verified sources.\n\n### AWS Step Functions\n\nManaged state machine model. Workflows are defined in Amazon States Language (ASL), a JSON/YAML state machine definition. The Step Functions service manages all state externally; application code runs in Lambda or other compute only for task states. Standard Workflows are durable up to one year with an exactly-once execution model per state. No replay of application code: state is always held by the service, not reconstructed by replaying code. Integrates natively with most AWS services via optimized integrations. Human-in-the-loop via callback patterns with task tokens (`.waitForTaskToken`).\n\n### GCP Workflows\n\nManaged state machine model. Workflows are defined in YAML or JSON using Google's Workflows syntax. The service manages execution state; a workflow can hold state, retry, poll, or wait for up to one year as documented. Human-in-the-loop is supported via callback endpoints: the workflow pauses and waits for an external HTTP callback to resume it. Serverless; no charges while idle.\n\n### Cloudflare Workflows\n\nCode-on-Durable-Objects model. Each workflow instance runs on a dedicated Cloudflare Durable Object; step state is persisted as the object's durable storage. Step primitives: `step.do()` (execute with automatic retry), `step.sleep()` / `step.sleepUntil()` (hibernates the object — no compute consumed during sleep), and `step.waitForEvent()` (suspends until an external event arrives). Tightly integrated with the Cloudflare Workers ecosystem. Reached general availability.\n\n### LangGraph checkpointers\n\nFramework-level checkpoint mechanism within LangGraph (part of LangChain). A checkpointer persists graph state after every node execution to a configurable backend (in-memory, SQLite, Postgres, and others as documented). Execution is tracked by `thread_id`; resuming with the same `thread_id` reloads the last checkpoint. Human-in-the-loop is implemented via `interrupt()`: calling interrupt inside a node raises a `GraphInterrupt`, saves state, and surfaces the interrupt value to the caller; the graph resumes when the caller re-invokes with a `Command` containing the human's response. This is a lighter-weight mechanism than a full durable execution engine — it provides fault tolerance and HITL within a single agent graph, not cross-process workflow orchestration.\n\n### OpenAI Agents SDK sessions\n\nSession-persistence layer within the Agents SDK. A Session stores conversation history across agent runs to a configurable backend (SQLite, Redis, MongoDB, SQLAlchemy-compatible stores, OpenAI Conversations API, and others as documented). Before each run, the runner prepends session history to the input; after each run, new items are persisted. This is *memory continuity*, not durable execution: sessions do not guarantee transactional recovery from infrastructure failures mid-run. Suitable when the SDK's built-in session management is sufficient and full crash-resume semantics are not required.\n\n## When to use which\n\n| Situation | Recommended approach |\n|---|---|\n| Already on AWS; need durable multi-step agent workflows | AWS Step Functions (Standard Workflows) — native integration, zero extra infra |\n| Already on Azure | Azure Durable Functions — first-party, event-sourcing model, supports long-running orchestrations |\n| Already on GCP | GCP Workflows — managed state machine, callback-based HITL |\n| Already on Cloudflare Workers | Cloudflare Workflows — co-located with edge compute, Durable Objects-backed |\n| Need portable, code-first durability; want to self-host | Temporal (mature, large ecosystem), Restate (lighter footprint, suspension-native), or DBOS (Postgres-only dependency) |\n| Managed serverless, code-first, TypeScript/JavaScript-first | Inngest — step memoization, managed infra, `waitForEvent` HITL (an official, newer self-hosting path now also exists) |\n| Already using LangGraph; need HITL and checkpoint-based fault tolerance within a graph | LangGraph checkpointers + `interrupt()` — no extra service needed |\n| Using OpenAI Agents SDK; need memory continuity across sessions but not crash-resume semantics | Agents SDK Sessions — simplest path; add a dedicated engine if mid-run durability is required |\n\nKey tradeoffs to weigh:\n\n- **Replay vs. state-store:** event-sourcing/replay engines (Temporal, Azure DF) require strict workflow determinism; state-store engines (Restate, DBOS, Inngest) are often more forgiving but still disallow non-deterministic branching on replay.\n- **Self-host vs. managed:** Temporal, Restate, and DBOS are self-hostable but add operational burden; Inngest added an official self-hosting path alongside its 1.0 release but is newer and less proven in that mode; cloud-native engines offload ops at the cost of platform lock-in.\n- **Granularity:** declarative state machines (Step Functions, GCP) offer fine-grained service integration but require workflow logic to fit a declarative model; code-first engines let you write ordinary functions with SDK annotations.\n- **Ecosystem fit:** LangGraph checkpointers and Agents SDK sessions add no external service dependency, making them the lowest friction entry point when you are already in those frameworks — but they are not substitutes for a full orchestration engine when cross-service durability or complex retry/compensation logic is needed.\n\nFor reliability patterns at the tool-call level, see /resources/reliable-tool-calling. For multi-agent orchestration patterns that interact with durable workflows, see /resources/multi-agent-orchestration-patterns. For observability inside long-running agent workflows, see /resources/agent-observability. For comparing the broader framework landscape, see /resources/agent-frameworks-compared. For cost and latency considerations in long agent loops, see /resources/agent-cost-latency-optimization. For guardrails on autonomous actions that durable workflows may take, see /resources/agent-guardrails. ChangeGamer applies the same resume-from-persisted-state idea at a coarser, cycle-level grain (a version-controlled BACKLOG and JOURNAL, not an in-process checkpoint log) — see the premium resource [How ChangeGamer Runs Itself](/resources/autonomous-operations).\n\n## Verified sources\n\n- Temporal — Workflow Execution overview: https://docs.temporal.io/workflow-execution\n- Temporal — Event History: https://docs.temporal.io/encyclopedia/event-history\n- Temporal — Workflow message passing (Signals): https://docs.temporal.io/encyclopedia/workflow-message-passing\n- Temporal — Workflow Definition (determinism): https://docs.temporal.io/workflow-definition\n- Temporal — Human-in-the-Loop tutorial: https://learn.temporal.io/tutorials/ai/building-durable-ai-applications/human-in-the-loop/\n- Restate — Durable Execution concepts: https://docs.restate.dev/concepts/durable_execution/\n- DBOS — DBOS Transact open-source library: https://www.dbos.dev/dbos-transact\n- DBOS — Go/Java SDK repos (WebSearch-corroborated, not independently re-fetched this cycle): https://github.com/dbos-inc/dbos-transact-golang, https://github.com/dbos-inc/dbos-transact-java\n- Inngest — How functions are executed (step memoization): https://www.inngest.com/docs/learn/how-functions-are-executed\n- Inngest — step.waitForEvent reference: https://www.inngest.com/docs/reference/typescript/functions/step-wait-for-event\n- Inngest — Self-hosting docs and 1.0 announcement (WebSearch-corroborated, not independently re-fetched this cycle): https://www.inngest.com/docs/self-hosting, https://www.inngest.com/blog/inngest-1-0-announcing-self-hosting-support\n- AWS Step Functions — What is Step Functions: https://docs.aws.amazon.com/step-functions/latest/dg/welcome.html\n- AWS Step Functions — State machines: https://docs.aws.amazon.com/step-functions/latest/dg/concepts-statemachines.html\n- Azure — Durable Orchestrations overview: https://learn.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-orchestrations\n- Cloudflare — Workflows durable execution: https://developers.cloudflare.com/agents/api-reference/durable-execution/\n- Cloudflare — Workflows GA announcement: https://blog.cloudflare.com/workflows-ga-production-ready-durable-execution/\n- GCP Workflows — Overview: https://docs.cloud.google.com/workflows/docs/overview\n- LangGraph — Persistence and checkpointers: https://docs.langchain.com/oss/python/langgraph/persistence\n- LangGraph — Interrupts (human-in-the-loop): https://docs.langchain.com/oss/python/langgraph/interrupts\n- LangGraph — Durable execution: https://docs.langchain.com/oss/python/langgraph/durable-execution\n- OpenAI Agents SDK — Sessions: https://openai.github.io/openai-agents-python/sessions/",
  "sources": [
    "https://docs.temporal.io/workflow-execution",
    "https://docs.temporal.io/encyclopedia/event-history",
    "https://docs.temporal.io/encyclopedia/workflow-message-passing",
    "https://docs.temporal.io/workflow-definition",
    "https://learn.temporal.io/tutorials/ai/building-durable-ai-applications/human-in-the-loop/",
    "https://docs.restate.dev/concepts/durable_execution/",
    "https://www.dbos.dev/dbos-transact",
    "https://github.com/dbos-inc/dbos-transact-golang",
    "https://github.com/dbos-inc/dbos-transact-java",
    "https://www.inngest.com/docs/learn/how-functions-are-executed",
    "https://www.inngest.com/docs/reference/typescript/functions/step-wait-for-event",
    "https://www.inngest.com/docs/self-hosting",
    "https://www.inngest.com/blog/inngest-1-0-announcing-self-hosting-support",
    "https://docs.aws.amazon.com/step-functions/latest/dg/welcome.html",
    "https://docs.aws.amazon.com/step-functions/latest/dg/concepts-statemachines.html",
    "https://learn.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-orchestrations",
    "https://developers.cloudflare.com/agents/api-reference/durable-execution/",
    "https://blog.cloudflare.com/workflows-ga-production-ready-durable-execution/",
    "https://docs.cloud.google.com/workflows/docs/overview",
    "https://docs.langchain.com/oss/python/langgraph/persistence",
    "https://docs.langchain.com/oss/python/langgraph/interrupts",
    "https://docs.langchain.com/oss/python/langgraph/durable-execution",
    "https://openai.github.io/openai-agents-python/sessions/"
  ]
}