activity error handling

38 notes, read out of this brain and free to use. Each one was extracted from a source and is re-checked against its exam.

Circuit breaker and Activity retry policy pairing

A breaker that opens after a burst of failures pairs well with a retry policy that backs off, so an open breaker returns fast errors that Temporal retries later rather than each attempt waiting on a timeout. Size the Activity retry policy according to the circuit breaker configuration.

Fast/Slow Retries pitfall: catching too broadly in Phase 1

Catch ActivityError specifically in Phase 1. Catching all exceptions may swallow errors that should propagate immediately, such as CancelledError in Python or PanicError in Go.

Fast/Slow Retries pitfall: setting finite MaximumAttempts in Phase 2

If you set a finite MaximumAttempts in Phase 2, it will eventually exhaust and propagate a failure to the Workflow. Only add a limit if the business process has a defined maximum wait time; pair it with a ScheduleToCloseTimeout to make the budget explicit.

Fast/Slow Retries Go example

package downstream import ( "time" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" ) func FastSlowRetryWorkflow(ctx workflow.Context, request string) (string, error) { log := workflow.GetLogger(ctx) // Phase 1: fast retries fastCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ StartToCloseTimeout: 30 * time.Second, RetryPolicy: &temporal.RetryPolicy{ InitialInterval: time.Second, BackoffCoefficient: 1.5, MaximumInterval: 30 * time.Second, MaximumAttempts: 10, }, }) var result string err := workflow.ExecuteActivity(fastCtx, CallDownstream, request).Get(fastCtx, &result) if err != nil { log.Warn("Fast retries exhausted — switching to slow retry phase", "request", request) // Phase 2: slow retries slowCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ StartToCloseTimeout: 30 * time.Second, RetryPolicy: &temporal.RetryPolicy{ InitialInterval: 5 * time.Minute, BackoffCoefficient: 1.0, // MaximumAttempts defaults to 0 (unlimited) }, }) err = workflow.ExecuteActivity(slowCtx, CallDownstream, request).Get(slowCtx, &result) } return result, err }

Fast/Slow Retries Python example

from datetime import timedelta from temporalio import workflow from temporalio.common import RetryPolicy from temporalio.exceptions import ActivityError import activities @workflow.defn class FastSlowRetryWorkflow: @workflow.run async def run(self, request: str) -> str: # Phase 1: fast retries fast_policy = RetryPolicy( initial_interval=timedelta(seconds=1), backoff_coefficient=1.5, maximum_interval=timedelta(seconds=30), maximum_attempts=10, ) try: return await workflow.execute_activity( activities.call_downstream, request, start_to_close_timeout=timedelta(seconds=30), retry_policy=fast_policy, ) except ActivityError: workflow.logger.warning( "Fast retries exhausted — switching to slow retry phase", extra={"request": request}, ) # Phase 2: slow retries slow_policy = RetryPolicy( initial_interval=timedelta(minutes=5), backoff_coefficient=1.0, ) return await workflow.execute_activity( activities.call_downstream, request, start_to_close_timeout=timedelta(seconds=30), retry_policy=slow_policy, )

Fast/Slow Retries pitfall: using exponential backoff in Phase 2

Do not use the default BackoffCoefficient of 2.0 in Phase 2, as it doubles the interval with each attempt. Set BackoffCoefficient to 1.0 in the slow phase to keep the interval fixed and predictable.

Fast/Slow Retries TypeScript example

import * as wf from '@temporalio/workflow'; import type * as activities from './activities'; const fastDownstream = wf.proxyActivities<typeof activities>({ startToCloseTimeout: '30s', retry: { initialInterval: '1s', backoffCoefficient: 1.5, maximumInterval: '30s', maximumAttempts: 10, }, }); const slowDownstream = wf.proxyActivities<typeof activities>({ startToCloseTimeout: '30s', retry: { initialInterval: '5m', backoffCoefficient: 1, // maximumAttempts defaults to unlimited }, }); export async function fastSlowRetryWorkflow(request: string): Promise<string> { // Phase 1: fast retries try { return await fastDownstream.callDownstream(request); } catch { wf.log.warn('Fast retries exhausted — switching to slow retry phase', { request }); // Phase 2: slow retries return await slowDownstream.callDownstream(request); } }

Fast/Slow Retries Java example

import io.temporal.activity.ActivityOptions; import io.temporal.common.RetryOptions; import io.temporal.failure.ActivityFailure; import io.temporal.workflow.Workflow; import java.time.Duration; public class FastSlowRetryWorkflowImpl implements FastSlowRetryWorkflow { @Override public String run(String request) { // Phase 1: fast retries DownstreamActivities fastActivities = Workflow.newActivityStub( DownstreamActivities.class, ActivityOptions.newBuilder() .setStartToCloseTimeout(Duration.ofSeconds(30)) .setRetryOptions(RetryOptions.newBuilder() .setInitialInterval(Duration.ofSeconds(1)) .setBackoffCoefficient(1.5) .setMaximumInterval(Duration.ofSeconds(30)) .setMaximumAttempts(10) .build()) .build() ); try { return fastActivities.callDownstream(request); } catch (ActivityFailure e) { Workflow.getLogger(getClass()).warn( "Fast retries exhausted — switching to slow retry phase: " + request ); // Phase 2: slow retries DownstreamActivities slowActivities = Workflow.newActivityStub( DownstreamActivities.class, ActivityOptions.newBuilder() .setStartToCloseTimeout(Duration.ofSeconds(30)) .setRetryOptions(RetryOptions.newBuilder() .setInitialInterval(Duration.ofMinutes(5)) .setBackoffCoefficient(1.0) // setMaximumAttempts not set — defaults to unlimited .build()) .build() ); return slowActivities.callDownstream(request); } } }

Fast/Slow Retries pattern overview

The Fast/Slow Retries pattern orchestrates two distinct retry phases within a Workflow: Phase 1 uses a fast retry policy with short intervals and bounded attempt counts for transient errors, while Phase 2 uses a slow retry policy with long fixed intervals and unlimited retries managed by the Temporal Service for extended outages. Use this pattern when a single RetryPolicy cannot adequately cover both brief blips and hour-long outages or maintenance windows.

Fast/Slow Retries Phase 1 purpose and configuration

Phase 1 executes the Activity with a short InitialInterval and bounded MaximumAttempts to recover from transient errors within seconds or minutes. Typical values are: InitialInterval of 1–5 seconds, BackoffCoefficient of 1.5–2.0 to spread retries and avoid overwhelming a briefly degraded system, and MaximumAttempts of 5–20 to cover a short transient period.

Fast/Slow Retries Phase 2 purpose and configuration

Phase 2 executes the Activity with a long InitialInterval and unlimited retries (MaximumAttempts unset or 0) managed by the Temporal Service. The Workflow catches the ActivityError from Phase 1, then executes the Activity again with the slow policy. Typical values are: InitialInterval of 1–15 minutes to avoid hammering a down system while recovering promptly, and BackoffCoefficient of 1.0 to keep the interval fixed and predictable.

Fast/Slow Retries Phase 2 timeout handling

Phase 2 runs indefinitely by default. If the business process has a maximum wait time, add a ScheduleToCloseTimeout or use a Workflow execution timeout to impose an outer bound on the slow retry phase.

Fast/Slow Retries implementation pattern

In the Workflow, execute the Activity with fast policy inside a try-catch block. If ActivityError is caught, log the phase transition, then execute the Activity again with the slow policy. The Workflow blocks until the Activity succeeds in Phase 2 or an outer timeout is reached.

Common pitfall: MaximumAttempts counts total attempts not retries

MaximumAttempts=3 means 3 total attempts (1 initial + 2 retries), not 3 retries after the initial attempt.

Common pitfall: disabling retries without safeguards

Setting maximum_attempts=1 on a call means any failure—including a Worker crash after the API responded—results in a permanent gap in execution. Use idempotency keys to protect against this scenario.

Common pitfall: ignoring ActivityError in Workflow

Exhausted retries raise an error in the Workflow. If you do not catch the ActivityError, the Workflow fails without any compensation or alerting.

Fixed Count of Retries Go implementation

In Go, set MaximumAttempts on the temporal.RetryPolicy passed to ActivityOptions. Check for RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED in the ActivityError to handle exhausted retries. Example: ```go func PaymentWorkflow(ctx workflow.Context, orderID string) (string, error) { ao := workflow.ActivityOptions{ StartToCloseTimeout: 10 * time.Second, RetryPolicy: &temporal.RetryPolicy{ MaximumAttempts: 3, }, } ctx = workflow.WithActivityOptions(ctx, ao) var result string err := workflow.ExecuteActivity(ctx, ChargePaymentAPI, orderID).Get(ctx, &result) if err != nil { var actErr *temporal.ActivityError if errors.As(err, &actErr) && actErr.RetryState() == enumspb.RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED { workflow.GetLogger(ctx).Error("Payment failed: all 3 attempts exhausted", "orderID", orderID) } return "", err } return result, nil } ```

Fixed Count of Retries Java implementation

In Java, set MaximumAttempts on the RetryOptions passed to ActivityOptions. Check for RetryState.RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED in the ActivityFailure catch block to handle exhausted retries. Example: ```java public class PaymentWorkflowImpl implements PaymentWorkflow { private final PaymentActivities activities = Workflow.newActivityStub( PaymentActivities.class, ActivityOptions.newBuilder() .setStartToCloseTimeout(Duration.ofSeconds(10)) .setRetryOptions(RetryOptions.newBuilder() .setMaximumAttempts(3) .build()) .build() ); @Override public String run(String orderId) { try { return activities.chargePaymentApi(orderId); } catch (ActivityFailure e) { if (e.getRetryState() == RetryState.RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED) { Workflow.getLogger(getClass()).error( "Payment failed: all 3 attempts exhausted: " + orderId, e); } throw e; } } } ```

Fixed Count of Retries TypeScript implementation

In TypeScript, set maximumAttempts on the retry property in proxyActivities(). Check for RetryState.MAXIMUM_ATTEMPTS_REACHED in the ActivityFailure catch block to handle exhausted retries. Example: ```typescript const { chargePaymentApi } = wf.proxyActivities<typeof activities>({ startToCloseTimeout: '10s', retry: { maximumAttempts: 3 }, }); export async function paymentWorkflow(orderId: string): Promise<string> { try { return await chargePaymentApi(orderId); } catch (err) { if (err instanceof wf.ActivityFailure && err.retryState === wf.RetryState.MAXIMUM_ATTEMPTS_REACHED) { wf.log.error('Payment failed: all 3 attempts exhausted', { orderId }); } throw err; } } ```

Disabling retries with maximum_attempts=1

Set maximum_attempts=1 (Python), MaximumAttempts=1 (Go/Java), or maximumAttempts=1 (TypeScript) to disable retries. The Activity starts once and any failure is immediately delivered to the Workflow. This is appropriate when the operation is not idempotent and a second attempt would cause a duplicate side effect such as a double charge or duplicate email. However, if a Worker crashes after the API call succeeds but before the result is recorded, Temporal will not retry and the call is lost.

Common pitfall: no timeout with low attempt cap

Without StartToCloseTimeout, a single hanging attempt can block all retries for minutes or hours, defeating the purpose of capping the number of attempts.

MaximumAttempts caps total Activity attempts including initial attempt

Set MaximumAttempts on the RetryPolicy to cap the total number of Activity execution attempts. The count includes the initial attempt, so MaximumAttempts=3 means one initial attempt plus two retries, for three total attempts. When the limit is reached, Temporal stops retrying and delivers an ActivityError to the Workflow.

Temporal default retry policy retries indefinitely

Temporal's default retry policy retries Activities indefinitely with exponential backoff. This is appropriate for most infrastructure failures but creates problems when the Activity calls a paid third-party API.

Fixed Count of Retries Python implementation

In Python, set maximum_attempts on the RetryPolicy passed to workflow.execute_activity(). Check for RetryState.MAXIMUM_ATTEMPTS_REACHED in the except ActivityError block to handle exhausted retries. Example: ```python @workflow.defn class PaymentWorkflow: @workflow.run async def run(self, order_id: str) -> str: try: return await workflow.execute_activity( activities.charge_payment_api, order_id, start_to_close_timeout=timedelta(seconds=10), retry_policy=RetryPolicy(maximum_attempts=3), ) except ActivityError as e: if e.retry_state == RetryState.MAXIMUM_ATTEMPTS_REACHED: workflow.logger.error( "Payment failed: all 3 attempts exhausted", extra={"order_id": order_id}, ) raise ```

Detecting SCHEDULE_TO_CLOSE timeout in exception handling

When the SLA expires and the budget runs out, distinguish SLA breaches from transient errors by inspecting the error cause. In Python, check that the ActivityError's cause is a TimeoutError with TimeoutType.SCHEDULE_TO_CLOSE. In TypeScript, check that the cause is a TimeoutFailure with TimeoutType.SCHEDULE_TO_CLOSE. In Go and Java, check for TimeoutType.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE. This allows logging or alerting specifically on SLA violations rather than treating all activity errors the same way.

Handle ActivityError explicitly when SLA expires

When the SLA expires, Temporal delivers an ActivityError to the Workflow. Catch it explicitly to send an alert, trigger a compensation, or record a breach in an audit log, rather than letting it propagate unhandled.

Non-critical Activity failure handling in Entity Workflows

For non-critical Activities like sending notifications, wrap execution in try-except blocks and log failures without terminating the entity. This allows the Workflow to continue operating even if notifications fail, isolating failures by concern.

Handling HTTP 429 rate limit errors from downstream APIs

When catching 429 errors from downstream APIs, check for the Retry-After response header. If present, raise an ApplicationError with next_retry_delay set to the header value converted to a timedelta. If the header is not present, raise the exception normally to use the Activity's Retry Policy. This respects the API's rate limit guidance while providing a sensible fallback strategy with exponential backoff.

Python SDK ApplicationError with next_retry_delay for rate limit handling

When handling 429 errors, raise an ApplicationError with next_retry_delay: raise ApplicationError('Rate limit exceeded', non_retryable=False, next_retry_delay=timedelta(seconds=int(retry_after))). This instructs Temporal to retry the Activity after the specified delay instead of using the Activity's Retry Policy.

Non-retryable application errors in activities

Use ApplicationError with non_retryable=True to mark errors that should not be retried. Example: When an HTTP 4xx error occurs during file download, raise ApplicationError(f"Client error downloading file: {e}", non_retryable=True) rather than letting Temporal retry the Activity, since 4xx errors indicate a permanent client-side problem.

ApplicationFailureException in .NET Activities

In .NET SDK Activities, any C# exception or custom exception raised from code is converted internally to an ApplicationFailureException. This allows the error's type, severity, and additional details to be sent to the Temporal Service, indexed by the Web UI, and serialized across language boundaries.

Temporal error classes should not be manually raised

In .NET SDK, Temporal uses several different error classes internally such as CanceledFailureException to handle platform-level failures like Workflow cancellation. These error classes should not be raised or manually implemented as they are tied to Temporal platform logic.

Catching ActivityFailureException in workflow code

In .NET SDK Workflows, you can catch ActivityFailureException to handle errors returned by Activities. When an Activity Failure occurs after all retries are exhausted or a non-retryable condition is met, you can decide how to handle the error using try/catch blocks, such as implementing a Saga Pattern to unwind steps performed up to the point of failure.

Activity Failure will not directly cause Workflow Failure

One of the core design principles of Temporal is that an Activity Failure will never directly cause a Workflow Failure. A Workflow should never return as Failed unless deliberately. The default retry policy associated with Temporal Activities retries them until reaching a certain timeout threshold. Activities will not return a failure to the Workflow until this condition or another non-retryable condition is met.

.NET custom exception example converted to ApplicationFailureException

A custom exception like InvalidDepartmentException thrown from an Activity is treated identically to throwing new ApplicationFailureException("Invalid department", errorType: "InvalidDepartmentException"). Both approaches work, but using ApplicationFailureException explicitly allows setting the nonRetryable parameter.

Handling ApplicationError in Go Workflows

Use errors.As(err, &applicationErr) to check for *ApplicationError. Call applicationErr.Error() to retrieve the error message. Call applicationErr.Details(&detailMsg) to extract strongly-typed details from errors created via NewApplicationError(). Call applicationErr.Type() to get the error type string for switching between different error types.

Error types returned by Activities and Child Workflows in Go

Within a Workflow, an Activity or Child Workflow execution might fail. If the Activity returns an error as errors.New() or fmt.Errorf(), that error is converted into *temporal.ApplicationError. If the Activity returns an error as temporal.NewNonRetryableApplicationError("error message", details), that error is returned as *temporal.ApplicationError. Other error types include *temporal.TimeoutError, *temporal.CanceledError, and *temporal.PanicError.

Example of handling Activity errors in Go Workflows

err := workflow.ExecuteActivity(ctx, YourActivity, ...).Get(ctx, nil) if err != nil { var applicationErr *ApplicationError if errors.As(err, &applicationErr) { workflow.GetLogger(ctx).Info("Application error", "error", applicationErr.Error()) var detailMsg string applicationErr.Details(&detailMsg) switch applicationErr.Type() { case "CustomErrTypeA": // handle CustomErrTypeA case CustomErrTypeB: // handle CustomErrTypeB default: } } var canceledErr *CanceledError if errors.As(err, &canceledErr) { // handle cancellation } var timeoutErr *TimeoutError if errors.As(err, &timeoutErr) { switch err.TimeoutType() { case commonpb.ScheduleToStart: // Handle ScheduleToStart timeout case commonpb.StartToClose: // Handle StartToClose timeout case commonpb.Heartbeat: // Handle heartbeat timeout default: } } var panicErr *PanicError if errors.As(err, &panicErr) { // handle panic with panicErr.Error() and panicErr.StackTrace() } }

Give your agent this brain