diff --git a/assets/temporal-cloud-serverless-worker-agentcore-role.yaml b/assets/temporal-cloud-serverless-worker-agentcore-role.yaml new file mode 100644 index 0000000..8872bcb --- /dev/null +++ b/assets/temporal-cloud-serverless-worker-agentcore-role.yaml @@ -0,0 +1,99 @@ +# CloudFormation template for creating an IAM role that Temporal Cloud can assume to invoke AgentCore runtimes. +AWSTemplateFormatVersion: "2010-09-09" +Description: Creates an IAM role that Temporal Cloud can assume to invoke Amazon Bedrock AgentCore runtimes for Serverless Workers. + +Parameters: + AssumeRoleExternalId: + Type: String + Description: The External ID provided by Temporal Cloud + AllowedPattern: "[a-zA-Z0-9_+=,.@-]*" + MinLength: 5 + MaxLength: 45 + + AgentRuntimeARNs: + Type: CommaDelimitedList + Description: >- + Comma-separated list of AgentCore runtime ARNs to invoke (e.g., + arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/temporal_worker-XXXXXXXXXX). + Append a wildcard (.../runtime/temporal_worker-XXXXXXXXXX*) to cover the + runtime's endpoints as well, which GetAgentRuntimeEndpoint is scoped to. + + RoleName: + Type: String + Default: "Temporal-Cloud-Serverless-Worker" + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Temporal Cloud Configuration" + Parameters: + - AssumeRoleExternalId + - Label: + default: "AgentCore Configuration" + Parameters: + - AgentRuntimeARNs + - RoleName + ParameterLabels: + AssumeRoleExternalId: + default: "External ID (provided by Temporal Cloud)" + AgentRuntimeARNs: + default: "AgentCore Runtime ARNs (comma-separated list)" + RoleName: + default: "IAM Role Name" + +Resources: + TemporalCloudServerlessWorker: + Type: AWS::IAM::Role + Properties: + RoleName: !Sub "${RoleName}-${AWS::StackName}" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + AWS: + [ + arn:aws:iam::902542641901:role/wci-lambda-invoke, + arn:aws:iam::160190466495:role/wci-lambda-invoke, + arn:aws:iam::819232936619:role/wci-lambda-invoke, + arn:aws:iam::829909441867:role/wci-lambda-invoke, + arn:aws:iam::354116250941:role/wci-lambda-invoke, + ] + Action: sts:AssumeRole + Condition: + StringEquals: + "sts:ExternalId": [!Ref AssumeRoleExternalId] + Description: "The role Temporal Cloud uses to invoke AgentCore runtimes for Serverless Workers" + MaxSessionDuration: 3600 # 1 hour + + TemporalCloudAgentCoreInvokePermissions: + Type: AWS::IAM::Policy + DependsOn: TemporalCloudServerlessWorker + Properties: + PolicyName: "Temporal-Cloud-AgentCore-Invoke-Permissions" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - bedrock-agentcore:InvokeAgentRuntime + - bedrock-agentcore:GetAgentRuntimeEndpoint + Resource: !Ref AgentRuntimeARNs + Roles: + - !Sub "${RoleName}-${AWS::StackName}" + +Outputs: + RoleARN: + Description: The ARN of the IAM role created for Temporal Cloud + Value: !GetAtt TemporalCloudServerlessWorker.Arn + Export: + Name: !Sub "${AWS::StackName}-RoleARN" + + RoleName: + Description: The name of the IAM role + Value: !Ref RoleName + + AgentRuntimeARNs: + Description: The AgentCore runtime ARNs that can be invoked + Value: !Join [", ", !Ref AgentRuntimeARNs] diff --git a/assets/temporal-self-hosted-serverless-worker-agentcore-role.yaml b/assets/temporal-self-hosted-serverless-worker-agentcore-role.yaml new file mode 100644 index 0000000..7e98aca --- /dev/null +++ b/assets/temporal-self-hosted-serverless-worker-agentcore-role.yaml @@ -0,0 +1,68 @@ +# CloudFormation template for creating an IAM role that a self-hosted Temporal Service can assume to invoke AgentCore runtimes. +AWSTemplateFormatVersion: '2010-09-09' +Description: + Creates an IAM role that a self-hosted Temporal Service can assume to invoke Amazon Bedrock AgentCore runtimes. + +Parameters: + TemporalIamRoleArn: + Type: String + Description: The ARN of the IAM role or user that the Temporal Service runs as. + + AssumeRoleExternalId: + Type: String + Description: A unique identifier to prevent confused deputy attacks. + AllowedPattern: '[a-zA-Z0-9_+=,.@-]*' + MinLength: 5 + MaxLength: 45 + + AgentRuntimeARNs: + Type: CommaDelimitedList + Description: >- + Comma-separated list of AgentCore Runtime ARNs that Temporal may access. Append a wildcard to each Runtime ARN + to include its endpoints. + + RoleName: + Type: String + Default: 'Temporal-AgentCore-Worker' + +Resources: + TemporalAgentCoreWorker: + Type: AWS::IAM::Role + Properties: + RoleName: !Ref RoleName + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + AWS: [!Ref TemporalIamRoleArn] + Action: sts:AssumeRole + Condition: + StringEquals: + 'sts:ExternalId': [!Ref AssumeRoleExternalId] + Description: The role the Temporal Service uses to invoke AgentCore runtimes for Serverless Workers + MaxSessionDuration: 3600 + + TemporalAgentCoreInvokePermissions: + Type: AWS::IAM::Policy + Properties: + PolicyName: 'Temporal-AgentCore-Invoke-Permissions' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - bedrock-agentcore:InvokeAgentRuntime + - bedrock-agentcore:GetAgentRuntimeEndpoint + Resource: !Ref AgentRuntimeARNs + Roles: + - !Ref TemporalAgentCoreWorker + +Outputs: + RoleARN: + Description: The ARN of the IAM role created for the Temporal Service + Value: !GetAtt TemporalAgentCoreWorker.Arn + + AgentRuntimeARNs: + Description: The AgentCore Runtime ARNs that Temporal may access + Value: !Join [', ', !Ref AgentRuntimeARNs] diff --git a/references/aws-agentcore/constraints.md b/references/aws-agentcore/constraints.md new file mode 100644 index 0000000..8bcb88b --- /dev/null +++ b/references/aws-agentcore/constraints.md @@ -0,0 +1,59 @@ +# AgentCore — Execution-model constraints + +These are the consequences of AgentCore Runtime's execution model. **The WCI invokes a named AgentCore Runtime endpoint when it needs capacity. The Runtime starts a standard long-running Worker inside a Runtime session, and that Worker polls until it drains or AgentCore ends its compute.** + +## Pre-release status + +AgentCore Runtime support is in **Pre-release**, and its APIs may change in backwards-incompatible ways. A Temporal Cloud Namespace needs Pre-release access, which the user requests through a [support ticket](https://docs.temporal.io/evaluate/cloud/support#support-ticket) or their account team. A self-hosted Temporal Service v1.32.0 or later does not need an access request. See `self-hosted.md`. + +The Cloud Namespace must be **hosted on AWS**. + +## Worker code is a standard Worker inside a Runtime handler + +AgentCore Workers are standard long-lived Temporal Workers. There is no serverless Worker package. The Runtime entry point must implement the AgentCore Runtime HTTP contract by serving `/invocations` and `/ping`. For a Serverless Worker, `/invocations` starts the Worker as background work and acknowledges the request without waiting for the Worker to stop. The per-language entry point is in the selected `sdk-.md` reference. + +## Compute lifetime + +- **Up to 8 hours per compute.** With the serverless microVM compute type, a Runtime session can run for up to 8 hours. The `maxLifetime` setting accepts 60–28800 seconds for microVM runtimes and defaults to 28800 seconds. Its timer starts when the microVM is created and cannot be reset. +- **No fixed invocation deadline.** An AgentCore Worker has no invocation deadline to monitor. It polls until it drains or AgentCore ends its compute. +- **The session is not durable storage.** AgentCore can resume a session on new compute after the previous compute ends, and a later Task can run on another Worker. Keep the state a Workflow needs in the Workflow or another durable store. +- **Process-local reuse is an optimization only.** The Worker can reuse initialization work, in-memory caches and temporary files while its Runtime compute remains available. Do not make Workflow progress depend on them. +- **Workflow duration is unbounded.** A Workflow continues across as many Worker processes as needed. Durable Timers and waits for approvals or external events do not keep compute running. + +## Two independent stop controls + +Two sets of controls determine when the Worker stops. Both must be configured deliberately. + +| Control | Owner | What it measures | +|---|---|---| +| Worker idle and graceful-shutdown policy | The Runtime handler you write | When the Worker has been idle. It then stops polling and waits for in-flight Activities. | +| `idleRuntimeSessionTimeout` | AgentCore lifecycle settings | Time since the last **AgentCore Runtime invocation** to the session. Default 900 seconds. Polling Temporal does **not** reset it. | +| `maxLifetime` | AgentCore lifecycle settings | Time since the compute was created. Default and microVM maximum: 28800 seconds (8 hours). | + +The idle session timeout is not a Worker idle timer, and it does not replace one. Implement a Worker shutdown policy in the handler: when the idle condition is met, stop polling and drain in-flight Activities before the handler releases the session. Choose the idle period and drain timeout for the workload, and keep both inside `maxLifetime`. + +`idleRuntimeSessionTimeout` must be less than or equal to `maxLifetime`, and both must be at least 60 seconds. A request that breaks these rules fails with `ValidationException`. When AgentCore ends compute, termination can take up to 15 seconds. + +In the Python sample, the Worker drains after `AGENTCORE_DEBOUNCE_SECONDS` without an Activity starting or finishing. The default is `60`. The sample's drain timeout is 120 seconds, and its `agentcore.json` keeps the AgentCore defaults (`idleRuntimeSessionTimeout: 900`, `maxLifetime: 28800`). These are configured values, not platform limits. + +## What bounds an Activity: AgentCore ending the compute + +AgentCore can end the compute before an Activity completes. Configure Activity timeouts. For long-running Activities, also configure [Activity Heartbeats](https://docs.temporal.io/encyclopedia/detecting-activity-failures#activity-heartbeat) so a retry can recover the work. A Worker crash follows standard retry semantics: the Activity Timeout fires and Temporal retries the Activity on another Worker. + +## Autoscaling + +Autoscaling is event-driven (`no-sync`). The WCI invokes individual Runtime sessions when it needs more capacity. It does not manage a target-sized pool. For the shared WCI lifecycle and inspection commands, see [Worker Controller Instance (WCI)](../wci.md). + +When the provider's capacity limit is reached, the WCI cannot add Workers. Tasks wait in the backlog without data loss. + +## One named endpoint per Worker Deployment Version + +The compute configuration names a Runtime **endpoint** ARN, not a Runtime ARN. Each Worker Deployment Version needs its own named endpoint pinned to one immutable Runtime version. Never configure the `DEFAULT` endpoint, because it moves to the latest Runtime version on every update. → `versioning.md`. + +## Region and high availability + +The AWS account must be in an [AgentCore-supported Region](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/agentcore-regions.html). Compute provider configuration is scoped to one region. On failover of a replicated Namespace, the WCI keeps invoking Workers in the original region until the compute provider is repointed manually. + +## AgentCore services are not configured by the compute provider + +Configuring AgentCore Runtime as a compute provider does not set up AgentCore Identity, Gateway, Policy, Memory or Observability. Call those services from Activities, and keep credentials out of Workflow state. Writing the agent itself (Workflow and Activity boundaries, Strands, model calls) is out of scope for this skill. Link to [Build a durable agent on Amazon Bedrock AgentCore](https://docs.temporal.io/guides/durable-agent-on-agentcore) instead. diff --git a/references/aws-agentcore/diagnostics.md b/references/aws-agentcore/diagnostics.md new file mode 100644 index 0000000..c193b67 --- /dev/null +++ b/references/aws-agentcore/diagnostics.md @@ -0,0 +1,72 @@ +# AgentCore — Diagnostics and troubleshooting + +Temporal has no AgentCore troubleshooting page. This guide uses only the gotchas documented in the sources above and the shared [Worker Controller Instance (WCI)](../wci.md) inspection flow. If a symptom is not covered here, gather the evidence below and report it to the user. Do not attach an undocumented explanation to it. + +## Invocation flow (when working correctly) + +1. Creating the Worker Deployment Version starts its WCI, which invokes the named endpoint and waits for the Worker to register. +2. The Runtime's `/invocations` handler starts the Worker in the background and acknowledges. +3. The Worker polls the Task Queue as `/`, and the version binds the Task Queue. +4. When Tasks arrive and no Worker is polling, the WCI invokes the endpoint again. +5. The Worker drains when its idle policy fires, or stops when AgentCore ends the compute. + +## Start here: did the expected Worker register? + +```bash +temporal worker deployment describe-version \ + --namespace --deployment-name --build-id \ + --report-task-queue-stats -o json +``` + +The decisive signal is that the expected Task Queue appears in `taskQueuesInfos` with the types the Worker polls. That only happens after a Worker that identifies as this deployment name and Build ID has polled. A successful Validate Connection or a Running WCI supports the diagnosis but does not replace this check. Wait for this state to appear rather than for a fixed time. Re-run the command while you wait. If the Task Queue is still not bound after several minutes, stop waiting and use the table below. A bound Task Queue does not show that the version is current or that Tasks complete. + +| Result | Next check | +|---|---| +| Task Queue bound | Registration succeeded. Confirm the version is current, then go to [Tasks not completing](#worker-runs-but-tasks-do-not-complete). | +| Not bound, and the WCI history shows invocation failures | [The Runtime is not invoked](#the-runtime-is-not-invoked) | +| Not bound, invocation succeeded | [Invoked, but no Worker registers](#invoked-but-no-worker-registers) | + +## The Runtime is not invoked + +1. **Validate Connection (Cloud).** In the version's **Actions → Validate Connection**, a failure means Temporal cannot assume the invocation role, get the named endpoint, or invoke the Runtime. Check the following: + - `--aws-agentcore-endpoint-arn` is the **endpoint** ARN (`…/runtime-endpoint/`), not the Runtime ARN. + - The role's `AgentRuntimeARNs` is the Runtime ARN **with a trailing `*`**. Without the wildcard, `GetAgentRuntimeEndpoint` on the endpoint is not covered. + - The External ID is identical in the stack and in the version. + - Self-hosted: the server's AWS identity is the stack's `TemporalIamRoleArn` and has `sts:AssumeRole` on the role. The server can also reach the AgentCore control-plane and data-plane APIs. → `self-hosted.md`. +2. **Stack never finished.** If the role stack failed, check the role-name length. On Cloud, `-` must be 64 characters or fewer. → `iam.md`. +3. **Endpoint not ready.** Check the endpoint's state with `agentcore status --type runtime-endpoint --json`. `CREATE_FAILED` and `UPDATE_FAILED` mean a failure caused by permissions, the artifact, or another configuration problem. The sources document no cause for an endpoint that stays `CREATING` or `UPDATING`. If it is still in one of those states after several minutes, report the state and the `agentcore deploy` output to the user. Do not redeploy to get it to move. +4. **Version not current.** If registration succeeded but new Workflows never reach the Worker, set the version current (`setup.md` Step 6). +5. **Read the WCI history** for the failing Activity result. Use the inspection commands and interpretation rules in [`../wci.md`](../wci.md#inspect-the-wci). + +## Invoked, but no Worker registers + +Read the Runtime logs first: + +```bash +aws logs tail /aws/bedrock-agentcore/runtimes/- --since 1h --region +``` + +- **Name mismatch.** The Runtime's `TEMPORAL_DEPLOYMENT_NAME` / `TEMPORAL_BUILD_ID` differ from the version's. Temporal starts compute that registers as some other version. Fix the environment, which creates a new Runtime version. Then follow `versioning.md`. +- **Missing versioning settings.** The sample Worker refuses to start without `TEMPORAL_DEPLOYMENT_NAME` and `TEMPORAL_BUILD_ID`. +- **Cannot reach Temporal.** With a VPC network mode, the VPC needs outbound access to the Temporal Service. For self-hosted, see `self-hosted.md` step 1. +- **Endpoint runs other code.** The version points at `DEFAULT`, or at a named endpoint that was moved to another Runtime version. → `versioning.md`. +- **Deploy failed with `uv install failed ... with exit code null`.** `uv` is missing on the operator's machine. + +## Worker runs but Tasks do not complete + +- **Non-determinism errors after a deploy.** Code changed underneath a live Worker Deployment Version, because the version uses `DEFAULT` or a repointed endpoint. Pinned Workflows are affected too. Restore the old endpoint-to-version mapping, as described in `versioning.md`. +- **Old code still running after a deploy.** Existing sessions keep the code that was deployed when their microVM was created, until they terminate. +- **Activities interrupted.** AgentCore ended the compute (`maxLifetime`) before the Activity completed, or the idle policy drained too early. Configure Activity timeouts and Heartbeats so retries recover the work. → `constraints.md`. + +## Sessions stay alive after work is done + +In the sample, `/ping` reports `HealthyBusy` while an async task is registered, and AgentCore keeps the session alive while it does. If the handler never calls `complete_async_task` when the Worker exits, the session stays `HealthyBusy` until `maxLifetime`. The idle session timeout does not help either, because it resets only on AgentCore Runtime invocations, not on Temporal polling. The fix is a Worker idle policy that drains the Worker and releases the async task in all cases, including errors. The language-specific code is in the selected `sdk-.md`. + +## Lifecycle configuration rejected + +`ValidationException` on a Runtime create or update means one of these: a lifecycle value is below 60 seconds or above 28800 seconds (microVMs), or `idleRuntimeSessionTimeout` is greater than `maxLifetime`. + +## Rules + +- Inspect the specific Runtime, endpoint, stack and version named in the inventory. Do not enumerate AgentCore resources across Regions to reconstruct state. +- Do not change IAM, redeploy, or repoint an endpoint until the evidence above identifies the cause. diff --git a/references/aws-agentcore/iam.md b/references/aws-agentcore/iam.md new file mode 100644 index 0000000..de4435b --- /dev/null +++ b/references/aws-agentcore/iam.md @@ -0,0 +1,90 @@ +# AgentCore — IAM and permissions + +This file covers three separate identities. Never merge them: + +| Identity | Who uses it | Created by | +|---|---|---| +| **Operator** | The person or CI running `agentcore`, `aws` and CloudFormation commands | Already exists | +| **Runtime execution role** | AgentCore, inside the Runtime session, when Worker code calls AWS (Bedrock, Code Interpreter, Secrets Manager) | `agentcore deploy` | +| **Invocation role** | Temporal, to get the named endpoint and invoke the Runtime. It never runs Worker code. | The CloudFormation templates below | + +## Operator permissions + +The operator needs permission to create AgentCore resources, CloudFormation stacks and IAM roles. See [IAM permissions for AgentCore Runtime](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-permissions.html). The CDK must also be bootstrapped in the target account and Region. The CloudFormation deploys below create named IAM roles, so they need `--capabilities CAPABILITY_NAMED_IAM`. + +## Runtime execution role + +`agentcore deploy` creates the execution role. To grant more permissions to Worker code, list policy files under the Runtime's `additionalPolicies` in `agentcore.json`. The sample uses this to add Code Interpreter access. The Temporal API key lives in AWS Secrets Manager, and **this** role, never the invocation role, gets `secretsmanager:GetSecretValue` on that one secret ARN, through a policy file in `additionalPolicies`. The policy file and the hand-off are in `setup.md`, [Store the Temporal API key](setup.md#store-the-temporal-api-key). + +## Temporal API key lifecycle + +This applies to Temporal Cloud with an API key. A self-hosted Service without API-key auth has no key to manage. The agent never sees the key at any stage: it never prints it, reads it back (including with `aws secretsmanager get-secret-value`), or passes it in a command. + +- **Source.** The user creates the API key in Temporal Cloud and stores it in the Secrets Manager secret from their own terminal (setup.md hand-off). Record who created it. The key never goes in `agentcore.json`, a Runtime env var, or the project directory. +- **Read.** The Runtime gets only the secret's ARN, in `TEMPORAL_API_KEY_SECRET_ARN`. Each time the entry point starts a Worker, it calls `GetSecretValue` as the execution role, then connects. The key stays in process memory for the life of that Worker. → the selected `sdk-.md`. +- **Rotate.** The user creates a new key in Temporal Cloud and runs the same hand-off script again. `put-secret-value` adds a new version, moves `AWSCURRENT` to it and moves `AWSPREVIOUS` to the old one. No redeploy is needed: every Worker started after that reads the new key. Workers already running keep the old key until they drain, which happens within the Runtime's configured `maxLifetime` at the latest (`constraints.md`). Verify with `describe-secret --query VersionIdsToStages` that the new version is `AWSCURRENT`. Keep the old Temporal key valid until the AgentCore logs show a Worker started after the rotation that has registered, and until `maxLifetime` has passed since the rotation. +- **Revoke.** The user deletes or disables the key in Temporal Cloud. Revoke an old key only after rotation, as described above. Revoking the only key stops every Worker that reads this secret from authenticating at its next start, so ask before suggesting it: a Temporal Cloud API key is account-scoped, not deployment-scoped. If the key leaked, revoke it first and rotate immediately afterwards. Deleting the secret is a teardown step (`setup.md`, [Teardown](setup.md#teardown)), not a revocation. + +## Invocation role + +The invocation role grants exactly `bedrock-agentcore:InvokeAgentRuntime` and `bedrock-agentcore:GetAgentRuntimeEndpoint` on the Runtime ARNs you pass in. Its trust policy requires an External ID, which prevents [confused deputy](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html) attacks. + +**The External ID.** The user chooses it, and it must be 5–45 characters from `[a-zA-Z0-9_+=,.@-]`. Use the same value in the stack and in the Worker Deployment Version. A fixed value is fine for a tutorial. Use a unique value in production. + +**`AgentRuntimeARNs` takes the Runtime ARN with a trailing `*`, not the endpoint ARN.** The wildcard makes the policy cover the Runtime and all of its endpoints, which `GetAgentRuntimeEndpoint` requires. As a result, new endpoints for later builds need no IAM change. + +```text +Runtime ARN (IAM, plus *): arn:aws:bedrock-agentcore:::runtime/ +Endpoint ARN (Temporal only): arn:aws:bedrock-agentcore:::runtime//runtime-endpoint/ +``` + +### Temporal Cloud + +Template: [`assets/temporal-cloud-serverless-worker-agentcore-role.yaml`](../../assets/temporal-cloud-serverless-worker-agentcore-role.yaml). It is vendored unchanged from Temporal's S3 copy. The trust policy lists Temporal Cloud's principals and requires `sts:ExternalId`. + +```bash +aws cloudformation create-stack \ + --stack-name \ + --template-body file://assets/temporal-cloud-serverless-worker-agentcore-role.yaml \ + --parameters \ + ParameterKey=AssumeRoleExternalId,ParameterValue= \ + ParameterKey=AgentRuntimeARNs,ParameterValue='*' \ + ParameterKey=RoleName,ParameterValue= \ + --capabilities CAPABILITY_NAMED_IAM \ + --region + +aws cloudformation wait stack-create-complete --stack-name --region + +aws cloudformation describe-stacks --stack-name \ + --query 'Stacks[0].Outputs[?OutputKey==`RoleARN`].OutputValue' \ + --output text --region +``` + +**To add a Runtime to an existing stack,** update it with the complete list, keeping the other parameters. `AgentRuntimeARNs` is a `CommaDelimitedList`, so escape the comma inside the CLI's shorthand `ParameterValue`: + +```bash +aws cloudformation update-stack --stack-name --use-previous-template \ + --parameters ParameterKey=AssumeRoleExternalId,UsePreviousValue=true \ + ParameterKey=RoleName,UsePreviousValue=true \ + 'ParameterKey=AgentRuntimeARNs,ParameterValue=*\,*' \ + --capabilities CAPABILITY_NAMED_IAM --region +``` + +Wait on `aws cloudformation wait stack-update-complete`, then confirm the `AgentRuntimeARNs` parameter lists both ARNs. + +**Check the role name length before you create the stack.** The template names the role `-`, and an IAM role name has at most 64 characters, counting the hyphen. If the name is longer, CloudFormation cannot create the role. With the default `RoleName` (`Temporal-Cloud-Serverless-Worker`, 32 characters), keep `` to 31 characters or fewer. + +### Self-hosted Temporal Service + +Template: [`assets/temporal-self-hosted-serverless-worker-agentcore-role.yaml`](../../assets/temporal-self-hosted-serverless-worker-agentcore-role.yaml). It is vendored from the Temporal docs. Instead of Temporal Cloud's principals, it trusts the single AWS identity in `TemporalIamRoleArn`, which is the identity the Temporal Service runs as. Unlike the Cloud template, it uses `RoleName` as given (default `Temporal-AgentCore-Worker`). Set a different `RoleName` when you deploy more than one copy of the stack. + +The deploy command, parameters and the server-side `sts:AssumeRole` grant are in `self-hosted.md`, [Step 4](self-hosted.md#4-create-the-invocation-role). + +## Validate Connection + +On Temporal Cloud, **Actions → Validate Connection** on the Worker Deployment Version checks that Temporal can assume the invocation role, get the named endpoint, and invoke the Runtime. A successful check does not show that the Worker registered and polled. Confirm that separately, as described in `diagnostics.md`. + +## Rules + +- Never paste or echo AWS secret keys or the Temporal API key in commands the agent runs, and never run `aws secretsmanager get-secret-value`. +- When an existing invocation stack is in use, add the new Runtime ARN to its `AgentRuntimeARNs` (comma-separated) instead of creating a parallel role. Ask the user before you change a stack this run did not create, and record the stack as *reused* in the `setup.md` inventory. Teardown then removes only this Runtime's ARN from it (`setup.md`, [Teardown](setup.md#teardown)). diff --git a/references/aws-agentcore/observability.md b/references/aws-agentcore/observability.md new file mode 100644 index 0000000..8d6f4ef --- /dev/null +++ b/references/aws-agentcore/observability.md @@ -0,0 +1,34 @@ +# AgentCore — Observability + +## Two complementary views + +| Question | Where to look | +|---|---| +| What did the Workflow or agent do, and which Activities ran, failed or retried? | Temporal Event History (`temporal workflow show --workflow-id `) and the Temporal UI | +| Which Worker process did the work, when did it start, and when did it drain? | AgentCore Runtime logs (`aws logs tail /aws/bedrock-agentcore/runtimes/- --since `) | +| Runtime, model and tool behavior | [AgentCore Observability](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-configure.html) | + +Use AgentCore telemetry for the Runtime, model and tool behavior. Use Event History for durable execution progress. Neither replaces the other. AgentCore Observability is a separate AgentCore service. Configuring AgentCore as a compute provider does not set it up. Set it up only when the user asks for Runtime telemetry export. + +## Logs + +The Runtime logs show the Worker starting and processing Tasks. AgentCore writes each endpoint's logs to its own log group, `/aws/bedrock-agentcore/runtimes/-`. Temporal invokes the named endpoint, so read that group: + +```bash +aws logs tail /aws/bedrock-agentcore/runtimes/- --since 1h --region +``` + +`agentcore logs --runtime ` (CLI 0.28.1) returned no Worker logs for a Runtime that Temporal invoked through a named endpoint, so use the log group directly. Log at least these points in the entry point, because they are what `diagnostics.md` correlates: Worker start (with deployment name, Build ID and Task Queue), duplicate-invocation skips, idle drain, and worker failure. The Python sample logs each of these, and the handlers in both SDK guides log them with the same messages; the exact lines are in the selected `sdk-.md`, under "Logging and diagnostic signatures". + +When the Worker has drained, a session that is no longer running emits no new Worker logs. An empty recent window after a drain is expected. It does not mean the deployment failed. + +## Provider-specific signals + +Standard Temporal SDK metrics describe the Worker. The signals below describe the AgentCore side. + +- **Version registration:** `taskQueuesInfos` from `temporal worker deployment describe-version` (see `diagnostics.md`). +- **Connection:** Validate Connection on the version (Cloud only). +- **Endpoint state and version:** `agentcore status --type runtime-endpoint --json`. +- **WCI activity:** WCI history, as described in [`../wci.md`](../wci.md#inspect-the-wci). + +Temporal SDK metrics and tracing are configured as for any long-running Worker. The language-specific setup, if any, is in the selected `sdk-.md`. diff --git a/references/aws-agentcore/self-hosted.md b/references/aws-agentcore/self-hosted.md new file mode 100644 index 0000000..afb0f06 --- /dev/null +++ b/references/aws-agentcore/self-hosted.md @@ -0,0 +1,88 @@ +# AgentCore — Self-hosted Temporal Service setup + +AgentCore Serverless Workers require **Temporal Service v1.32.0 or later**. A self-hosted Service needs no Pre-release access request. Complete this page, then follow `setup.md`. + +Confirm the server version first. If the Service is older than v1.32.0, stop and tell the user that it must be upgraded. Do not attempt a partial setup. + +## 1. Configure network access in both directions + +- **Runtime → Temporal.** The Temporal Service frontend must be reachable from the AgentCore Runtime. If the frontend is public, use `"networkMode": "PUBLIC"` on the Runtime. If it is reachable only on a private network, configure the Runtime for [VPC access](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/agentcore-vpc.html) and connect that VPC to the network that hosts the Service. +- **Temporal → AgentCore APIs.** The Service must reach the AgentCore control-plane and data-plane APIs. If it runs in an AWS VPC without internet access, configure [AgentCore interface VPC endpoints](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/vpc-interface-endpoints.html) for both. + +Ask the user which case applies. Do not infer it from the address format. + +## 2. Enable the Worker Controller Instance + +The WCI is disabled by default. Enable it, the `aws-agentcore` compute provider and the `no-sync` scaling algorithm in the Service's [dynamic configuration](https://docs.temporal.io/references/dynamic-configuration): + +```yaml +workercontroller.enabled: + - value: true + +workercontroller.compute_providers.enabled: + - value: + - aws-agentcore + +workercontroller.scaling_algorithms.enabled: + - value: + - no-sync +``` + +Both lists are allowlists. **If either already holds values for other providers or algorithms, such as `aws-lambda` or `rate-based`, append to the list. Do not replace it.** + +To enable the WCI for specific Namespaces only: + +```yaml +workercontroller.enabled: + - value: true + constraints: + namespace: 'your-namespace' +``` + +The Service watches the file and applies changes **without a restart**. + +Keep `workercontroller.compute_providers.aws.require_role_and_external_id` at its default (enabled). With that default, every Worker Deployment Version must carry an invocation role and External ID. + +## 3. Give the Temporal Service AWS credentials + +The Service assumes the invocation role, so it needs an AWS identity with `sts:AssumeRole` on that role. + +- **On AWS (EC2, ECS, EKS):** the server uses the attached instance, task or pod role automatically. Grant that role `sts:AssumeRole` on the invocation role from step 4. +- **Outside AWS:** use [IAM Roles Anywhere](https://aws.amazon.com/iam/roles-anywhere/). Static `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_REGION` in the server environment also work, but they are not recommended. Never ask the user to paste access keys into the session. + +To identify the ARN of the identity the server runs as, run `aws sts get-caller-identity` **in the server environment**. That ARN becomes `TemporalIamRoleArn` in step 4. + +## 4. Create the invocation role + +Deploy [`assets/temporal-self-hosted-serverless-worker-agentcore-role.yaml`](../../assets/temporal-self-hosted-serverless-worker-agentcore-role.yaml). Its trust policy allows only `TemporalIamRoleArn`, and only with the External ID. Pass each Runtime ARN with a trailing `*` so the policy covers the Runtime and its endpoints. The Runtime must exist first, so run this after `setup.md` Step 3 has produced the Runtime ARN. + +```bash +aws cloudformation create-stack \ + --stack-name \ + --template-body file://assets/temporal-self-hosted-serverless-worker-agentcore-role.yaml \ + --parameters \ + ParameterKey=TemporalIamRoleArn,ParameterValue= \ + ParameterKey=AssumeRoleExternalId,ParameterValue= \ + ParameterKey=AgentRuntimeARNs,ParameterValue='*' \ + --capabilities CAPABILITY_NAMED_IAM \ + --region + +aws cloudformation wait stack-create-complete --stack-name --region + +aws cloudformation describe-stacks --stack-name \ + --query 'Stacks[0].Outputs[?OutputKey==`RoleARN`].OutputValue' \ + --output text --region +``` + +| Parameter | Value | +|---|---| +| `TemporalIamRoleArn` | ARN of the IAM role or user the Temporal Service runs as (step 3). | +| `AssumeRoleExternalId` | A string of 5–45 characters that the user chooses. Use the same value in the Worker Deployment Version. | +| `AgentRuntimeARNs` | Comma-separated Runtime ARNs, each with a trailing `*`. | +| `RoleName` | Optional. Defaults to `Temporal-AgentCore-Worker`, and the full name must be 64 characters or fewer. Set a different name for each extra copy of the stack. | + +## 5. Continue with `setup.md`, with these differences + +- **Runtime environment:** set `TEMPORAL_ADDRESS` and `TEMPORAL_NAMESPACE` to the self-hosted frontend and Namespace. Configure the self-hosted Service's authentication settings in the Worker entry point. The Temporal Cloud API-key variables do not apply unless the Service uses API keys. +- **IAM:** skip the Temporal Cloud IAM step. Use the role ARN and External ID from step 4. +- **Worker Deployment Version:** create it with the Temporal CLI. The Cloud UI and **Validate Connection** are Temporal Cloud only. Confirm registration with the checks in `diagnostics.md` instead. diff --git a/references/aws-agentcore/setup.md b/references/aws-agentcore/setup.md new file mode 100644 index 0000000..9541d9f --- /dev/null +++ b/references/aws-agentcore/setup.md @@ -0,0 +1,345 @@ +# AgentCore — Setup (happy path) + +This page covers the full path. You configure the Runtime in `agentcore/agentcore.json`, add the Worker entry point, deploy with the AgentCore CLI (the **CodeZip** build for Python, the **Container** build for TypeScript), grant Temporal an invocation role, register a Worker Deployment Version against a named endpoint, set it current, and verify. Related references: `iam.md` for the three identities, `constraints.md` for lifecycle limits, `versioning.md` for new builds and rollback, `diagnostics.md` when something fails, and `self-hosted.md` for a self-hosted Service, which you complete first. + +This skill deploys and operates the Worker. To design the agent itself (Workflow and Activity boundaries, Strands, Bedrock model calls), link the user to [Build a durable agent on Amazon Bedrock AgentCore](https://docs.temporal.io/guides/durable-agent-on-agentcore). Do not reproduce that guide. + +## Prerequisites + +- **Temporal:** either a Temporal Cloud account with an **AWS-hosted Namespace that has AgentCore Pre-release access**, or a self-hosted Temporal Service v1.32.0+ with `self-hosted.md` completed. +- [Temporal CLI v1.8.3](https://github.com/temporalio/cli/releases/tag/v1.8.3) or later, configured for the Namespace. +- An AWS account in an [AgentCore-supported Region](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/agentcore-regions.html). The AWS CLI must be configured for that account, and the operator needs the permissions listed in `iam.md`. +- Node.js 20+, the [AgentCore CLI](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/agentcore-get-started-cli.html) (`npm install -g @aws/agentcore`), and the AWS CDK, bootstrapped in the target account and Region. +- For Python: `uv` on the operator's machine. The CDK build uses it to install dependencies for the Runtime's platform, and without it the build fails with `uv install failed ... with exit code null`. +- **No local Docker.** The CLI also handles AgentCore's ARM64 (Graviton) requirement. Do not build images locally or create ECR repositories by hand. + - **Python: CodeZip.** The CLI uploads a zip of the code directory and AgentCore runs it on a managed runtime. + - **TypeScript: Container.** The CLI's Node CodeZip packager bundles only `main.ts` into one file and ships no `node_modules`, so it cannot package the Temporal TypeScript Worker (see `sdk-typescript.md`). With `"build": "Container"`, the CLI uploads the code directory, builds the image in an AWS CodeBuild project, and pushes it to an ECR repository that it creates. No local Docker daemon is needed. + +### Select the Namespace + +Select the Namespace and, for Temporal Cloud, confirm its AgentCore Pre-release access in SKILL.md Step 1. + +Use the gRPC endpoint that Temporal Cloud shows for the Namespace as `TEMPORAL_ADDRESS`. Do not construct it from the Namespace name. + +## Step 1: Configure the Runtime + +`agentcore/agentcore.json` is the AgentCore CLI project file. Its `runtimes` array defines the Runtimes that the CLI deploys, and `agentcore/aws-targets.json` names the target account and Region. + +**No project yet:** `agentcore create` generates one, including the `agentcore/cdk/` scaffold that `agentcore deploy` synthesizes. The sample runs it once to obtain only the CDK scaffold, and keeps its own checked-in `agentcore.json`: + +```bash +agentcore create --project-name --no-agent \ + --output-dir --skip-git --skip-python-setup +mv //agentcore/cdk agentcore/cdk +``` + +**Apply the naming prefix to the project `name` in `agentcore.json`, not only to the Runtime.** `agentcore deploy` names its CloudFormation stack `AgentCore--`, and AgentCore prefixes the Runtime ID with the project name (`_-`). Two projects with the same name in one account and Region deploy into the same stack. + +**Fill in `aws-targets.json`.** `agentcore create` writes it as an empty array (`[]`), and the CLI has no command that adds a target. Add one object with the target `name` that `agentcore deploy --target` uses, the `account` ID and the `region`. + +```json +[ + { "name": "default", "account": "", "region": "" } +] +``` + +The Runtime object needs a `build` (`"CodeZip"` for Python, `"Container"` for TypeScript, from the selected `sdk-.md`), an `entrypoint`, a `codeLocation`, and a **named endpoint** pinned to a version. Never rely on `DEFAULT` (see `versioning.md`): + +```json +{ + "name": "", + "build": "CodeZip", + "entrypoint": "", + "codeLocation": ".", + "runtimeVersion": "PYTHON_3_12", + "networkMode": "PUBLIC", + "protocol": "HTTP", + "authorizerType": "AWS_IAM", + "endpoints": { + "temporal": { "version": 1, "description": "Invoked by Temporal Serverless Workers" } + } +} +``` + +`runtimeVersion` above is the Python value from the sample. Take the value for other languages from the selected `sdk-.md`. Add `lifecycleConfiguration` (`idleRuntimeSessionTimeout`, `maxLifetime`) only after you read `constraints.md`. The sample sets the AgentCore defaults explicitly. + +Add the Temporal settings to the Runtime's `envVars`: + +| Variable | Value | +|---|---| +| `TEMPORAL_ADDRESS` | Namespace gRPC endpoint (Cloud) or the frontend address (self-hosted) | +| `TEMPORAL_NAMESPACE` | Namespace | +| `TEMPORAL_API_KEY_SECRET_ARN` | ARN of the Secrets Manager secret that holds the Temporal Cloud API key (Temporal Cloud with an API key only). This is the ARN, never the key. | +| `TEMPORAL_TASK_QUEUE` | Task Queue the application uses | +| `TEMPORAL_DEPLOYMENT_NAME` | Worker Deployment name. It must match Step 5. | +| `TEMPORAL_BUILD_ID` | Build ID. It must match Step 5. | + +**The API key never goes in `agentcore.json`, an env var, or any command the agent runs.** It lives only in AWS Secrets Manager. The Runtime **execution** role reads it, and the entry point loads it before it connects. Set up the secret, the grant and the user's hand-off as described in [Store the Temporal API key](#store-the-temporal-api-key) before Step 3. A self-hosted Service without API-key auth needs no secret, so leave `TEMPORAL_API_KEY_SECRET_ARN` unset. + +### Store the Temporal API key + +Skip this for a self-hosted Service that does not use API keys. The full key lifecycle (source, rotation, revocation) is in [`iam.md`](iam.md#temporal-api-key-lifecycle). + +1. **Create the secret, with no value.** First check whether it exists: `aws secretsmanager describe-secret --secret-id --region --query ARN --output text`. If it returns `ResourceNotFoundException`, create it and record it in the inventory as *created*. If it exists, reuse it only when the user confirms it is theirs to use for this Worker, and record it as *reused*. A secret can exist without a version; the hand-off adds the first one. + + ```bash + aws secretsmanager create-secret --name \ + --description "Temporal API key for " \ + --region --query ARN --output text + ``` + + Put the printed ARN in `TEMPORAL_API_KEY_SECRET_ARN`. + +2. **Grant the execution role read access to this one secret.** Add a policy file in the Runtime's `codeLocation` directory, as the sample does for Code Interpreter, and list it in the Runtime's `additionalPolicies`. The CLI resolves `additionalPolicies` paths relative to `codeLocation`: in the sample's layout that is the project root, and in the TypeScript layout it is `app//`. A file anywhere else fails CDK synthesis with `ENOENT`. `agentcore deploy` attaches it to the execution role. Never grant it to the invocation role, and never use `"Resource": "*"`. + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ReadTemporalApiKey", + "Effect": "Allow", + "Action": "secretsmanager:GetSecretValue", + "Resource": "" + } + ] + } + ``` + + ```json + "additionalPolicies": ["temporal-api-key-secret-policy.json"] + ``` + +3. **Hand off the key.** The user runs this once, **in their own interactive terminal**, after the secret exists. It reads the key once and writes it as the secret's first version. Do not run it through the agent's shell: `read -s` needs a terminal, and the key would pass through the agent's session. Steps 2–4 need no key and can continue meanwhile. **Step 5 must wait:** creating the version makes Temporal start the Worker, which fails to connect until the secret has an `AWSCURRENT` version. + + ```bash + ( + if [ ! -t 0 ]; then + printf 'Run this in an interactive terminal; nothing was changed\n' >&2; exit 1 + fi + printf 'Temporal API key: ' >&2 + IFS= read -r -s key + printf '\nlen=%s\n' "${#key}" >&2 + dots=${key//[^.]/} + if [ "${#dots}" -ne 2 ]; then + printf 'Expected a JWT-shaped key with two dots; nothing was changed\n' >&2; exit 1 + fi + printf %s "$key" | aws secretsmanager put-secret-value \ + --secret-id --secret-string file:///dev/stdin \ + --region --query VersionId --output text + ) + ``` + + The parentheses run the script in a subshell, so the key does not stay in the user's shell. `printf %s` pipes the key without a trailing newline; pasting it into a file or a here-doc would store one and corrupt the credential. The key never appears on a command line, so it is not visible in the process list. + + Write the script to a file outside the code directory, with every placeholder filled in, such as `/-handoff.sh`. It contains no key. Then end your turn with the user's action first: one line saying the run is waiting on them; the file's absolute path and the command `bash ` to run in their own terminal; and what to reply when it finishes. Put status and notes after that. While the run is blocked on the hand-off, repeat these steps in full rather than pointing back to them. + +4. **Verify without reading the key back.** When the user replies, check the secret's versions rather than trusting the reply: + + ```bash + aws secretsmanager describe-secret --secret-id \ + --region --query VersionIdsToStages + ``` + + Expect exactly one version labelled `AWSCURRENT`. Never run `aws secretsmanager get-secret-value`, and never print, log or pass the key in a command. The Worker registering in Step 5 is the proof that the key authenticates. + +If the Runtime uses a VPC instead of `PUBLIC`, configure outbound access from the VPC to the Temporal Service. + +## Step 2: Add the Worker entry point + +For CodeZip, AgentCore packages the `codeLocation` directory, and `entrypoint` is relative to it. That directory must contain the entry point, every local module it imports, and the language's dependency manifest. + +The entry point must: + +- connect a Client and build a standard long-running Worker with Worker Versioning, using the deployment name and Build ID from the environment; +- serve the AgentCore Runtime HTTP contract (`/invocations`, `/ping`); +- start the Worker as background work on `/invocations` and acknowledge immediately; and +- stop polling and drain when its idle policy decides to release the Runtime. + +The language-specific code is in the selected `sdk-.md`. + +## Step 3: Deploy the Runtime + +First check that no Runtime with this name is already deployed: `agentcore status --runtime --json`. If one is, it belongs to an earlier run or another user. Ask before you redeploy it, and record it as *reused*. + +From the project directory, run the following, using the `name` of the target in `aws-targets.json`: + +```bash +agentcore validate +agentcore deploy --target -y +``` + +The CLI packages the code, deploys the Runtime, creates its execution role, and creates the named endpoint. A Container build also creates the ECR repository and its KMS key, the CodeBuild project and its role, and a build-trigger Lambda function and its role, all in the same stack. `agentcore deploy` can also turn on CloudWatch Transaction Search for the account (it prints `Transaction search enabled`), which is an account-level setting. Name both in the Step 2 approval list. Re-running the same commands deploys changes, which creates a new Runtime version. Read `versioning.md` before you redeploy a Runtime that serves a live Worker Deployment Version. + +Record both ARNs: + +```bash +agentcore status --runtime --json +agentcore status --type runtime-endpoint --json +``` + +- **Runtime ARN** ends in `/runtime/`. It is used only for IAM, with a trailing `*`. +- **Endpoint ARN** ends in `/runtime//runtime-endpoint/`. It is used only for `--aws-agentcore-endpoint-arn`. **Never pass the Runtime ARN there.** + +Also record the execution role that `agentcore deploy` created, for the inventory and teardown: `aws bedrock-agentcore-control get-agent-runtime --agent-runtime-id --region --query roleArn --output text`. + +Do not continue until both ARNs are printed and the endpoint state is `READY`. `agentcore status` reports only whether the CLI deployed the endpoint (`deploymentState: deployed`), not its AgentCore state, so read the state from the AWS API: + +```bash +aws bedrock-agentcore-control get-agent-runtime-endpoint \ + --agent-runtime-id --endpoint-name \ + --region --query '{status: status, liveVersion: liveVersion}' +``` + +If it reports `CREATE_FAILED` or `UPDATE_FAILED`, or still reports `CREATING` or `UPDATING` after several minutes, stop and go to [`diagnostics.md`](diagnostics.md#the-runtime-is-not-invoked), "Endpoint not ready". `READY` shows only that the endpoint exists. It does not show that the Worker code starts or that a Worker registers. Step 5 checks those. + +## Step 4: Grant Temporal permission to invoke the Runtime + +For Temporal Cloud, deploy the Cloud invocation-role template with the Runtime ARN plus `*` and an External ID the user chooses, then read the stack's `RoleARN` output. Before you create a stack, check whether the user already has one (`aws cloudformation describe-stacks --stack-name `). If they do, add this Runtime to it as `iam.md` describes, and record it as *reused*. → `iam.md`. For a self-hosted Service, use the role from `self-hosted.md` step 4 and skip this step. + +## Step 5: Create the Worker Deployment Version + +The deployment name and Build ID must match `TEMPORAL_DEPLOYMENT_NAME` and `TEMPORAL_BUILD_ID`. If they differ, Temporal can start compute that never registers as the version waiting for work. + +**Temporal Cloud UI:** go to **Workers → Create Worker Deployment** and set Name, Build ID, Compute Provider **Amazon Bedrock AgentCore Runtime**, Runtime endpoint ARN, IAM role ARN and External ID. A version created in the UI is current automatically. + +**Temporal CLI** (required for self-hosted): + +```bash +temporal worker deployment create \ + --namespace \ + --name + +temporal worker deployment create-version \ + --namespace \ + --deployment-name \ + --build-id \ + --aws-agentcore-endpoint-arn \ + --aws-agentcore-assume-role-arn \ + --aws-agentcore-assume-role-external-id +``` + +Creating the version makes Temporal invoke the Runtime and wait for the Worker to register. On Cloud, use **Actions → Validate Connection** to check the role, the endpoint lookup and the invocation. A passing Validate Connection shows that Temporal can assume the role and invoke the endpoint. It does not show that a Worker registered. Then confirm registration with the `describe-version` check in [`diagnostics.md`](diagnostics.md#start-here-did-the-expected-worker-register). Re-run it while you wait, and wait on that state, not on a fixed delay. If the Task Queue is still not bound after several minutes, stop waiting and follow that section's result table. + +## Step 6: Set the version current + +```bash +temporal worker deployment set-current-version \ + --namespace \ + --deployment-name \ + --build-id \ + --yes +``` + +The command asks for confirmation because it changes which version receives new Tasks. `--yes` skips the prompt. Skip this step if the version was created in the Cloud UI. + +## Step 7: Verify + +Submit work to the Task Queue. When no Worker is polling, Temporal invokes the endpoint, the Runtime starts the Worker, and the Worker processes Tasks. The Workflow's Event History shows what ran, and the AgentCore logs show which Worker process ran it and when it drained. Run the warm check. The scale-from-zero check is optional. + +**Warm check (required).** Run it right after Step 6, while the Worker that registered in Step 5 may still be polling. Start a Workflow on the Task Queue and confirm all three signals: + +- **Temporal UI:** the Worker Deployment Version shows a Worker polling the Task Queue. On Cloud, the connection also shows as valid. +- **AgentCore logs:** the endpoint's log group (`aws logs tail /aws/bedrock-agentcore/runtimes/-`) shows the Worker starting and processing Tasks. +- **Temporal CLI:** `temporal worker deployment describe --name ` shows the deployment and its current version. + +Read the Workflow's history with `temporal workflow describe --workflow-id `. If it shows no progress after several minutes, stop and go to [`diagnostics.md`](diagnostics.md#start-here-did-the-expected-worker-register). The warm check passing shows that the Worker processes Tasks. It does not show that Temporal can start a new Worker from zero, because the Worker that registered may have served the Workflow. + +**Scale-from-zero check (optional, slow).** This check shows that the WCI invokes the Runtime again after the Worker has drained. Offer it to the user rather than running it by default, because it waits for the idle policy to fire. Wait for the drain line (`worker idle for s; drained`) in the endpoint's log group (`aws logs tail /aws/bedrock-agentcore/runtimes/-`). The handler's configured idle period sets how long this takes (see the selected `sdk-.md`). Then start another Workflow and confirm that the logs show a new `polling as /` line and that the Workflow completes. If no drain line appears within several minutes of the configured idle period, stop and go to [`diagnostics.md`](diagnostics.md#sessions-stay-alive-after-work-is-done). If the drain line appears but the new Workflow shows no progress after several minutes, stop and go to [`diagnostics.md`](diagnostics.md#start-here-did-the-expected-worker-register). + +## Resource inventory + +Record each resource as you go, and mark each one **created** (this run made it) or **reused** (it existed before this run). Each step above checks for an existing resource before it creates or grants one. Teardown removes only what is marked *created*. + +| Resource | Record | +|---|---| +| AgentCore project | Directory, target name, account, Region | +| CDK stack | `AgentCore--`, which `agentcore deploy` creates and which owns the Runtime, endpoint and execution role | +| Container build (TypeScript) | ECR repository and its KMS key, CodeBuild project and role, build-trigger Lambda function and role, all in the CDK stack | +| Log groups | `/aws/bedrock-agentcore/runtimes/-`, one per endpoint including `DEFAULT` | +| CloudWatch Transaction Search | Whether `agentcore deploy` turned it on in this run. It is account-wide, so never turn it off without asking. | +| Runtime | Name, Runtime ID, Runtime ARN | +| Named endpoint | Name, endpoint ARN, Runtime version it points to | +| Runtime execution role | Role ARN (created by `agentcore deploy` with the Runtime) | +| Invocation role | Cloud: CloudFormation stack name and `RoleARN`, and if reused, the Runtime ARN this run added to `AgentRuntimeARNs`. Self-hosted: the role from `self-hosted.md`. External ID. | +| Secret | Secret ARN, and the policy file in `additionalPolicies` | +| Temporal | Worker Deployment name, Build ID, Task Queue, and whether this run created the Worker Deployment | +| Temporal API key | Who created it. The user owns it; it is never in the inventory. | + +Hand the inventory to the user at the end. + +## Teardown + +Remove resources in this order: Temporal first, so that nothing invokes the Runtime, then the Runtime and endpoint, then shared resources. Remove a shared resource (invocation role, stack, secret) only if this run created it **and** nothing else uses it. Ask before changing or deleting anything marked *reused*. Never unset or delete a version while Pinned Workflows still need it (see `versioning.md`). + +1. **Unset the current version.** A current version cannot be deleted. + + ```bash + temporal worker deployment set-current-version \ + --namespace --deployment-name \ + --unversioned --yes + ``` + +2. **Delete the version once it has no pollers and is not draining, then the deployment.** Running Workers stop on their idle policy (see the selected `sdk-.md`). `delete-version` refuses a draining version, so read the drainage status and retry rather than adding `--skip-drainage`. If `delete-version` still refuses after several minutes, stop retrying and check what holds the version: + - If the AgentCore logs show no drain line for the last Worker, go to [`diagnostics.md`](diagnostics.md#sessions-stay-alive-after-work-is-done). + - If the status is still draining, Pinned Workflows can still need the version (`versioning.md`). Report the status to the user, and ask whether to wait for those Workflows or to skip drainage. Pass `--skip-drainage` only if the user approves. + + Delete the deployment only if this run created it and it has no other versions. + + ```bash + temporal worker deployment describe-version \ + --namespace --deployment-name \ + --build-id -o json | jq -r '.drainageInfo.drainageStatus // empty' + temporal worker deployment delete-version \ + --namespace --deployment-name --build-id + temporal worker deployment delete --namespace --name + ``` + +3. **Remove the Runtime and its endpoint.** The AgentCore CLI removes resources from `agentcore.json`, and the next deploy deletes them from AWS. To remove one endpoint of an earlier build and keep the Runtime, use `agentcore remove runtime-endpoint --name -y` instead. Never remove an endpoint that a live version still points to. + + ```bash + agentcore remove agent --name -y + agentcore deploy --target -y + ``` + + Verify by reading state back. `agentcore status --runtime --json` must no longer list the Runtime as deployed. `aws bedrock-agentcore-control get-agent-runtime --agent-runtime-id --region ` must return `ResourceNotFoundException`. Then check the execution role with `aws iam get-role --role-name `. If it still exists, report it to the user instead of deleting it, because the CLI's stack owns it. + + The Runtime's log groups outlive it: deleting the Runtime and its stack leaves one `/aws/bedrock-agentcore/runtimes/-` group per endpoint, including `DEFAULT`. A Container build also leaves its CodeBuild project's log group (`/aws/codebuild/AgentCore---container-builder`) and its build-trigger Lambda function's group (`/aws/lambda/AgentCore-…`, named after a truncated stack name, so read the function name from the stack before deleting it). If this run created the Runtime, list and delete them; use the same `describe-log-groups` prefix query for the two Container-build groups: + + ```bash + aws logs describe-log-groups --region \ + --log-group-name-prefix /aws/bedrock-agentcore/runtimes/ \ + --query 'logGroups[].logGroupName' --output text + aws logs delete-log-group --log-group-name --region + ``` + +4. **Invocation role (Temporal Cloud).** Delete the stack only if this run created it. First read its Runtime list: + + ```bash + aws cloudformation describe-stacks --stack-name --region \ + --query 'Stacks[0].Parameters[?ParameterKey==`AgentRuntimeARNs`].ParameterValue' --output text + ``` + + - It lists only this Runtime's ARN with `*`: delete the stack and wait on its state. `describe-stacks` then reports that the stack does not exist. + + ```bash + aws cloudformation delete-stack --stack-name --region + aws cloudformation wait stack-delete-complete --stack-name --region + ``` + + - It lists other Runtime ARNs, or the stack was *reused*: keep it. With the user's approval, update it to drop only this Runtime's ARN, keeping every other parameter value. + + For a self-hosted Service, apply the same rule to the stack from `self-hosted.md` Step 4. Leave the server-side `sts:AssumeRole` grant to the user, because other Runtimes may rely on it. + +5. **Secret.** If this run created the secret and no other Runtime lists it in `additionalPolicies`, schedule its deletion. Keep the recovery window so that a mistake can be undone, and do not add `--force-delete-without-recovery` unless the user asks. Keep a *reused* secret. Delete the policy file from the project. + + ```bash + aws secretsmanager delete-secret --secret-id \ + --recovery-window-in-days 7 --region + ``` + + Verify with `aws secretsmanager describe-secret --secret-id --query DeletedDate`, which must print a date. + +6. **Temporal API key.** Ask before you suggest revoking it, because a Temporal Cloud API key is account-scoped, not deployment-scoped. The user revokes it, as described in [`iam.md`](iam.md#temporal-api-key-lifecycle). + +Leave the local AgentCore project directory to the user. diff --git a/references/aws-agentcore/versioning.md b/references/aws-agentcore/versioning.md new file mode 100644 index 0000000..8391ac7 --- /dev/null +++ b/references/aws-agentcore/versioning.md @@ -0,0 +1,38 @@ +# AgentCore — Versioning, updates and rollback + +## How the two version models map + +Serverless Workers require Worker Versioning. **Map each Worker Deployment Version to a named AgentCore Runtime endpoint that points to exactly one Runtime version.** + +| AgentCore primitive | Behavior | Role in Temporal | +|---|---|---| +| Runtime version | AgentCore creates version 1 when you create a Runtime. Every update to the Runtime creates a new immutable version, including code, protocol and network changes. | The immutable build behind one Worker Deployment Version. | +| Named endpoint | Has a stable ARN and points to a chosen version. It moves only when you update it explicitly. | The `--aws-agentcore-endpoint-arn` of one Worker Deployment Version. | +| `DEFAULT` endpoint | Moves to the latest version automatically on every Runtime update. | **Never** use it for a live Worker Deployment Version. | + +**The hazard:** if a Worker Deployment Version points to an endpoint whose Runtime version changes, the code behind that Temporal version changes. Replay-unsafe changes can then cause non-determinism errors for in-flight Workflows, including Pinned ones. This happens with `DEFAULT` on every update, and with a named endpoint if you repoint it. + +The Build ID in the Runtime environment (`TEMPORAL_BUILD_ID`) and the Worker Deployment Version's Build ID must match. The same applies to the deployment name. Changing an environment variable is a Runtime update, so it also creates a new Runtime version. + +## Rolling out a new build + +1. Choose a new Build ID and set `TEMPORAL_BUILD_ID` to it in the Runtime's `envVars` in `agentcore/agentcore.json`. +2. Add a **new** named endpoint for the new Runtime version under the Runtime's `endpoints`, and keep the old endpoint unchanged. In the sample, `"version": 1` selects the first Runtime version. Point the new endpoint at the version that the deploy creates. +3. Run `agentcore validate` and `agentcore deploy --target -y`. Then read the Runtime version and endpoint ARNs from `agentcore status --type runtime-endpoint --json`. Before continuing, confirm that the new endpoint is `READY` and points to the intended version. Endpoint states are `CREATING`, `CREATE_FAILED`, `READY`, `UPDATING` and `UPDATE_FAILED`. If the endpoint reaches a `_FAILED` state, or is not `READY` after several minutes, stop and go to [`diagnostics.md`](diagnostics.md#the-runtime-is-not-invoked), "Endpoint not ready". Leave the live version and its endpoint unchanged while you diagnose. +4. Create a new Worker Deployment Version with the new Build ID and the **new endpoint ARN**. Reuse the invocation role and External ID. A role created with the Runtime ARN plus a trailing wildcard already covers every endpoint of that Runtime. → `iam.md`. +5. Set the new version current, or ramp to it. +6. **Keep the old endpoint** while Pinned Workflows can still need the older Worker code. + +If the user's AgentCore CLI version or project layout does not let them declare the version an endpoint points to, ask them how they want to manage endpoints. Do not fall back to `DEFAULT` or repoint the live endpoint. + +## Sessions already running keep their code + +A microVM session uses the code assets that were deployed when the microVM was created. After a Runtime update, existing sessions keep the previous code until they terminate. A Worker that is already polling therefore keeps running the old build until its idle policy drains it or `maxLifetime` ends its compute. This is another reason to give every build its own Worker Deployment Version and endpoint. + +## Lifecycle settings are part of the Runtime configuration + +`idleRuntimeSessionTimeout` and `maxLifetime` are set on the Runtime (`lifecycleConfiguration` in `agentcore.json`, or `LifecycleConfiguration` on `CreateAgentRuntime` and `UpdateAgentRuntime`). Changing them is a Runtime update, so it creates a new Runtime version. Roll the change out like a new build: new endpoint and new Worker Deployment Version. For the meaning of each setting, see `constraints.md`. + +## Rollback + +Set the previous Worker Deployment Version current again. Its endpoint still points to the old immutable Runtime version, so the next invocation starts the old code without a redeploy. This works only if the old endpoint was kept. If it was deleted or repointed, recreate an endpoint for the known-good Runtime version and register it on a new Worker Deployment Version.