From dc103419d25e27922595b25690f08684ebb1f308 Mon Sep 17 00:00:00 2001 From: Gajendra Malviya Date: Fri, 9 Oct 2026 13:42:33 +0530 Subject: [PATCH 1/2] fix(aws-ecs): close the real-cloud parity gaps in #1449 --- docs/coverage/README.md | 2 +- docs/coverage/aws/README.md | 2 +- docs/coverage/aws/ecs.md | 58 +- docs/coverage/coverage.json | 276 +++++--- docs/coverage/nongoals/ecs.md | 9 + docs/services.md | 67 +- internal/settle/settle.go | 2 + providers/aws/aws.go | 1 + providers/aws/ecs/clone.go | 52 +- providers/aws/ecs/clusters.go | 17 +- providers/aws/ecs/container_instances.go | 43 +- providers/aws/ecs/ecs.go | 49 +- providers/aws/ecs/ecs_test.go | 7 +- providers/aws/ecs/engine_test.go | 15 +- providers/aws/ecs/errors.go | 5 + providers/aws/ecs/events.go | 361 ++++++++-- providers/aws/ecs/events_test.go | 403 +++++++++++ providers/aws/ecs/exec.go | 86 ++- providers/aws/ecs/exec_enablement_test.go | 208 ++++++ providers/aws/ecs/metrics.go | 174 +++++ providers/aws/ecs/metrics_test.go | 301 +++++++++ providers/aws/ecs/namespace.go | 60 ++ providers/aws/ecs/nitin_review_test.go | 10 +- providers/aws/ecs/parity_helpers_test.go | 85 +++ providers/aws/ecs/service_deployments.go | 499 ++++++++++++++ providers/aws/ecs/service_deployments_test.go | 533 +++++++++++++++ providers/aws/ecs/services.go | 294 +++++++-- providers/aws/ecs/snapshot.go | 9 + providers/aws/ecs/snapshot_test.go | 71 ++ providers/aws/ecs/tags.go | 6 + providers/aws/ecs/task_protection.go | 231 +++++++ providers/aws/ecs/task_protection_test.go | 278 ++++++++ providers/aws/ecs/task_sets.go | 623 ++++++++++++++++++ providers/aws/ecs/task_sets_test.go | 472 +++++++++++++ providers/aws/ecs/tasks.go | 152 ++++- providers/aws/ecs/validation_defaults_test.go | 383 +++++++++++ providers/aws/ecs/wave4b_test.go | 4 +- providers/aws/lifecycle_events_test.go | 77 ++- server/aws/ecs/dispatch_test.go | 68 ++ server/aws/ecs/errors_test.go | 45 ++ server/aws/ecs/handler.go | 1 + server/aws/ecs/metrics_sdk_test.go | 105 +++ server/aws/ecs/parity_validation_sdk_test.go | 303 +++++++++ server/aws/ecs/sdk_roundtrip_test.go | 1 + server/aws/ecs/service_deployments.go | 302 +++++++++ .../aws/ecs/service_deployments_sdk_test.go | 141 ++++ server/aws/ecs/services.go | 5 + server/aws/ecs/task_protection.go | 104 +++ server/aws/ecs/task_protection_sdk_test.go | 52 ++ server/aws/ecs/task_sets.go | 238 +++++++ server/aws/ecs/task_sets_sdk_test.go | 140 ++++ server/aws/ecs/tasks.go | 2 + server/aws/ecs/types.go | 77 ++- services/ecs/capabilities.go | 196 ++++++ services/ecs/driver/capabilities.go | 225 +++++++ services/ecs/driver/driver.go | 50 +- services/ecs/ecs_test.go | 120 +++- 57 files changed, 7813 insertions(+), 287 deletions(-) create mode 100644 docs/coverage/nongoals/ecs.md create mode 100644 providers/aws/ecs/events_test.go create mode 100644 providers/aws/ecs/exec_enablement_test.go create mode 100644 providers/aws/ecs/metrics.go create mode 100644 providers/aws/ecs/metrics_test.go create mode 100644 providers/aws/ecs/namespace.go create mode 100644 providers/aws/ecs/parity_helpers_test.go create mode 100644 providers/aws/ecs/service_deployments.go create mode 100644 providers/aws/ecs/service_deployments_test.go create mode 100644 providers/aws/ecs/task_protection.go create mode 100644 providers/aws/ecs/task_protection_test.go create mode 100644 providers/aws/ecs/task_sets.go create mode 100644 providers/aws/ecs/task_sets_test.go create mode 100644 providers/aws/ecs/validation_defaults_test.go create mode 100644 server/aws/ecs/dispatch_test.go create mode 100644 server/aws/ecs/errors_test.go create mode 100644 server/aws/ecs/metrics_sdk_test.go create mode 100644 server/aws/ecs/parity_validation_sdk_test.go create mode 100644 server/aws/ecs/service_deployments.go create mode 100644 server/aws/ecs/service_deployments_sdk_test.go create mode 100644 server/aws/ecs/task_protection.go create mode 100644 server/aws/ecs/task_protection_sdk_test.go create mode 100644 server/aws/ecs/task_sets.go create mode 100644 server/aws/ecs/task_sets_sdk_test.go create mode 100644 services/ecs/capabilities.go create mode 100644 services/ecs/driver/capabilities.go diff --git a/docs/coverage/README.md b/docs/coverage/README.md index 7e23f62a3..57d2a78db 100644 --- a/docs/coverage/README.md +++ b/docs/coverage/README.md @@ -80,7 +80,7 @@ code does not implement. Machine-readable: [`coverage.json`](./coverage.json). | `digitaltwins` | - | [DigitalTwins](./azure/digitaltwins.md) | - | - | 8 | | `disks` | - | [Disks](./azure/disks.md) | - | - | 7 | | `dns` | [Route53](./aws/route53.md) | [DNS](./azure/dns.md) | [CloudDNS](./gcp/clouddns.md) | - | 16 | -| `ecs` | [ECS](./aws/ecs.md) | - | - | - | 41 | +| `ecs` | [ECS](./aws/ecs.md) | - | - | - | 5 | | `efs` | [EFS](./aws/efs.md) | - | - | - | 27 | | `eks` | [EKS](./aws/eks.md) | - | - | - | 40 | | `elasticsan` | - | [ElasticSan](./azure/elasticsan.md) | - | - | 8 | diff --git a/docs/coverage/aws/README.md b/docs/coverage/aws/README.md index 8f21f315f..37e976060 100644 --- a/docs/coverage/aws/README.md +++ b/docs/coverage/aws/README.md @@ -31,7 +31,7 @@ Services cloudemu emulates for AWS, by native name. Back to the [cross-provider | [DynamoDB](./dynamodb.md) | `database` | 24 | | [EC2](./ec2.md) | `compute` | 37 | | [ECR](./ecr.md) | `containerregistry` | 15 | -| [ECS](./ecs.md) | `ecs` | 41 | +| [ECS](./ecs.md) | `ecs` | 5 | | [EFS](./efs.md) | `efs` | 27 | | [EKS](./eks.md) | (provider-native) | 40 | | [ELB](./elb.md) | `loadbalancer` | 19 | diff --git a/docs/coverage/aws/ecs.md b/docs/coverage/aws/ecs.md index d6713245a..600bd5ee1 100644 --- a/docs/coverage/aws/ecs.md +++ b/docs/coverage/aws/ecs.md @@ -1,9 +1,25 @@ # ECS -AWS's `ecs` service · portable interface `driver.ECS` · [AWS index](./README.md) +AWS's `ecs` service · portable interface `driver.TaskSets` · [AWS index](./README.md) -## Operations (41) +## Operations (5) + +| Operation | Description | +| --- | --- | +| `CreateTaskSet` | | +| `DeleteTaskSet` | | +| `DescribeTaskSets` | | +| `UpdateServicePrimaryTaskSet` | | +| `UpdateTaskSet` | | + +## Optional capabilities + +Discovered by type assertion; only some providers implement these. + +### ECS + +ECS is the interface an ECS provider implementation must satisfy. | Operation | Description | | --- | --- | @@ -49,6 +65,42 @@ AWS's `ecs` service · portable interface `driver.ECS` · [AWS index](./README.m | `UpdateContainerInstancesState` | | | `UpdateService` | | +### ServiceDeployments + +ServiceDeployments exposes the deployment history of ECS-controller services. + +| Operation | Description | +| --- | --- | +| `DescribeServiceDeployments` | | +| `DescribeServiceRevisions` | | +| `ListServiceDeployments` | | +| `StopServiceDeployment` | | + +### ServiceNamespaces + +ServiceNamespaces lists the services that joined a Service Connect namespace. + +| Operation | Description | +| --- | --- | +| `ListServicesByNamespace` | | + +### TaskProtection + +TaskProtection reads and changes the scale-in protection of service tasks. + +| Operation | Description | +| --- | --- | +| `GetTaskProtection` | | +| `UpdateTaskProtection` | | + ## Not in scope -_Not documented yet. See the [emulator boundary](../../../README.md) for cloudemu-wide non-goals._ +- No workload runs unless a container engine is attached, so the usage-driven metrics (`CPUUtilization`, `MemoryUtilization`, network and storage series) are never published. Only metrics derived from state the emulator holds are: `AWS/ECS` `CPUReservation`, `MemoryReservation` and `LiveTaskCount`, and the `ECS/ContainerInsights` task, service, instance, deployment and task-set counts (only for clusters with `containerInsights` enabled or enhanced, or enabled on the account). +- Task lifecycle events cover PROVISIONING (awsvpc tasks), PENDING, ACTIVATING, RUNNING, DEACTIVATING, STOPPING, DEPROVISIONING (awsvpc tasks) and STOPPED. The transient `DELETED` state is not published. A task counts as awsvpc when it has an ENI attachment, which today only Fargate tasks do. +- Container-instance state-change events omit `versionInfo` (agent and Docker versions), since the emulator runs no agent. +- `ECS Deployment State Change` events are published for the in-progress and completed states. The deployment circuit breaker is stored and echoed but never trips, so `SERVICE_DEPLOYMENT_FAILED` and automatic rollback do not happen. `SERVICE_TASK_PLACEMENT_FAILURE` is published when a service create or update leaves tasks unplaced, not when a replacement task cannot be placed. +- Service deployments and revisions exist for services that use the ECS deployment controller; they complete immediately, so `StopServiceDeployment` only succeeds while the deployment is still in progress, which is the `--async-settle` window. A stop of a completed deployment is a `ConflictException`. The lifecycle stage is not reported, and blue/green, linear and canary strategies, lifecycle hooks, alarms and `ContinueServiceDeployment` are not modelled. The newest 100 deployments of a service are kept. +- Task sets run their tasks directly and replace a stopped task to restore `computedDesiredCount`. They are not registered with load balancer target groups or service discovery, and a capacity provider strategy on a task set is stored and echoed but not used for placement (the same as for services). +- `ListServicesByNamespace` matches a service by the namespace string (name or ARN) it was configured with. There is no Cloud Map in the emulator to resolve one to the other, so an unknown namespace lists nothing instead of returning `NamespaceNotFoundException`. +- Where AWS does not document an exact message or exception, the emulator picks the closest documented exception and its own wording: a task-set call on a service that does not use the EXTERNAL controller, deleting a task set that is not scaled to zero without `force`, an invalid `launchType`, `ExecuteCommand` on a task that is not RUNNING, and `StopTask` on an already-stopped task (it keeps the original reason). +- `CreateCluster` on an existing ACTIVE name returns that cluster unchanged and ignores the tags and settings in the new request. diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index b1c005a94..e6ab5f6ee 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -6587,130 +6587,192 @@ }, { "service": "ecs", - "interface": "ECS", + "interface": "TaskSets", "operations": [ { - "name": "CreateCapacityProvider" + "name": "CreateTaskSet" }, { - "name": "CreateCluster" - }, - { - "name": "CreateService" - }, - { - "name": "DeleteAccountSetting" - }, - { - "name": "DeleteAttributes" - }, - { - "name": "DeleteCapacityProvider" - }, - { - "name": "DeleteCluster" - }, - { - "name": "DeleteService" - }, - { - "name": "DeregisterContainerInstance" - }, - { - "name": "DeregisterTaskDefinition" - }, - { - "name": "DescribeCapacityProviders" - }, - { - "name": "DescribeClusters" - }, - { - "name": "DescribeContainerInstances" - }, - { - "name": "DescribeServices" - }, - { - "name": "DescribeTaskDefinition" - }, - { - "name": "DescribeTasks" - }, - { - "name": "ExecuteCommand" - }, - { - "name": "ListAccountSettings" - }, - { - "name": "ListAttributes" - }, - { - "name": "ListClusters" - }, - { - "name": "ListContainerInstances" - }, - { - "name": "ListServices" - }, - { - "name": "ListTagsForResource" - }, - { - "name": "ListTaskDefinitionFamilies" - }, - { - "name": "ListTaskDefinitions" - }, - { - "name": "ListTasks" - }, - { - "name": "PutAccountSetting" - }, - { - "name": "PutAccountSettingDefault" - }, - { - "name": "PutAttributes" + "name": "DeleteTaskSet" }, { - "name": "PutClusterCapacityProviders" + "name": "DescribeTaskSets" }, { - "name": "RegisterContainerInstance" - }, - { - "name": "RegisterTaskDefinition" - }, - { - "name": "RunTask" - }, - { - "name": "StopTask" - }, - { - "name": "TagResource" - }, - { - "name": "UntagResource" + "name": "UpdateServicePrimaryTaskSet" }, { - "name": "UpdateCapacityProvider" - }, + "name": "UpdateTaskSet" + } + ], + "optionalCapabilities": [ { - "name": "UpdateCluster" + "name": "ECS", + "doc": "ECS is the interface an ECS provider implementation must satisfy.", + "operations": [ + { + "name": "CreateCapacityProvider" + }, + { + "name": "CreateCluster" + }, + { + "name": "CreateService" + }, + { + "name": "DeleteAccountSetting" + }, + { + "name": "DeleteAttributes" + }, + { + "name": "DeleteCapacityProvider" + }, + { + "name": "DeleteCluster" + }, + { + "name": "DeleteService" + }, + { + "name": "DeregisterContainerInstance" + }, + { + "name": "DeregisterTaskDefinition" + }, + { + "name": "DescribeCapacityProviders" + }, + { + "name": "DescribeClusters" + }, + { + "name": "DescribeContainerInstances" + }, + { + "name": "DescribeServices" + }, + { + "name": "DescribeTaskDefinition" + }, + { + "name": "DescribeTasks" + }, + { + "name": "ExecuteCommand" + }, + { + "name": "ListAccountSettings" + }, + { + "name": "ListAttributes" + }, + { + "name": "ListClusters" + }, + { + "name": "ListContainerInstances" + }, + { + "name": "ListServices" + }, + { + "name": "ListTagsForResource" + }, + { + "name": "ListTaskDefinitionFamilies" + }, + { + "name": "ListTaskDefinitions" + }, + { + "name": "ListTasks" + }, + { + "name": "PutAccountSetting" + }, + { + "name": "PutAccountSettingDefault" + }, + { + "name": "PutAttributes" + }, + { + "name": "PutClusterCapacityProviders" + }, + { + "name": "RegisterContainerInstance" + }, + { + "name": "RegisterTaskDefinition" + }, + { + "name": "RunTask" + }, + { + "name": "StopTask" + }, + { + "name": "TagResource" + }, + { + "name": "UntagResource" + }, + { + "name": "UpdateCapacityProvider" + }, + { + "name": "UpdateCluster" + }, + { + "name": "UpdateClusterSettings" + }, + { + "name": "UpdateContainerInstancesState" + }, + { + "name": "UpdateService" + } + ] }, { - "name": "UpdateClusterSettings" + "name": "ServiceDeployments", + "doc": "ServiceDeployments exposes the deployment history of ECS-controller services.", + "operations": [ + { + "name": "DescribeServiceDeployments" + }, + { + "name": "DescribeServiceRevisions" + }, + { + "name": "ListServiceDeployments" + }, + { + "name": "StopServiceDeployment" + } + ] }, { - "name": "UpdateContainerInstancesState" + "name": "ServiceNamespaces", + "doc": "ServiceNamespaces lists the services that joined a Service Connect namespace.", + "operations": [ + { + "name": "ListServicesByNamespace" + } + ] }, { - "name": "UpdateService" + "name": "TaskProtection", + "doc": "TaskProtection reads and changes the scale-in protection of service tasks.", + "operations": [ + { + "name": "GetTaskProtection" + }, + { + "name": "UpdateTaskProtection" + } + ] } ], "providers": { diff --git a/docs/coverage/nongoals/ecs.md b/docs/coverage/nongoals/ecs.md new file mode 100644 index 000000000..4acf38ed6 --- /dev/null +++ b/docs/coverage/nongoals/ecs.md @@ -0,0 +1,9 @@ +- No workload runs unless a container engine is attached, so the usage-driven metrics (`CPUUtilization`, `MemoryUtilization`, network and storage series) are never published. Only metrics derived from state the emulator holds are: `AWS/ECS` `CPUReservation`, `MemoryReservation` and `LiveTaskCount`, and the `ECS/ContainerInsights` task, service, instance, deployment and task-set counts (only for clusters with `containerInsights` enabled or enhanced, or enabled on the account). +- Task lifecycle events cover PROVISIONING (awsvpc tasks), PENDING, ACTIVATING, RUNNING, DEACTIVATING, STOPPING, DEPROVISIONING (awsvpc tasks) and STOPPED. The transient `DELETED` state is not published. A task counts as awsvpc when it has an ENI attachment, which today only Fargate tasks do. +- Container-instance state-change events omit `versionInfo` (agent and Docker versions), since the emulator runs no agent. +- `ECS Deployment State Change` events are published for the in-progress and completed states. The deployment circuit breaker is stored and echoed but never trips, so `SERVICE_DEPLOYMENT_FAILED` and automatic rollback do not happen. `SERVICE_TASK_PLACEMENT_FAILURE` is published when a service create or update leaves tasks unplaced, not when a replacement task cannot be placed. +- Service deployments and revisions exist for services that use the ECS deployment controller; they complete immediately, so `StopServiceDeployment` only succeeds while the deployment is still in progress, which is the `--async-settle` window. A stop of a completed deployment is a `ConflictException`. The lifecycle stage is not reported, and blue/green, linear and canary strategies, lifecycle hooks, alarms and `ContinueServiceDeployment` are not modelled. The newest 100 deployments of a service are kept. +- Task sets run their tasks directly and replace a stopped task to restore `computedDesiredCount`. They are not registered with load balancer target groups or service discovery, and a capacity provider strategy on a task set is stored and echoed but not used for placement (the same as for services). +- `ListServicesByNamespace` matches a service by the namespace string (name or ARN) it was configured with. There is no Cloud Map in the emulator to resolve one to the other, so an unknown namespace lists nothing instead of returning `NamespaceNotFoundException`. +- Where AWS does not document an exact message or exception, the emulator picks the closest documented exception and its own wording: a task-set call on a service that does not use the EXTERNAL controller, deleting a task set that is not scaled to zero without `force`, an invalid `launchType`, `ExecuteCommand` on a task that is not RUNNING, and `StopTask` on an already-stopped task (it keeps the original reason). +- `CreateCluster` on an existing ACTIVE name returns that cluster unchanged and ignores the tags and settings in the new request. diff --git a/docs/services.md b/docs/services.md index de9a8f545..dea78c36a 100644 --- a/docs/services.md +++ b/docs/services.md @@ -2370,8 +2370,69 @@ real EC2 instance subject to managed-resource visibility. | Container instances | RegisterContainerInstance, DeregisterContainerInstance, UpdateContainerInstancesState (DRAINING), ListContainerInstances, DescribeContainerInstances | | Tagging | TagResource, UntagResource, ListTagsForResource | | Account & attributes | PutAccountSetting(+Default), ListAccountSettings, DeleteAccountSetting, PutAttributes, DeleteAttributes, ListAttributes | +| Capacity providers | CreateCapacityProvider, DescribeCapacityProviders, UpdateCapacityProvider, DeleteCapacityProvider | -**Total: 37 operations.** +**Behaviour added for real-cloud parity.** `ExecuteCommand` needs a task started with +`enableExecuteCommand` that is RUNNING (the task reports the `ExecuteCommandAgent`); +`CreateCluster` on an ACTIVE name returns that cluster; `assignPublicIp` defaults to +`DISABLED`; an invalid `launchType` is an `InvalidParameterException`; a repeated +`StopTask` keeps the first stop reason; awsvpc containers report `networkInterfaces`. +Tasks publish one EventBridge event per lifecycle state (`PROVISIONING` for awsvpc tasks, +`PENDING`, `ACTIVATING`, `RUNNING`, `DEACTIVATING`, `STOPPING`, `DEPROVISIONING` for +awsvpc tasks, `STOPPED`) with `detail.version` +1 each, plus ECS Deployment State Change, +Container Instance State Change and more Service Action events. With a CloudWatch backend +wired, ECS publishes `ECS/ContainerInsights` counts (clusters with `containerInsights` +enabled or enhanced) and `AWS/ECS` `CPUReservation`, `MemoryReservation` and `LiveTaskCount`; +no usage-driven series. What is deliberately not emulated is listed in +[coverage/nongoals/ecs.md](coverage/nongoals/ecs.md). + +### Task Sets (optional capability: `TaskSets`) + +Task sets of services that use the `EXTERNAL` deployment controller. Each set runs its own +tasks (`computedDesiredCount` = the service desired count x scale, rounded up), starts +`ACTIVE`, and `UpdateServicePrimaryTaskSet` makes one `PRIMARY` (the previous primary becomes +`ACTIVE`). A set that is not scaled to zero needs `force` to be deleted. `DescribeServices` +reports a service's task sets. + +| Operation | Signature | +|-----------|-----------| +| `CreateTaskSet` | `(ctx, CreateTaskSetInput) (*TaskSet, error)` | +| `UpdateTaskSet` | `(ctx, UpdateTaskSetInput) (*TaskSet, error)` | +| `DeleteTaskSet` | `(ctx, DeleteTaskSetInput) (*TaskSet, error)` | +| `DescribeTaskSets` | `(ctx, cluster, service, ids) ([]TaskSet, []Failure, error)` | +| `UpdateServicePrimaryTaskSet` | `(ctx, cluster, service, primary) (*TaskSet, error)` | + +### Task Protection (optional capability: `TaskProtection`) + +Scale-in protection (1-2880 minutes, default 120) for service tasks; a scale-in or redeployment +leaves protected tasks running. + +| Operation | Signature | +|-----------|-----------| +| `GetTaskProtection` | `(ctx, cluster, tasks) ([]ProtectedTask, []Failure, error)` | +| `UpdateTaskProtection` | `(ctx, UpdateTaskProtectionInput) ([]ProtectedTask, []Failure, error)` | + +### Service Deployments (optional capability: `ServiceDeployments`) + +The deployment history of ECS-controller services: every create, force-new-deployment or +deploying update records a service deployment and a service revision (the newest 100 are +kept). `StopServiceDeployment` works while a deployment is still in progress (the +`--async-settle` window); a completed deployment is a `ConflictException`. + +| Operation | Signature | +|-----------|-----------| +| `ListServiceDeployments` | `(ctx, ListServiceDeploymentsInput) ([]ServiceDeployment, nextToken, error)` | +| `DescribeServiceDeployments` | `(ctx, arns) ([]ServiceDeployment, []Failure, error)` | +| `DescribeServiceRevisions` | `(ctx, arns) ([]ServiceRevision, []Failure, error)` | +| `StopServiceDeployment` | `(ctx, arn, stopType) (arn, error)` | + +### Service Connect Namespaces (optional capability: `ServiceNamespaces`) + +| Operation | Signature | +|-----------|-----------| +| `ListServicesByNamespace` | `(ctx, namespace, maxResults, nextToken) ([]arn, nextToken, error)` | + +**Total: 41 operations (+12 optional)** --- @@ -3250,7 +3311,7 @@ delete) is still enforced. | Machine Learning: Azure AI (CognitiveServices + MachineLearningServices + data plane) | 92 | | Machine Learning: GCP Vertex AI (Go API/driver) | 128 | | AI Search: Azure AI Search (control + data plane) | 53 | -| Container Orchestration: AWS ECS | 37 | +| Container Orchestration: AWS ECS | 41 (+12 optional) | | DNS Resolver: AWS Route 53 Resolver | 72 | | Application Networking: AWS VPC Lattice | 73 | | Key Management: AWS KMS | 45 | @@ -3266,7 +3327,7 @@ delete) is still enforced. | Data Integration: AWS Glue | 299 | | Threat Detection: Amazon GuardDuty | 87 | | Streaming: Amazon MSK | 59 | -| **Grand Total** | **2749** (+138 optional) | +| **Grand Total** | **2753** (+150 optional) | Optional operations are capabilities a driver may implement but is not required to; see the sections marked "optional capability". They are counted separately diff --git a/internal/settle/settle.go b/internal/settle/settle.go index 808db8365..879df1e75 100644 --- a/internal/settle/settle.go +++ b/internal/settle/settle.go @@ -51,6 +51,8 @@ const ( DefaultECSTaskStartSettle = 2 * time.Second // ECS task PROVISIONING/PENDING->RUNNING DefaultECSTaskStopSettle = 1 * time.Second // ECS task STOPPING/DEPROVISIONING->STOPPED + DefaultECSDeploymentSettle = 3 * time.Second // ECS service deployment IN_PROGRESS->SUCCESSFUL + DefaultTargetHealthSettle = 2 * time.Second // ELBv2 target initial->healthy DefaultTargetDrainSettle = 2 * time.Second // ELBv2 target draining->removed diff --git a/providers/aws/aws.go b/providers/aws/aws.go index abb2fc311..f370e067b 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -417,6 +417,7 @@ func newProvider(o *config.Options, shared *GlobalServices) *Provider { p.CloudWatchLogs.SetMonitoring(p.CloudWatch) p.SNS.SetMonitoring(p.CloudWatch) p.ECR.SetMonitoring(p.CloudWatch) + p.ECS.SetMonitoring(p.CloudWatch) p.EventBridge.SetMonitoring(p.CloudWatch) p.RDS.SetMonitoring(p.CloudWatch) p.Kinesis.SetMonitoring(p.CloudWatch) diff --git a/providers/aws/ecs/clone.go b/providers/aws/ecs/clone.go index fe42c8d2e..70a7257e5 100644 --- a/providers/aws/ecs/clone.go +++ b/providers/aws/ecs/clone.go @@ -200,7 +200,7 @@ func cloneTask(t *driver.Task) driver.Task { } // cloneContainers deep-copies a slice of containers and each container's -// NetworkBindings slice. +// NetworkBindings, NetworkInterfaces and ManagedAgents slices. func cloneContainers(in []driver.Container) []driver.Container { if len(in) == 0 { return nil @@ -211,6 +211,8 @@ func cloneContainers(in []driver.Container) []driver.Container { for i := range in { c := in[i] c.NetworkBindings = append([]driver.NetworkBinding(nil), in[i].NetworkBindings...) + c.NetworkInterfaces = append([]driver.ContainerNetworkInterface(nil), in[i].NetworkInterfaces...) + c.ManagedAgents = append([]driver.ManagedAgent(nil), in[i].ManagedAgents...) out[i] = c } @@ -250,6 +252,7 @@ func cloneService(s *driver.Service) driver.Service { out.DeploymentConfiguration = cloneDeploymentConfig(s.DeploymentConfiguration) out.NetworkConfiguration = cloneNetworkConfig(s.NetworkConfiguration) out.HealthCheckGracePeriodSeconds = cloneIntPtr(s.HealthCheckGracePeriodSeconds) + out.ServiceConnect = cloneServiceConnect(s.ServiceConnect) return out } @@ -331,3 +334,50 @@ func cloneManagedScaling(in *driver.ManagedScaling) *driver.ManagedScaling { return &out } + +// cloneTaskSet deep-copies a task set's slice and pointer fields. +func cloneTaskSet(ts *driver.TaskSet) driver.TaskSet { + out := *ts + out.CapacityProviderStrategy = append([]driver.CapacityProviderStrategyItem(nil), ts.CapacityProviderStrategy...) + out.NetworkConfiguration = cloneNetworkConfig(ts.NetworkConfiguration) + out.LoadBalancers = append([]driver.LoadBalancer(nil), ts.LoadBalancers...) + out.ServiceRegistries = append([]driver.ServiceRegistry(nil), ts.ServiceRegistries...) + out.Tags = copyTags(ts.Tags) + + return out +} + +// cloneServiceConnect deep-copies a Service Connect configuration. +func cloneServiceConnect(in *driver.ServiceConnectConfiguration) *driver.ServiceConnectConfiguration { + if in == nil { + return nil + } + + return &driver.ServiceConnectConfiguration{Namespace: in.Namespace, Raw: append([]byte(nil), in.Raw...)} +} + +// cloneServiceDeployment deep-copies a service deployment. +func cloneServiceDeployment(d *driver.ServiceDeployment) driver.ServiceDeployment { + out := *d + out.SourceServiceRevisions = append([]driver.ServiceRevisionSummary(nil), d.SourceServiceRevisions...) + out.DeploymentConfiguration = cloneDeploymentConfig(d.DeploymentConfiguration) + + if d.Rollback != nil { + rollback := *d.Rollback + out.Rollback = &rollback + } + + return out +} + +// cloneServiceRevision deep-copies a service revision. +func cloneServiceRevision(r *driver.ServiceRevision) driver.ServiceRevision { + out := *r + out.CapacityProviderStrategy = append([]driver.CapacityProviderStrategyItem(nil), r.CapacityProviderStrategy...) + out.NetworkConfiguration = cloneNetworkConfig(r.NetworkConfiguration) + out.LoadBalancers = append([]driver.LoadBalancer(nil), r.LoadBalancers...) + out.ServiceRegistries = append([]driver.ServiceRegistry(nil), r.ServiceRegistries...) + out.ServiceConnect = cloneServiceConnect(r.ServiceConnect) + + return out +} diff --git a/providers/aws/ecs/clusters.go b/providers/aws/ecs/clusters.go index 83111a06e..0356b57e0 100644 --- a/providers/aws/ecs/clusters.go +++ b/providers/aws/ecs/clusters.go @@ -8,7 +8,8 @@ import ( "github.com/stackshy/cloudemu/v2/services/ecs/driver" ) -// CreateCluster creates a cluster, defaulting the name to "default". +// CreateCluster creates a cluster, defaulting the name to "default". Creating a +// cluster whose name is already ACTIVE returns the existing cluster, as AWS does. func (m *Mock) CreateCluster(ctx context.Context, in driver.CreateClusterInput) (*driver.Cluster, error) { name := in.Name if name == "" { @@ -23,21 +24,24 @@ func (m *Mock) CreateCluster(ctx context.Context, in driver.CreateClusterInput) Settings: append([]driver.Setting(nil), in.Settings...), } - // Serialize the reject-if-ACTIVE / create-or-reuse compare-and-set so two - // concurrent creates of the same name can't both succeed. Only an ACTIVE - // cluster of the same name is a conflict; a previously deleted (INACTIVE) + // Serialize the return-if-ACTIVE / create-or-reuse compare-and-set so two + // concurrent creates of the same name can't both create. CreateCluster is + // idempotent: an ACTIVE cluster of the same name is returned unchanged (the + // new tags and settings are ignored), while a previously deleted (INACTIVE) // tombstone is overwritten, so a deleted cluster name can be recreated. - // Real ECS lets the name be reused once the old cluster is gone. m.clusterMu.Lock() if existing, ok := m.clusters.Get(name); ok && existing.Status == statusActive { + out := cloneCluster(existing) m.clusterMu.Unlock() - return nil, errors.Newf(errors.AlreadyExists, "cluster %q already exists", name) + + return &out, nil } m.clusters.Set(name, c) m.clusterMu.Unlock() m.recordTags(c.ARN, in.Tags) + m.publishClusterMetrics(name) out := cloneCluster(c) @@ -182,6 +186,7 @@ func (m *Mock) mutateCluster(id string, fn func(*driver.Cluster)) (*driver.Clust } out := m.describeCluster(&updated) + m.publishClusterMetrics(name) return &out, nil } diff --git a/providers/aws/ecs/container_instances.go b/providers/aws/ecs/container_instances.go index d78a83a8f..0ada2f3d7 100644 --- a/providers/aws/ecs/container_instances.go +++ b/providers/aws/ecs/container_instances.go @@ -49,6 +49,7 @@ func (m *Mock) newInstance(cluster, ec2InstanceID string, cpu, memory int) *driv RegisteredMemory: memory, RemainingCPU: cpu, RemainingMemory: memory, + Version: 1, } } @@ -102,7 +103,7 @@ func (m *Mock) SeedContainerInstance(cluster, ec2InstanceID string, opts ...Inst // //nolint:gocritic // in is passed by value to satisfy the driver.ECS interface; the copy is cheap for a mock. func (m *Mock) RegisterContainerInstance( - _ context.Context, in driver.RegisterContainerInstanceInput, + ctx context.Context, in driver.RegisterContainerInstanceInput, ) (*driver.ContainerInstance, error) { cpu, memory := capacityFromResources(in.TotalResources) @@ -124,6 +125,9 @@ func (m *Mock) RegisterContainerInstance( out := *ci out.Tags = copyTags(ci.Tags) + m.emitContainerInstanceDetail(ctx, &out) + m.publishClusterMetrics(resolveClusterName(in.Cluster)) + return &out, nil } @@ -141,9 +145,13 @@ func (m *Mock) DeregisterContainerInstance( // The force-stopped tasks are published once placeMu is released, so an // event target calling back into ECS cannot deadlock on it. for _, t := range stopped { - m.emitTaskStateChange(ctx, t, taskEventVersionStop) + m.emitTaskStop(ctx, t) } + out.Version++ + m.emitContainerInstanceDetail(ctx, out) + m.publishClusterMetrics(instanceClusterName(out.ARN)) + return out, nil } @@ -203,6 +211,7 @@ func (m *Mock) stopTasksOnInstance(instanceARN string) []*driver.Task { updated.DesiredStatus = statusStopped updated.StoppedReason = "Container instance deregistered." updated.StopCode = "TerminationNotice" + updated.EventVersion = t.EventVersion + len(stopSteps(&updated)) m.stampStop(&updated) for i := range updated.Containers { @@ -223,19 +232,32 @@ func (m *Mock) stopTasksOnInstance(instanceARN string) []*driver.Task { // DRAINING; unresolved ids become failures. Only ACTIVE and DRAINING are valid // target states. func (m *Mock) UpdateContainerInstancesState( - _ context.Context, _ string, ids []string, status string, + ctx context.Context, _ string, ids []string, status string, ) ([]driver.ContainerInstance, []driver.Failure, error) { if status != statusActive && status != statusDraining { return nil, nil, apiErrf(errors.InvalidArgument, excInvalidParameter, "container instance status must be ACTIVE or DRAINING, got %q", status) } - // Hold placeMu for the whole read-modify-write: reserve (capacity.go), - // release/StopTask, and DeregisterContainerInstance all mutate an instance - // under placeMu, so resolving-then-Set without it would let a concurrent - // capacity reserve be silently reverted (a logical lost update -race can't - // see, since memstore.Set is per-key locked). Resolving under the lock also - // guarantees the clone reflects the freshest capacity counts. + found, failures := m.setInstanceStatusLocked(ids, status) + + // The state changes are published once placeMu is released. + for i := range found { + m.emitContainerInstanceDetail(ctx, &found[i]) + m.publishClusterMetrics(instanceClusterName(found[i].ARN)) + } + + return found, failures, nil +} + +// setInstanceStatusLocked is UpdateContainerInstancesState's placeMu-guarded +// core. It holds placeMu for the whole read-modify-write: reserve (capacity.go), +// release/StopTask, and DeregisterContainerInstance all mutate an instance under +// placeMu, so resolving-then-Set without it would let a concurrent capacity +// reserve be silently reverted (a logical lost update -race can't see, since +// memstore.Set is per-key locked). Resolving under the lock also guarantees the +// clone reflects the freshest capacity counts. +func (m *Mock) setInstanceStatusLocked(ids []string, status string) ([]driver.ContainerInstance, []driver.Failure) { m.placeMu.Lock() defer m.placeMu.Unlock() @@ -253,12 +275,13 @@ func (m *Mock) UpdateContainerInstancesState( // never race the status write. updated := *ci updated.Status = status + updated.Version++ m.instances.Set(updated.ARN, &updated) found = append(found, updated) } - return found, failures, nil + return found, failures } // capacityFromResources derives CPU units and memory (MiB) from the CPU and diff --git a/providers/aws/ecs/ecs.go b/providers/aws/ecs/ecs.go index 42edda89e..20e89b43d 100644 --- a/providers/aws/ecs/ecs.go +++ b/providers/aws/ecs/ecs.go @@ -16,6 +16,7 @@ import ( "github.com/stackshy/cloudemu/v2/internal/settle" "github.com/stackshy/cloudemu/v2/services/ecs/driver" logdriver "github.com/stackshy/cloudemu/v2/services/logging/driver" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" ) // Compile-time check that Mock implements driver.ECS. @@ -50,10 +51,15 @@ type Mock struct { settings *memstore.Store[*driver.AccountSetting] // keyed by setting name attributes *memstore.Store[*driver.Attribute] // keyed by targetId + "\x00" + name capacityProviders *memstore.Store[*driver.CapacityProvider] // keyed by name; excludes the predefined FARGATE/FARGATE_SPOT - opts *config.Options - regMu sync.Mutex // serializes task-definition revision allocation - placeMu sync.Mutex // serializes container-instance capacity reserve/release - clusterMu sync.Mutex // serializes CreateCluster name-reuse compare-and-set + taskSets *memstore.Store[*driver.TaskSet] // keyed by "cluster/service/id" (the id itself contains a slash) + + serviceDeployments *memstore.Store[*driver.ServiceDeployment] // keyed by deployment ARN + serviceRevisions *memstore.Store[*driver.ServiceRevision] // keyed by revision ARN + opts *config.Options + regMu sync.Mutex // serializes task-definition revision allocation + placeMu sync.Mutex // serializes container-instance capacity reserve/release + clusterMu sync.Mutex // serializes CreateCluster name-reuse compare-and-set + protectMu sync.Mutex // serializes UpdateTaskProtection's count check and writes // reconcileLock serializes reconcileServiceAfterStop per service (see // service_reconcile_lock.go), closing the concurrent-StopTask over-launch @@ -63,6 +69,24 @@ type Mock struct { // deadlock against them. reconcileLock *serviceReconcileLock + // taskSetLock serializes every task-set mutation of one service (create, + // scale, promote, delete, rescale on a desired-count change, replacement + // after a stop) and DeleteService's cascade over the same set, so a task + // set can never be created into a service that is concurrently deleted. It + // is taken before placeMu (task launches/stops inside it) and never while + // holding placeMu, reconcileLock or an event publish. + taskSetLock *serviceReconcileLock + + // deploymentLock serializes one service's deployment-history writes (record, + // trim, stop, delete cascade). It is taken after taskSetLock and never while + // holding placeMu. + deploymentLock *serviceReconcileLock + + // deploymentSettle overlays a transient status (IN_PROGRESS after a deploy, + // ROLLBACK_IN_PROGRESS / STOP_REQUESTED after a stop) onto a deployment's + // stored final status when opts.AsyncSettle is enabled. + deploymentSettle *settle.Set + launcher ManagedInstanceLauncher // optional: provisions backing managed EC2 instances // engineHandles maps a task ARN to the config.ContainerEngine handle backing @@ -84,6 +108,8 @@ type Mock struct { events awsevents.Emitter // optional: EventBridge default bus for task state change / service action events + monitoring mondriver.Monitoring // optional: CloudWatch backend for AWS/ECS and ECS/ContainerInsights metrics + // portCounter draws successive dynamic host ports for bridge-mode container // port mappings that leave hostPort unset. portCounter atomic.Uint32 @@ -118,10 +144,17 @@ func New(opts *config.Options) *Mock { settings: memstore.New[*driver.AccountSetting](), attributes: memstore.New[*driver.Attribute](), capacityProviders: memstore.New[*driver.CapacityProvider](), - engineHandles: memstore.New[string](), - taskSettle: settle.NewSet(), - reconcileLock: newServiceReconcileLock(), - opts: opts, + taskSets: memstore.New[*driver.TaskSet](), + + serviceDeployments: memstore.New[*driver.ServiceDeployment](), + serviceRevisions: memstore.New[*driver.ServiceRevision](), + deploymentLock: newServiceReconcileLock(), + deploymentSettle: settle.NewSet(), + engineHandles: memstore.New[string](), + taskSettle: settle.NewSet(), + reconcileLock: newServiceReconcileLock(), + taskSetLock: newServiceReconcileLock(), + opts: opts, } } diff --git a/providers/aws/ecs/ecs_test.go b/providers/aws/ecs/ecs_test.go index 506e1f48b..6da55a26e 100644 --- a/providers/aws/ecs/ecs_test.go +++ b/providers/aws/ecs/ecs_test.go @@ -30,9 +30,10 @@ func TestClusterLifecycle(t *testing.T) { assert.Equal(t, statusActive, c.Status) assert.Contains(t, c.ARN, "cluster/prod") - // Duplicate name is rejected. - _, err = m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) - assert.True(t, errors.IsAlreadyExists(err)) + // Re-creating an ACTIVE cluster is idempotent. + again, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + assert.Equal(t, c.ARN, again.ARN) list, err := m.ListClusters(ctx) require.NoError(t, err) diff --git a/providers/aws/ecs/engine_test.go b/providers/aws/ecs/engine_test.go index ab141f11f..9188a8407 100644 --- a/providers/aws/ecs/engine_test.go +++ b/providers/aws/ecs/engine_test.go @@ -72,6 +72,13 @@ func newEngineTestMock(eng config.ContainerEngine) *Mock { func registerAndRun(t *testing.T, m *Mock, cd driver.ContainerDefinition) driver.Task { t.Helper() + return registerAndRunExec(t, m, cd, false) +} + +// registerAndRunExec is registerAndRun with ECS Exec enabled or not on the task. +func registerAndRunExec(t *testing.T, m *Mock, cd driver.ContainerDefinition, enableExec bool) driver.Task { + t.Helper() + ctx := context.Background() _, err := m.RegisterTaskDefinition(ctx, driver.RegisterTaskDefinitionInput{ @@ -81,7 +88,7 @@ func registerAndRun(t *testing.T, m *Mock, cd driver.ContainerDefinition) driver require.NoError(t, err) tasks, failures, err := m.RunTask(ctx, driver.RunTaskInput{ - TaskDefinition: "web", LaunchType: "EXTERNAL", Count: 1, + TaskDefinition: "web", LaunchType: "EXTERNAL", Count: 1, EnableExecuteCommand: enableExec, }) require.NoError(t, err) require.Empty(t, failures) @@ -258,7 +265,7 @@ func TestExecuteCommandExecsEngine(t *testing.T) { m := newEngineTestMock(eng) ctx := context.Background() - task := registerAndRun(t, m, driver.ContainerDefinition{Name: "app", Image: "img"}) + task := registerAndRunExec(t, m, driver.ContainerDefinition{Name: "app", Image: "img"}, true) res, err := m.ExecuteCommand(ctx, driver.ExecuteCommandInput{ Task: task.ARN, Container: "app", Command: "ls -la /tmp", Interactive: true, @@ -273,7 +280,7 @@ func TestExecuteCommandExecsEngine(t *testing.T) { func TestNilEngineLeavesTaskSynthetic(t *testing.T) { m := newTestMock() - task := registerAndRun(t, m, driver.ContainerDefinition{Name: "app", Image: "img"}) + task := registerAndRunExec(t, m, driver.ContainerDefinition{Name: "app", Image: "img"}, true) require.Len(t, task.Containers, 1) assert.Equal(t, statusRunning, task.Containers[0].LastStatus) @@ -282,7 +289,7 @@ func TestNilEngineLeavesTaskSynthetic(t *testing.T) { assert.Equal(t, statusRunning, task.LastStatus) // ExecuteCommand stays purely synthetic with no engine wired. - res, err := m.ExecuteCommand(context.Background(), driver.ExecuteCommandInput{Task: task.ARN}) + res, err := m.ExecuteCommand(context.Background(), driver.ExecuteCommandInput{Task: task.ARN, Interactive: true}) require.NoError(t, err) assert.NotEmpty(t, res.Session.SessionID) } diff --git a/providers/aws/ecs/errors.go b/providers/aws/ecs/errors.go index 33d04bb07..15f416d1a 100644 --- a/providers/aws/ecs/errors.go +++ b/providers/aws/ecs/errors.go @@ -14,6 +14,11 @@ const ( excClusterContainsTasks = "ClusterContainsTasksException" excClusterContainsInstances = "ClusterContainsContainerInstancesException" excServer = "ServerException" + excTaskSetNotFound = "TaskSetNotFoundException" + excServiceNotActive = "ServiceNotActiveException" + excConflict = "ConflictException" + + excServiceDeploymentNotFound = "ServiceDeploymentNotFoundException" ) // apiError pairs a canonical cloudemu error with the precise ECS exception name diff --git a/providers/aws/ecs/events.go b/providers/aws/ecs/events.go index a9d6210e4..ea89599ee 100644 --- a/providers/aws/ecs/events.go +++ b/providers/aws/ecs/events.go @@ -8,22 +8,99 @@ import ( ) // EventBridge identity of ECS's native events. See "Amazon ECS events" in the -// ECS developer guide (task state change and service action events). +// ECS developer guide (task, container instance, service deployment and service +// action events). const ( - eventSource = "aws.ecs" - eventTaskStateChange = "ECS Task State Change" - eventServiceAction = "ECS Service Action" - serviceEventTypeInfo = "INFO" - serviceSteadyStateName = "SERVICE_STEADY_STATE" - - // taskEventVersionLaunch / taskEventVersionStop are the detail.version of a - // task's launch and stop state-change events. Real ECS increments version on - // every task state change so consumers can drop out-of-order events; the - // emulator publishes one event per stored transition, in order. - taskEventVersionLaunch = 1 - taskEventVersionStop = 2 + eventSource = "aws.ecs" + eventTaskStateChange = "ECS Task State Change" + eventServiceAction = "ECS Service Action" + eventDeploymentStateChange = "ECS Deployment State Change" + eventContainerInstanceStateChange = "ECS Container Instance State Change" + + serviceEventTypeInfo = "INFO" + serviceEventTypeError = "ERROR" + + serviceSteadyStateName = "SERVICE_STEADY_STATE" + serviceDesiredCountUpdated = "SERVICE_DESIRED_COUNT_UPDATED" + serviceDeploymentInProgress = "SERVICE_DEPLOYMENT_IN_PROGRESS" + serviceDeploymentCompleted = "SERVICE_DEPLOYMENT_COMPLETED" + serviceTaskPlacementFailure = "SERVICE_TASK_PLACEMENT_FAILURE" + containerInstanceResourceType = "INTEGER" +) + +// Task lifecycle states, from the ECS task lifecycle in the developer guide: +// PROVISIONING (awsvpc only) -> PENDING -> ACTIVATING -> RUNNING -> DEACTIVATING +// -> STOPPING -> DEPROVISIONING (awsvpc only) -> STOPPED. The emulator moves a +// task through them synchronously and publishes one event per state. +const ( + taskStatusActivating = "ACTIVATING" + taskStatusDeactivating = "DEACTIVATING" + + // maxLaunchSteps bounds the launch-then-stop state list of one task. + maxLaunchSteps = 8 ) +// launchSteps lists the lastStatus values a task passes through when it is +// launched, ending at the task's current state: an unplaced EC2 task stays +// PENDING, a task that already ran to completion continues through its stop. +func launchSteps(t *driver.Task) []string { + steps := make([]string, 0, maxLaunchSteps) + + if len(t.Attachments) > 0 { + steps = append(steps, taskStatusProvisioning) + } + + steps = append(steps, statusPending) + if t.LastStatus == statusPending { + return steps + } + + steps = append(steps, taskStatusActivating, statusRunning) + if t.LastStatus == statusStopped { + steps = append(steps, stopSteps(t)...) + } + + return steps +} + +// stopSteps lists the lastStatus values a task passes through when it stops. +func stopSteps(t *driver.Task) []string { + steps := []string{taskStatusDeactivating, taskStatusStopping} + if len(t.Attachments) > 0 { + steps = append(steps, taskStatusDeprovisioning) + } + + return append(steps, statusStopped) +} + +// stoppingPhase reports whether step belongs to the stop half of the lifecycle. +func stoppingPhase(step string) bool { + switch step { + case taskStatusDeactivating, taskStatusStopping, taskStatusDeprovisioning, statusStopped: + return true + default: + return false + } +} + +// runningOrLater reports whether the task has reached RUNNING by step. +func runningOrLater(step string) bool { + return step == statusRunning || stoppingPhase(step) +} + +// containerStatusAt is the lastStatus a task's containers report while the task +// is in step. +func containerStatusAt(step string) string { + switch step { + case statusStopped: + return statusStopped + case statusRunning, taskStatusDeactivating, taskStatusStopping, taskStatusDeprovisioning: + return statusRunning + default: + return statusPending + } +} + // taskStateChangeDetail is the detail payload of an "ECS Task State Change" // event: the task record as DescribeTasks reports it. type taskStateChangeDetail struct { @@ -66,6 +143,7 @@ type taskEventContainer struct { } type taskEventAttachment struct { + ID string `json:"id,omitempty"` Details []taskEventKeyValue `json:"details"` Status string `json:"status"` Type string `json:"type"` @@ -82,50 +160,158 @@ type serviceActionDetail struct { EventName string `json:"eventName"` ClusterArn string `json:"clusterArn"` CreatedAt string `json:"createdAt"` + Reason string `json:"reason,omitempty"` +} + +// deploymentStateChangeDetail is the detail payload of an "ECS Deployment State +// Change" event. +type deploymentStateChangeDetail struct { + EventType string `json:"eventType"` + EventName string `json:"eventName"` + DeploymentID string `json:"deploymentId"` + UpdatedAt string `json:"updatedAt"` + Reason string `json:"reason"` } -// pendingTaskEvents buffers task state changes made while a service record is -// not yet committed (CreateService/UpdateService/DeleteService converging or -// draining it). They are published only after the caller stores the service: -// an event target that stops a task re-enters reconcileServiceAfterStop, which -// must see the committed record; against the half-built or superseded one, it -// computes the wrong shortfall and over-provisions the service. +// containerInstanceDetail is the detail payload of an "ECS Container Instance +// State Change" event. +type containerInstanceDetail struct { + AgentConnected bool `json:"agentConnected"` + Attributes []containerInstanceAttr `json:"attributes"` + ClusterArn string `json:"clusterArn"` + ContainerInstanceArn string `json:"containerInstanceArn"` + EC2InstanceID string `json:"ec2InstanceId"` + RegisteredResources []containerInstanceResource `json:"registeredResources"` + RemainingResources []containerInstanceResource `json:"remainingResources"` + Status string `json:"status"` + Version int `json:"version"` + UpdatedAt string `json:"updatedAt"` +} + +type containerInstanceAttr struct { + Name string `json:"name"` +} + +type containerInstanceResource struct { + Name string `json:"name"` + Type string `json:"type"` + IntegerValue int `json:"integerValue"` +} + +// pendingTaskEvents buffers events produced while a service record or task-set +// lock is held (CreateService/UpdateService/DeleteService converging or draining +// a service, task-set scaling). They are published only after the caller commits +// and releases its locks: an event target that stops a task re-enters +// reconcileServiceAfterStop, which must see the committed record and can take +// the same non-reentrant locks, and against a half-built or superseded record it +// would compute the wrong shortfall and over-provision the service. type pendingTaskEvents struct { - tasks []*driver.Task - versions []int + emits []func(context.Context) +} + +// addLaunch buffers the launch events of a task snapshot. +func (p *pendingTaskEvents) addLaunch(m *Mock, t *driver.Task) { + p.emits = append(p.emits, func(ctx context.Context) { m.emitTaskLaunch(ctx, t) }) } -func (p *pendingTaskEvents) add(t *driver.Task, version int) { - p.tasks = append(p.tasks, t) - p.versions = append(p.versions, version) +// addStop buffers the stop events of a task snapshot. +func (p *pendingTaskEvents) addStop(m *Mock, t *driver.Task) { + p.emits = append(p.emits, func(ctx context.Context) { m.emitTaskStop(ctx, t) }) +} + +// addFunc buffers an arbitrary emit. +func (p *pendingTaskEvents) addFunc(fn func(context.Context)) { + p.emits = append(p.emits, fn) } // publish emits the buffered events in the order they happened. -func (m *Mock) publish(ctx context.Context, p *pendingTaskEvents) { - for i, t := range p.tasks { - m.emitTaskStateChange(ctx, t, p.versions[i]) +func (*Mock) publish(ctx context.Context, p *pendingTaskEvents) { + for _, emit := range p.emits { + emit(ctx) } } -// SetEventPublisher wires the EventBridge default bus that task state changes -// and service actions are published to. Safe to leave unset. No events are -// emitted. +// SetEventPublisher wires the EventBridge default bus that task, container +// instance, deployment and service action events are published to. Safe to +// leave unset: no events are emitted then. func (m *Mock) SetEventPublisher(p awsevents.Publisher) { m.events.SetPublisher(p) } -// emitTaskStateChange publishes an "ECS Task State Change" event carrying the -// task's stored (final) state. It must be called without placeMu held. -func (m *Mock) emitTaskStateChange(ctx context.Context, t *driver.Task, version int) { +// emitTaskLaunch publishes the task state changes of a task's launch, one event +// per lifecycle state, plus the change in its container instance's resources. +// It must be called without any emulator lock held. +func (m *Mock) emitTaskLaunch(ctx context.Context, t *driver.Task) { + m.emitTaskSteps(ctx, t, launchSteps(t)) + m.emitInstanceChangeOfTask(ctx, t) +} + +// emitTaskStop publishes the task state changes of a task's stop. It must be +// called without any emulator lock held. +func (m *Mock) emitTaskStop(ctx context.Context, t *driver.Task) { + m.emitTaskSteps(ctx, t, stopSteps(t)) + m.emitInstanceChangeOfTask(ctx, t) +} + +// emitTaskSteps publishes one "ECS Task State Change" event per step. The task +// snapshot's EventVersion is the version of its last step, so the steps carry +// consecutive versions ending there. +func (m *Mock) emitTaskSteps(ctx context.Context, t *driver.Task, steps []string) { + first := t.EventVersion - len(steps) + 1 + + for i, step := range steps { + m.emitTaskStep(ctx, t, step, first+i) + } +} + +// emitTaskStep publishes the "ECS Task State Change" event of one lifecycle +// state, masking the snapshot's fields to what is true in that state. +func (m *Mock) emitTaskStep(ctx context.Context, t *driver.Task, step string, version int) { + d := taskStateChangeDetail{ + Attachments: eventAttachments(t), AvailabilityZone: t.AvailabilityZone, ClusterArn: t.ClusterARN, + ContainerInstanceArn: t.ContainerInstanceARN, Containers: eventContainers(t, step), + CPU: t.CPU, CreatedAt: t.CreatedAt, DesiredStatus: statusRunning, Group: t.Group, + LastStatus: step, LaunchType: t.LaunchType, Memory: t.Memory, PlatformVersion: t.PlatformVersion, + StartedBy: t.StartedBy, TaskArn: t.ARN, TaskDefinitionArn: t.TaskDefinitionARN, UpdatedAt: m.now(), Version: version, + } + + maskTaskDetail(&d, t, step) + + m.events.Emit(ctx, eventSource, eventTaskStateChange, d, t.ARN) +} + +// maskTaskDetail sets the fields of a task event that are only true from a given +// lifecycle state on: start data once running, stop data once stopping. +func maskTaskDetail(d *taskStateChangeDetail, t *driver.Task, step string) { + if runningOrLater(step) { + d.StartedAt, d.Connectivity = t.StartedAt, t.Connectivity + } + + if stoppingPhase(step) { + d.DesiredStatus = statusStopped + d.StopCode, d.StoppedReason = t.StopCode, t.StoppedReason + } + + if step == taskStatusStopping || step == taskStatusDeprovisioning || step == statusStopped { + d.StoppingAt = t.StoppingAt + } + + if step == statusStopped { + d.StoppedAt = t.StoppedAt + } +} + +// eventContainers lists the task's containers as they appear in an event at step. +func eventContainers(t *driver.Task, step string) []taskEventContainer { containers := make([]taskEventContainer, 0, len(t.Containers)) for i := range t.Containers { c := &t.Containers[i] ec := taskEventContainer{ - ContainerArn: c.ARN, Image: c.Image, LastStatus: c.LastStatus, Name: c.Name, Reason: c.Reason, TaskArn: t.ARN, + ContainerArn: c.ARN, Image: c.Image, LastStatus: containerStatusAt(step), Name: c.Name, Reason: c.Reason, TaskArn: t.ARN, } - if c.LastStatus == statusStopped { + if step == statusStopped { code := c.ExitCode ec.ExitCode = &code } @@ -133,6 +319,11 @@ func (m *Mock) emitTaskStateChange(ctx context.Context, t *driver.Task, version containers = append(containers, ec) } + return containers +} + +// eventAttachments lists the task's attachments as they appear in an event. +func eventAttachments(t *driver.Task) []taskEventAttachment { attachments := make([]taskEventAttachment, 0, len(t.Attachments)) for _, a := range t.Attachments { @@ -141,18 +332,10 @@ func (m *Mock) emitTaskStateChange(ctx context.Context, t *driver.Task, version details = append(details, taskEventKeyValue{Name: kv.Name, Value: kv.Value}) } - attachments = append(attachments, taskEventAttachment{Details: details, Status: a.Status, Type: a.Type}) + attachments = append(attachments, taskEventAttachment{ID: a.ID, Details: details, Status: a.Status, Type: a.Type}) } - m.events.Emit(ctx, eventSource, eventTaskStateChange, taskStateChangeDetail{ - Attachments: attachments, AvailabilityZone: t.AvailabilityZone, ClusterArn: t.ClusterARN, - Connectivity: t.Connectivity, ContainerInstanceArn: t.ContainerInstanceARN, Containers: containers, - CPU: t.CPU, CreatedAt: t.CreatedAt, DesiredStatus: t.DesiredStatus, Group: t.Group, - LastStatus: t.LastStatus, LaunchType: t.LaunchType, Memory: t.Memory, PlatformVersion: t.PlatformVersion, - StartedAt: t.StartedAt, StartedBy: t.StartedBy, StopCode: t.StopCode, StoppedAt: t.StoppedAt, - StoppedReason: t.StoppedReason, StoppingAt: t.StoppingAt, TaskArn: t.ARN, - TaskDefinitionArn: t.TaskDefinitionARN, UpdatedAt: m.now(), Version: version, - }, t.ARN) + return attachments } // emitServiceSteadyState publishes the SERVICE_STEADY_STATE service action once @@ -163,8 +346,92 @@ func (m *Mock) emitServiceSteadyState(ctx context.Context, svc *driver.Service) return } + m.emitServiceAction(ctx, svc.ARN, svc.ClusterARN, serviceSteadyStateName, serviceEventTypeInfo, "") +} + +// emitServiceAction publishes an "ECS Service Action" event. +func (m *Mock) emitServiceAction(ctx context.Context, serviceARN, clusterARN, name, eventType, reason string) { m.events.Emit(ctx, eventSource, eventServiceAction, serviceActionDetail{ - EventType: serviceEventTypeInfo, EventName: serviceSteadyStateName, - ClusterArn: svc.ClusterARN, CreatedAt: m.now(), + EventType: eventType, EventName: name, ClusterArn: clusterARN, CreatedAt: m.now(), Reason: reason, + }, serviceARN) +} + +// emitDeploymentEvents publishes a started deployment: it is in progress, and +// completed once the service runs its desired count (a service that cannot +// place its tasks stays in progress). Each is both an "ECS Deployment State +// Change" and an "ECS Service Action" event. It must be called without any +// emulator lock held. +func (m *Mock) emitDeploymentEvents(ctx context.Context, svc *driver.Service) { + id := primaryDeploymentID(svc.Deployments) + if id == "" { + return + } + + m.emitDeploymentChange(ctx, svc, id, serviceDeploymentInProgress, "ECS deployment "+id+" in progress.") + + if svc.RunningCount >= svc.DesiredCount { + m.emitDeploymentChange(ctx, svc, id, serviceDeploymentCompleted, "ECS deployment "+id+" completed.") + } +} + +func (m *Mock) emitDeploymentChange(ctx context.Context, svc *driver.Service, id, name, reason string) { + m.emitServiceAction(ctx, svc.ARN, svc.ClusterARN, name, serviceEventTypeInfo, "") + m.events.Emit(ctx, eventSource, eventDeploymentStateChange, deploymentStateChangeDetail{ + EventType: serviceEventTypeInfo, EventName: name, DeploymentID: id, UpdatedAt: m.now(), Reason: reason, }, svc.ARN) } + +// emitInstanceChangeOfTask publishes the container-instance state change a task +// placement or stop caused. Tasks with no instance (Fargate, unplaced) cause +// none. +func (m *Mock) emitInstanceChangeOfTask(ctx context.Context, t *driver.Task) { + if t.ContainerInstanceARN == "" { + return + } + + m.placeMu.Lock() + + var bumped *driver.ContainerInstance + + m.instances.Update(t.ContainerInstanceARN, func(ci *driver.ContainerInstance) *driver.ContainerInstance { + updated := *ci + updated.Version++ + bumped = &updated + + return &updated + }) + m.placeMu.Unlock() + + if bumped != nil { + m.emitContainerInstanceDetail(ctx, bumped) + } +} + +// emitContainerInstanceDetail publishes an "ECS Container Instance State Change" +// event for the instance as given (its Version already counts this change). +func (m *Mock) emitContainerInstanceDetail(ctx context.Context, ci *driver.ContainerInstance) { + cluster := instanceClusterName(ci.ARN) + + var attrs []containerInstanceAttr + + for _, a := range m.attributes.SortedValues() { + if a.TargetID == ci.ARN { + attrs = append(attrs, containerInstanceAttr{Name: a.Name}) + } + } + + m.events.Emit(ctx, eventSource, eventContainerInstanceStateChange, containerInstanceDetail{ + AgentConnected: ci.AgentConnected, Attributes: attrs, + ClusterArn: m.arnIn(arnRegion(ci.ARN, m.opts.Region), "cluster/"+cluster), + ContainerInstanceArn: ci.ARN, EC2InstanceID: ci.EC2InstanceID, + RegisteredResources: []containerInstanceResource{ + {Name: "CPU", Type: containerInstanceResourceType, IntegerValue: ci.RegisteredCPU}, + {Name: "MEMORY", Type: containerInstanceResourceType, IntegerValue: ci.RegisteredMemory}, + }, + RemainingResources: []containerInstanceResource{ + {Name: "CPU", Type: containerInstanceResourceType, IntegerValue: ci.RemainingCPU}, + {Name: "MEMORY", Type: containerInstanceResourceType, IntegerValue: ci.RemainingMemory}, + }, + Status: ci.Status, Version: ci.Version, UpdatedAt: m.now(), + }, ci.ARN) +} diff --git a/providers/aws/ecs/events_test.go b/providers/aws/ecs/events_test.go new file mode 100644 index 000000000..1b2f2efa9 --- /dev/null +++ b/providers/aws/ecs/events_test.go @@ -0,0 +1,403 @@ +package ecs + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// eventsOf returns the recorded events of one detail type, in order. +func eventsOf(rec *recordingPublisher, detailType string) []publishedEvent { + var out []publishedEvent + + for _, e := range rec.snapshot() { + if e.DetailType == detailType { + out = append(out, e) + } + } + + return out +} + +// taskSteps returns the (lastStatus, version) sequence of the task state change +// events published for one task. +func taskSteps(rec *recordingPublisher, taskARN string) (statuses []string, versions []int) { + for _, e := range eventsOf(rec, eventTaskStateChange) { + d, ok := e.Detail.(taskStateChangeDetail) + if ok && d.TaskArn == taskARN { + statuses = append(statuses, d.LastStatus) + versions = append(versions, d.Version) + } + } + + return statuses, versions +} + +func TestTaskStateChange_AllTransitions_Fargate(t *testing.T) { + m := newTestMock() + rec := &recordingPublisher{} + m.SetEventPublisher(rec) + netCfg := fargateFixture(t, m) + + task := runFargate(t, m, netCfg, false) + + statuses, versions := taskSteps(rec, task.ARN) + assert.Equal(t, []string{"PROVISIONING", "PENDING", "ACTIVATING", "RUNNING"}, statuses) + assert.Equal(t, []int{1, 2, 3, 4}, versions) + + _, err := m.StopTask(context.Background(), "prod", task.ARN, "bye") + require.NoError(t, err) + + statuses, versions = taskSteps(rec, task.ARN) + assert.Equal(t, []string{ + "PROVISIONING", "PENDING", "ACTIVATING", "RUNNING", "DEACTIVATING", "STOPPING", "DEPROVISIONING", "STOPPED", + }, statuses) + assert.Equal(t, []int{1, 2, 3, 4, 5, 6, 7, 8}, versions, "version increases by one per change") + + last, ok := eventsOf(rec, eventTaskStateChange)[7].Detail.(taskStateChangeDetail) + require.True(t, ok) + assert.Equal(t, "bye", last.StoppedReason) + assert.NotEmpty(t, last.StoppedAt) + assert.Equal(t, statusStopped, last.DesiredStatus) + + // A running-phase event has not started stopping yet. + running, ok := eventsOf(rec, eventTaskStateChange)[3].Detail.(taskStateChangeDetail) + require.True(t, ok) + assert.Empty(t, running.StoppedAt) + assert.NotEmpty(t, running.StartedAt) + + early, ok := eventsOf(rec, eventTaskStateChange)[0].Detail.(taskStateChangeDetail) + require.True(t, ok) + assert.Empty(t, early.StartedAt, "a task that is still provisioning has not started") +} + +func TestTaskStateChange_AllTransitions_EC2(t *testing.T) { + m := newTestMock() + rec := &recordingPublisher{} + m.SetEventPublisher(rec) + + _, err := m.CreateCluster(context.Background(), driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + registerWeb(t, m, 128, 128) + m.SeedContainerInstance("prod", "i-1", WithCapacity(1024, 1024)) + + tasks, _, err := m.RunTask(context.Background(), driver.RunTaskInput{Cluster: "prod", TaskDefinition: "web"}) + require.NoError(t, err) + + statuses, versions := taskSteps(rec, tasks[0].ARN) + assert.Equal(t, []string{"PENDING", "ACTIVATING", "RUNNING"}, statuses) + assert.Equal(t, []int{1, 2, 3}, versions) + + _, err = m.StopTask(context.Background(), "prod", tasks[0].ARN, "bye") + require.NoError(t, err) + + statuses, versions = taskSteps(rec, tasks[0].ARN) + assert.Equal(t, []string{"PENDING", "ACTIVATING", "RUNNING", "DEACTIVATING", "STOPPING", "STOPPED"}, statuses, + "no DEPROVISIONING without an ENI") + assert.Equal(t, []int{1, 2, 3, 4, 5, 6}, versions) + + // A second StopTask publishes nothing more. + _, err = m.StopTask(context.Background(), "prod", tasks[0].ARN, "again") + require.NoError(t, err) + + again, _ := taskSteps(rec, tasks[0].ARN) + assert.Len(t, again, 6) +} + +func TestDeploymentStateChange(t *testing.T) { + m := newTestMock() + rec := &recordingPublisher{} + m.SetEventPublisher(rec) + netCfg := fargateFixture(t, m) + ctx := context.Background() + + svc, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "web", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, + NetworkConfiguration: netCfg, DesiredCount: 1, + }) + require.NoError(t, err) + + deployID := svc.Deployments[0].ID + + var got []deploymentStateChangeDetail + + for _, e := range eventsOf(rec, eventDeploymentStateChange) { + assert.Equal(t, []string{svc.ARN}, e.Resources) + + d, ok := e.Detail.(deploymentStateChangeDetail) + require.True(t, ok) + + got = append(got, d) + } + + require.Len(t, got, 2) + assert.Equal(t, "SERVICE_DEPLOYMENT_IN_PROGRESS", got[0].EventName) + assert.Equal(t, "SERVICE_DEPLOYMENT_COMPLETED", got[1].EventName) + assert.Equal(t, serviceEventTypeInfo, got[0].EventType) + assert.Equal(t, deployID, got[0].DeploymentID) + assert.Equal(t, "ECS deployment "+deployID+" in progress.", got[0].Reason) + assert.Equal(t, "ECS deployment "+deployID+" completed.", got[1].Reason) + assert.NotEmpty(t, got[0].UpdatedAt) + + // The same deployment also surfaces as service actions. + var actions []string + + for _, e := range eventsOf(rec, eventServiceAction) { + actions = append(actions, e.Detail.(serviceActionDetail).EventName) + } + + assert.Contains(t, actions, "SERVICE_DEPLOYMENT_IN_PROGRESS") + assert.Contains(t, actions, "SERVICE_DEPLOYMENT_COMPLETED") + assert.Contains(t, actions, serviceSteadyStateName) + + before := len(eventsOf(rec, eventDeploymentStateChange)) + + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", ForceNewDeployment: true}) + require.NoError(t, err) + assert.Len(t, eventsOf(rec, eventDeploymentStateChange), before+2, "a new deployment emits again") +} + +func TestDeploymentStateChange_NoCompletionWhileShort(t *testing.T) { + m := newTestMock() + rec := &recordingPublisher{} + m.SetEventPublisher(rec) + + _, err := m.CreateCluster(context.Background(), driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + registerWeb(t, m, 128, 128) + + _, err = m.CreateService(context.Background(), driver.CreateServiceInput{ + ServiceName: "web", Cluster: "prod", TaskDefinition: "web", DesiredCount: 1, + }) + require.NoError(t, err) + + evs := eventsOf(rec, eventDeploymentStateChange) + require.Len(t, evs, 1, "an unplaceable service never completes its deployment") + + d, ok := evs[0].Detail.(deploymentStateChangeDetail) + require.True(t, ok) + assert.Equal(t, "SERVICE_DEPLOYMENT_IN_PROGRESS", d.EventName) +} + +func TestContainerInstanceStateChange(t *testing.T) { + m := newTestMock() + rec := &recordingPublisher{} + m.SetEventPublisher(rec) + ctx := context.Background() + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + registerWeb(t, m, 128, 256) + + ci, err := m.RegisterContainerInstance(ctx, driver.RegisterContainerInstanceInput{ + Cluster: "prod", TotalResources: []driver.Resource{ + {Name: "CPU", Type: "INTEGER", IntegerValue: 1024}, {Name: "MEMORY", Type: "INTEGER", IntegerValue: 2048}, + }, + }) + require.NoError(t, err) + + tasks, _, err := m.RunTask(ctx, driver.RunTaskInput{Cluster: "prod", TaskDefinition: "web"}) + require.NoError(t, err) + + _, err = m.StopTask(ctx, "prod", tasks[0].ARN, "bye") + require.NoError(t, err) + + _, _, err = m.UpdateContainerInstancesState(ctx, "prod", []string{ci.ARN}, statusDraining) + require.NoError(t, err) + + _, err = m.DeregisterContainerInstance(ctx, "prod", ci.ARN, false) + require.NoError(t, err) + + var got []containerInstanceDetail + + for _, e := range eventsOf(rec, eventContainerInstanceStateChange) { + assert.Equal(t, []string{ci.ARN}, e.Resources) + + d, ok := e.Detail.(containerInstanceDetail) + require.True(t, ok) + + got = append(got, d) + } + + require.Len(t, got, 5, "register, task placed, task stopped, draining, deregistered") + + for i, d := range got { + assert.Equal(t, i+1, d.Version, "version increases by one per change") + assert.Equal(t, ci.ARN, d.ContainerInstanceArn) + assert.Equal(t, ci.EC2InstanceID, d.EC2InstanceID) + assert.Equal(t, tasks[0].ClusterARN, d.ClusterArn) + } + + assert.Equal(t, statusActive, got[0].Status) + assert.Equal(t, 1024, ciResourceInt(got[0].RegisteredResources, "CPU")) + assert.Equal(t, 1024, ciResourceInt(got[0].RemainingResources, "CPU")) + assert.Equal(t, 896, ciResourceInt(got[1].RemainingResources, "CPU"), "placing the task reserves 128 CPU") + assert.Equal(t, 1024, ciResourceInt(got[2].RemainingResources, "CPU"), "stopping it gives the CPU back") + assert.Equal(t, statusDraining, got[3].Status) + assert.Equal(t, statusInactive, got[4].Status) +} + +func ciResourceInt(rs []containerInstanceResource, name string) int { + for _, r := range rs { + if r.Name == name { + return r.IntegerValue + } + } + + return -1 +} + +func TestServiceActionEvents(t *testing.T) { + m := newTestMock() + rec := &recordingPublisher{} + m.SetEventPublisher(rec) + ctx := context.Background() + netCfg := fargateFixture(t, m) + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "web", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, DesiredCount: 1, + }) + require.NoError(t, err) + + three := 3 + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", DesiredCount: &three}) + require.NoError(t, err) + + var names []string + + for _, e := range eventsOf(rec, eventServiceAction) { + names = append(names, e.Detail.(serviceActionDetail).EventName) + } + + assert.Contains(t, names, "SERVICE_DESIRED_COUNT_UPDATED") + + // EC2 service on a cluster too small for its task: placement failure. + _, err = m.CreateCluster(ctx, driver.CreateClusterInput{Name: "small"}) + require.NoError(t, err) + registerWeb(t, m, 512, 512) + m.SeedContainerInstance("small", "i-small", WithCapacity(256, 4096)) + + _, err = m.CreateService(ctx, driver.CreateServiceInput{ServiceName: "cpu", Cluster: "small", TaskDefinition: "web", DesiredCount: 1}) + require.NoError(t, err) + + var failure serviceActionDetail + + for _, e := range eventsOf(rec, eventServiceAction) { + if d := e.Detail.(serviceActionDetail); d.EventName == "SERVICE_TASK_PLACEMENT_FAILURE" { + failure = d + } + } + + assert.Equal(t, "SERVICE_TASK_PLACEMENT_FAILURE", failure.EventName) + assert.Equal(t, "ERROR", failure.EventType) + assert.Equal(t, "RESOURCE:CPU", failure.Reason) +} + +// TestEvents_PublishedWithoutLocksHeld re-enters ECS from the event consumer at +// every emit site; an event published while a store, placement, task-set or +// reconcile lock is held would deadlock (the test then times out). +func TestEvents_PublishedWithoutLocksHeld(t *testing.T) { + m := newTestMock() + ctx := context.Background() + netCfg := fargateFixture(t, m) + + // The consumer stops a service task on its first few events only: stopping on + // every event would make the service replace tasks forever. + var stops atomic.Int32 + + rec := &recordingPublisher{} + rec.onEmit = func() { + // Every call below takes locks the emitting operation must have released. + _, _ = m.ListTasks(ctx, "prod", "", "", "") + _, _, _ = m.DescribeServices(ctx, "prod", []string{"web", "ext"}) + _, _, _ = m.DescribeTaskSets(ctx, "prod", "ext", nil) + _, _, _ = m.ListServiceDeployments(ctx, driver.ListServiceDeploymentsInput{Cluster: "prod", Service: "web"}) + + tasks, _ := m.ListTasks(ctx, "prod", "", "RUNNING", "web") + if len(tasks) > 0 && stops.Add(1) <= 3 { + _, _ = m.StopTask(ctx, "prod", tasks[0].ARN, "from-consumer") + } + } + m.SetEventPublisher(rec) + + done := make(chan struct{}) + + go func() { + defer close(done) + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "web", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, DesiredCount: 2, + }) + assert.NoError(t, err) + + two := 1 + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", DesiredCount: &two, ForceNewDeployment: true}) + assert.NoError(t, err) + + _, err = m.CreateService(ctx, driver.CreateServiceInput{ServiceName: "ext", Cluster: "prod", DeploymentController: deployControllerExternal, DesiredCount: 2}) + assert.NoError(t, err) + + ts, err := m.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + }) + assert.NoError(t, err) + + half := driver.Scale{Unit: driver.ScaleUnitPercent, Value: 50} + _, err = m.UpdateTaskSet(ctx, driver.UpdateTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: ts.ID, Scale: half}) + assert.NoError(t, err) + + _, err = m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: ts.ID, Force: true}) + assert.NoError(t, err) + + tasks, _, err := m.RunTask(ctx, driver.RunTaskInput{Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg}) + assert.NoError(t, err) + + if len(tasks) == 1 { + _, err = m.StopTask(ctx, "prod", tasks[0].ARN, "x") + assert.NoError(t, err) + } + + ci, err := m.RegisterContainerInstance(ctx, driver.RegisterContainerInstanceInput{Cluster: "prod"}) + assert.NoError(t, err) + + _, err = m.DeregisterContainerInstance(ctx, "prod", ci.ARN, true) + assert.NoError(t, err) + + _, err = m.DeleteService(ctx, "prod", "web", true) + assert.NoError(t, err) + + _, err = m.DeleteService(ctx, "prod", "ext", true) + assert.NoError(t, err) + }() + + select { + case <-done: + case <-time.After(20 * time.Second): + t.Fatal("deadlock: an event was published while a lock was held") + } + + assert.NotEmpty(t, rec.snapshot()) +} + +func TestEvents_NoPublisherNoPanic(t *testing.T) { + m := newTestMock() + ctx := context.Background() + netCfg := fargateFixture(t, m) + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "web", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, DesiredCount: 1, + }) + require.NoError(t, err) + + _, err = m.RegisterContainerInstance(ctx, driver.RegisterContainerInstanceInput{Cluster: "prod"}) + require.NoError(t, err) + + _, err = m.DeleteService(ctx, "prod", "web", true) + require.NoError(t, err) +} diff --git a/providers/aws/ecs/exec.go b/providers/aws/ecs/exec.go index 2f9c3562f..1a2a4d69d 100644 --- a/providers/aws/ecs/exec.go +++ b/providers/aws/ecs/exec.go @@ -5,29 +5,47 @@ import ( "strings" "github.com/stackshy/cloudemu/v2/errors" - "github.com/stackshy/cloudemu/v2/internal/idgen" "github.com/stackshy/cloudemu/v2/services/container/containerengine" "github.com/stackshy/cloudemu/v2/services/ecs/driver" ) +// execNotEnabledMessage is the text AWS returns when ECS Exec was not enabled +// for the task or its agent is not running. +const execNotEnabledMessage = "The execute command failed because execute command was not enabled when the task was run " + + "or the execute command agent isn't running. Wait and try again or run a new task with execute command enabled and try again." + // ExecuteCommand resolves a running task and returns a synthetic SSM session for -// the target container. When the task is engine-backed the requested command is -// actually run inside the container via the configured ContainerEngine (its -// output travels the SSM data channel in real ECS, so it is not carried in this -// response). An unresolved task surfaces an InvalidParameterException. +// the target container. The task must belong to the cluster, have been started +// with enableExecuteCommand, be RUNNING, and the call must be interactive and +// name an existing container (optional only for a single-container task); every +// other case is an InvalidParameterException. When the task is engine-backed +// the requested command is actually run inside the container via the configured +// ContainerEngine (its output travels the SSM data channel in real ECS, so it +// is not carried in this response). func (m *Mock) ExecuteCommand(ctx context.Context, in driver.ExecuteCommandInput) (*driver.ExecuteCommandResult, error) { - t, ok := m.resolveTask(in.Task) - if !ok { + want := resolveClusterName(in.Cluster) + if !m.clusterExists(want) { + return nil, apiErrf(errors.NotFound, excClusterNotFound, "cluster %q not found", want) + } + + stored, ok := m.resolveTask(in.Task) + if !ok || clusterNameFromARN(stored.ClusterARN) != want { return nil, apiErrf(errors.NotFound, excInvalidParameter, "task %q not found", in.Task) } - containerName := in.Container - if containerName == "" && len(t.Containers) > 0 { - containerName = t.Containers[0].Name + t := m.observedTask(stored) + + if err := validateExec(&t, &in); err != nil { + return nil, err + } + + container, err := execContainer(&t, in.Container) + if err != nil { + return nil, err } if handle, backed := m.taskHandle(t.ARN); backed { - if _, err := containerengine.Exec(ctx, m.opts.ContainerEngine, handle, containerName, execCommand(in.Command)); err != nil { + if _, err := containerengine.Exec(ctx, m.opts.ContainerEngine, handle, container.Name, execCommand(in.Command)); err != nil { return nil, apiErrf(errors.Internal, excInvalidParameter, "execute-command failed: %v", err) } } @@ -35,12 +53,11 @@ func (m *Mock) ExecuteCommand(ctx context.Context, in driver.ExecuteCommandInput region := arnRegion(t.ClusterARN, m.opts.Region) sessionID := "ecs-execute-command-" + m.hexID() streamURL := "wss://ssmmessages." + region + ".amazonaws.com/v1/data-channel/" + sessionID + "?role=publish_subscribe" - containerARN := m.arnIn(region, "container/"+clusterNameFromARN(t.ClusterARN)+"/"+trailingID(t.ARN)+"/"+idgen.GenerateID("")) return &driver.ExecuteCommandResult{ ClusterARN: t.ClusterARN, - ContainerARN: containerARN, - ContainerName: containerName, + ContainerARN: container.ARN, + ContainerName: container.Name, TaskARN: t.ARN, Interactive: in.Interactive, Session: driver.Session{ @@ -51,6 +68,47 @@ func (m *Mock) ExecuteCommand(ctx context.Context, in driver.ExecuteCommandInput }, nil } +// validateExec checks the task-level preconditions of an ExecuteCommand call. +func validateExec(t *driver.Task, in *driver.ExecuteCommandInput) error { + if !t.EnableExecuteCommand { + return apiErrf(errors.InvalidArgument, excInvalidParameter, "%s", execNotEnabledMessage) + } + + if t.LastStatus != statusRunning { + return apiErrf(errors.FailedPrecondition, excInvalidParameter, + "The execute command failed because the task is in %s state; it must be RUNNING.", t.LastStatus) + } + + if !in.Interactive { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "The interactive parameter must be set to true for execute-command.") + } + + return nil +} + +// execContainer picks the container an ExecuteCommand call targets: the named +// one, or the only one when the call names none. +func execContainer(t *driver.Task, name string) (*driver.Container, error) { + if name == "" { + if len(t.Containers) != 1 { + return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "The task has %d containers; specify the container to run the command in.", len(t.Containers)) + } + + return &t.Containers[0], nil + } + + for i := range t.Containers { + if t.Containers[i].Name == name { + return &t.Containers[i], nil + } + } + + return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "The container %q does not exist in the task.", name) +} + // execCommand splits an execute-command command string into the argv the // ContainerEngine expects. An empty command yields a nil argv. func execCommand(cmd string) []string { diff --git a/providers/aws/ecs/exec_enablement_test.go b/providers/aws/ecs/exec_enablement_test.go new file mode 100644 index 000000000..cb9e7444e --- /dev/null +++ b/providers/aws/ecs/exec_enablement_test.go @@ -0,0 +1,208 @@ +package ecs + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +const wantExecNotEnabledMessage = "The execute command failed because execute command was not enabled when the task was run " + + "or the execute command agent isn't running. Wait and try again or run a new task with execute command enabled and try again." + +func runFargate(t *testing.T, m *Mock, netCfg *driver.NetworkConfiguration, enableExec bool) driver.Task { + t.Helper() + + tasks, failures, err := m.RunTask(context.Background(), driver.RunTaskInput{ + Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, + NetworkConfiguration: netCfg, EnableExecuteCommand: enableExec, + }) + require.NoError(t, err) + require.Empty(t, failures) + require.Len(t, tasks, 1) + + return tasks[0] +} + +func TestExecuteCommand_NotEnabled(t *testing.T) { + m := newTestMock() + netCfg := fargateFixture(t, m) + task := runFargate(t, m, netCfg, false) + + _, err := m.ExecuteCommand(context.Background(), driver.ExecuteCommandInput{ + Cluster: "prod", Task: task.ARN, Command: "ls", Interactive: true, + }) + require.Error(t, err) + assert.Equal(t, excInvalidParameter, ecsException(t, err)) + + var ce *errors.Error + + require.ErrorAs(t, err, &ce) + assert.Equal(t, wantExecNotEnabledMessage, ce.Message) +} + +func TestExecuteCommand_TaskNotRunning(t *testing.T) { + m := newTestMock() + netCfg := fargateFixture(t, m) + task := runFargate(t, m, netCfg, true) + + _, err := m.StopTask(context.Background(), "prod", task.ARN, "bye") + require.NoError(t, err) + + _, err = m.ExecuteCommand(context.Background(), driver.ExecuteCommandInput{ + Cluster: "prod", Task: task.ARN, Command: "ls", Interactive: true, + }) + require.Error(t, err) + assert.Equal(t, excInvalidParameter, ecsException(t, err)) +} + +func TestExecuteCommand_ContainerAndInteractiveValidation(t *testing.T) { + m := newTestMock() + netCfg := fargateFixture(t, m) + task := runFargate(t, m, netCfg, true) + ctx := context.Background() + + _, err := m.ExecuteCommand(ctx, driver.ExecuteCommandInput{ + Cluster: "prod", Task: task.ARN, Container: "nope", Command: "ls", Interactive: true, + }) + require.Error(t, err) + assert.Equal(t, excInvalidParameter, ecsException(t, err)) + + _, err = m.ExecuteCommand(ctx, driver.ExecuteCommandInput{ + Cluster: "prod", Task: task.ARN, Command: "ls", Interactive: false, + }) + require.Error(t, err) + assert.Equal(t, excInvalidParameter, ecsException(t, err)) + + _, err = m.ExecuteCommand(ctx, driver.ExecuteCommandInput{ + Cluster: "ghost", Task: task.ARN, Command: "ls", Interactive: true, + }) + require.Error(t, err) + assert.Equal(t, excClusterNotFound, ecsException(t, err)) +} + +func TestExecuteCommand_EnabledRunning(t *testing.T) { + m := newTestMock() + netCfg := fargateFixture(t, m) + task := runFargate(t, m, netCfg, true) + + res, err := m.ExecuteCommand(context.Background(), driver.ExecuteCommandInput{ + Cluster: "prod", Task: task.ARN, Command: "/bin/sh", Interactive: true, + }) + require.NoError(t, err) + assert.Equal(t, task.ARN, res.TaskARN) + assert.Equal(t, "c", res.ContainerName) + assert.Equal(t, task.Containers[0].ARN, res.ContainerARN, "the task's own container ARN, not a fresh one") + assert.NotEmpty(t, res.Session.SessionID) +} + +func TestEnableExecuteCommand_Propagation(t *testing.T) { + m := newTestMock() + netCfg := fargateFixture(t, m) + ctx := context.Background() + + task := runFargate(t, m, netCfg, true) + assert.True(t, task.EnableExecuteCommand) + assert.False(t, runFargate(t, m, netCfg, false).EnableExecuteCommand) + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "on", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, + NetworkConfiguration: netCfg, DesiredCount: 1, EnableExecuteCommand: true, + }) + require.NoError(t, err) + + _, err = m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "off", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, + NetworkConfiguration: netCfg, DesiredCount: 1, + }) + require.NoError(t, err) + + on, err := m.ListTasks(ctx, "prod", "", "", "on") + require.NoError(t, err) + require.Len(t, on, 1) + assert.True(t, on[0].EnableExecuteCommand) + + off, err := m.ListTasks(ctx, "prod", "", "", "off") + require.NoError(t, err) + require.Len(t, off, 1) + assert.False(t, off[0].EnableExecuteCommand) + + // Enabling on the service does not change running tasks; a new deployment does. + enable := true + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "off", Cluster: "prod", EnableExecuteCommand: &enable}) + require.NoError(t, err) + + off, err = m.ListTasks(ctx, "prod", "", "", "off") + require.NoError(t, err) + assert.False(t, off[0].EnableExecuteCommand, "existing task keeps its launch-time setting") + + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "off", Cluster: "prod", ForceNewDeployment: true}) + require.NoError(t, err) + + running, err := m.ListTasks(ctx, "prod", "", "RUNNING", "off") + require.NoError(t, err) + require.Len(t, running, 1) + assert.True(t, running[0].EnableExecuteCommand) +} + +func TestManagedAgents_ExecuteCommandAgent(t *testing.T) { + m := newTestMock() + netCfg := fargateFixture(t, m) + + on := runFargate(t, m, netCfg, true) + require.Len(t, on.Containers[0].ManagedAgents, 1) + assert.Equal(t, "ExecuteCommandAgent", on.Containers[0].ManagedAgents[0].Name) + assert.Equal(t, statusRunning, on.Containers[0].ManagedAgents[0].LastStatus) + assert.NotEmpty(t, on.Containers[0].ManagedAgents[0].LastStartedAt) + + off := runFargate(t, m, netCfg, false) + assert.Empty(t, off.Containers[0].ManagedAgents) + + stopped, err := m.StopTask(context.Background(), "prod", on.ARN, "bye") + require.NoError(t, err) + assert.Equal(t, statusStopped, stopped.Containers[0].ManagedAgents[0].LastStatus) +} + +func TestAwsvpcTask_ContainerNetworkInterfaces(t *testing.T) { + m := newTestMock() + netCfg := fargateFixture(t, m) + task := runFargate(t, m, netCfg, false) + + require.Len(t, task.Attachments, 1) + attachment := task.Attachments[0] + require.NotEmpty(t, attachment.ID) + + var privateIP string + + for _, d := range attachment.Details { + if d.Name == "privateIPv4Address" { + privateIP = d.Value + } + } + + require.NotEmpty(t, privateIP) + + for _, c := range task.Containers { + require.Len(t, c.NetworkInterfaces, 1) + assert.Equal(t, attachment.ID, c.NetworkInterfaces[0].AttachmentID) + assert.Equal(t, privateIP, c.NetworkInterfaces[0].PrivateIPv4Address) + } +} + +func TestBridgeTask_NoNetworkInterfaces(t *testing.T) { + m := newTestMock() + ctx := context.Background() + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + registerWeb(t, m, 128, 128) + + tasks, _, err := m.RunTask(ctx, driver.RunTaskInput{Cluster: "prod", TaskDefinition: "web", LaunchType: launchExternal}) + require.NoError(t, err) + require.Len(t, tasks, 1) + assert.Empty(t, tasks[0].Containers[0].NetworkInterfaces) +} diff --git a/providers/aws/ecs/metrics.go b/providers/aws/ecs/metrics.go new file mode 100644 index 000000000..b603a9f26 --- /dev/null +++ b/providers/aws/ecs/metrics.go @@ -0,0 +1,174 @@ +package ecs + +import ( + "context" + "time" + + "github.com/stackshy/cloudemu/v2/services/ecs/driver" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" +) + +// ECS publishes two metric namespaces. AWS/ECS carries the reservation and +// live-task metrics the service always emits; ECS/ContainerInsights carries the +// cluster and service counts and exists only for clusters with Container +// Insights enabled (cluster setting containerInsights = enabled or enhanced, or +// the account setting when the cluster sets none). +// +// The emulator runs no workload, so it never publishes CPU/memory utilization or +// the other usage-driven series: only metrics derived from state it really holds +// (task and service counts, reserved container-instance capacity). +const ( + metricsNamespaceECS = "AWS/ECS" + metricsNamespaceInsights = "ECS/ContainerInsights" + metricUnitCount = "Count" + metricUnitPercent = "Percent" + percentOf = 100.0 + + settingContainerInsights = "containerInsights" + insightsEnabled = "enabled" + insightsEnhanced = "enhanced" +) + +// SetMonitoring wires the CloudWatch backend that receives the AWS/ECS and +// ECS/ContainerInsights metrics. Safe to leave unset: no metrics are published. +func (m *Mock) SetMonitoring(mon mondriver.Monitoring) { m.monitoring = mon } + +// containerInsightsOn reports whether Container Insights is enabled for the +// cluster: its own containerInsights setting wins, else the account setting. +func (m *Mock) containerInsightsOn(c *driver.Cluster) bool { + value := "" + + if c != nil { + for _, s := range c.Settings { + if s.Name == settingContainerInsights { + value = s.Value + } + } + } + + if value == "" { + if s, ok := m.settings.Get(settingContainerInsights); ok { + value = s.Value + } + } + + return value == insightsEnabled || value == insightsEnhanced +} + +// metricBatch accumulates the datums of one publish. +type metricBatch struct { + data []mondriver.MetricDatum + now time.Time +} + +func (b *metricBatch) add(ns, name, unit string, value float64, dims map[string]string) { + b.data = append(b.data, mondriver.MetricDatum{ + Namespace: ns, MetricName: name, Value: value, Unit: unit, Timestamp: b.now, Dimensions: dims, + }) +} + +// publishClusterMetrics publishes the metrics of one cluster's current state. It +// is called after every mutation, outside every emulator lock, and does nothing +// for an unknown or deleted cluster. +func (m *Mock) publishClusterMetrics(cluster string) { + if m.monitoring == nil { + return + } + + c, stored := m.clusters.Get(cluster) + if (stored && c.Status != statusActive) || (!stored && cluster != defaultCluster) { + return + } + + batch := &metricBatch{now: m.opts.Clock.Now()} + insights := m.containerInsightsOn(c) + + m.addClusterMetrics(batch, cluster, insights) + + for _, svc := range m.services.SortedValues() { + if clusterNameFromARN(svc.ClusterARN) == cluster && svc.Status == statusActive { + m.addServiceMetrics(batch, cluster, svc, insights) + } + } + + if len(batch.data) > 0 { + _ = m.monitoring.PutMetricData(context.Background(), batch.data) + } +} + +// addClusterMetrics adds the cluster-level series: the reservation of its +// container instances and, with Container Insights, the instance, service and +// task counts. +func (m *Mock) addClusterMetrics(b *metricBatch, cluster string, insights bool) { + dims := map[string]string{"ClusterName": cluster} + + if cpuPct, memPct, ok := m.reservationPercent(cluster); ok { + b.add(metricsNamespaceECS, "CPUReservation", metricUnitPercent, cpuPct, dims) + b.add(metricsNamespaceECS, "MemoryReservation", metricUnitPercent, memPct, dims) + } + + if !insights { + return + } + + services, running, _, instances := m.clusterCounts(cluster) + + b.add(metricsNamespaceInsights, "ContainerInstanceCount", metricUnitCount, float64(instances), dims) + b.add(metricsNamespaceInsights, "ServiceCount", metricUnitCount, float64(services), dims) + b.add(metricsNamespaceInsights, "TaskCount", metricUnitCount, float64(running), dims) +} + +// addServiceMetrics adds one service's series: LiveTaskCount while it has +// running tasks and, with Container Insights, its task, deployment and task-set +// counts. +func (m *Mock) addServiceMetrics(b *metricBatch, cluster string, svc *driver.Service, insights bool) { + running, pending := m.liveServiceTaskCounts(cluster, serviceGroup(svc.Name)) + dims := map[string]string{"ClusterName": cluster, "ServiceName": svc.Name} + + if running > 0 { + b.add(metricsNamespaceECS, "LiveTaskCount", metricUnitCount, float64(running), dims) + } + + if !insights { + return + } + + b.add(metricsNamespaceInsights, "DesiredTaskCount", metricUnitCount, float64(svc.DesiredCount), dims) + b.add(metricsNamespaceInsights, "RunningTaskCount", metricUnitCount, float64(running), dims) + b.add(metricsNamespaceInsights, "PendingTaskCount", metricUnitCount, float64(pending), dims) + b.add(metricsNamespaceInsights, "DeploymentCount", metricUnitCount, float64(len(svc.Deployments)), dims) + b.add(metricsNamespaceInsights, "TaskSetCount", metricUnitCount, float64(len(m.taskSetsOf(svc))), dims) +} + +// reservationPercent returns the share of the cluster's registered CPU and +// memory that running tasks have reserved, and whether the cluster has any +// ACTIVE container instance at all (without one there is nothing to reserve). +func (m *Mock) reservationPercent(cluster string) (cpuPct, memPct float64, ok bool) { + var regCPU, remCPU, regMem, remMem int + + for _, ci := range m.instances.All() { + if instanceClusterName(ci.ARN) != cluster || ci.Status != statusActive { + continue + } + + ok = true + regCPU += ci.RegisteredCPU + remCPU += ci.RemainingCPU + regMem += ci.RegisteredMemory + remMem += ci.RemainingMemory + } + + if !ok { + return 0, 0, false + } + + return reservedPercent(regCPU, remCPU), reservedPercent(regMem, remMem), true +} + +func reservedPercent(registered, remaining int) float64 { + if registered <= 0 { + return 0 + } + + return float64(registered-remaining) / float64(registered) * percentOf +} diff --git a/providers/aws/ecs/metrics_test.go b/providers/aws/ecs/metrics_test.go new file mode 100644 index 000000000..7f1f7f64c --- /dev/null +++ b/providers/aws/ecs/metrics_test.go @@ -0,0 +1,301 @@ +package ecs + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/providers/aws/cloudwatch" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" +) + +const ( + insightsNS = "ECS/ContainerInsights" + awsECSNS = "AWS/ECS" +) + +// metricsMock builds an ECS mock wired to a real CloudWatch mock on one FakeClock. +func metricsMock(t *testing.T) (*Mock, *cloudwatch.Mock, *config.FakeClock) { + t.Helper() + + clock := config.NewFakeClock(time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)) + opts := config.NewOptions(config.WithClock(clock), config.WithRegion("us-east-1")) + cw := cloudwatch.New(opts) + m := New(opts) + m.SetMonitoring(cw) + + return m, cw, clock +} + +// lastValue returns the most recent value of a metric series, or false when +// the series has no data. +func lastValue(t *testing.T, cw *cloudwatch.Mock, clock *config.FakeClock, ns, name string, dims map[string]string) (float64, bool) { + t.Helper() + + res, err := cw.GetMetricData(context.Background(), mondriver.GetMetricInput{ + Namespace: ns, MetricName: name, Dimensions: dims, + StartTime: clock.Now().Add(-24 * time.Hour), EndTime: clock.Now().Add(time.Hour), Period: 60, Stat: "Average", + }) + require.NoError(t, err) + + if len(res.Values) == 0 { + return 0, false + } + + return res.Values[len(res.Values)-1], true +} + +func metricNames(t *testing.T, cw *cloudwatch.Mock, ns string) []string { + t.Helper() + + names, err := cw.ListMetrics(context.Background(), ns) + require.NoError(t, err) + + return names +} + +func insightsCluster(t *testing.T, m *Mock, name, value string) { + t.Helper() + + _, err := m.CreateCluster(context.Background(), driver.CreateClusterInput{ + Name: name, Settings: []driver.Setting{{Name: "containerInsights", Value: value}}, + }) + require.NoError(t, err) +} + +func TestContainerInsights_ClusterAndServiceCounts(t *testing.T) { + m, cw, clock := metricsMock(t) + ctx := context.Background() + + insightsCluster(t, m, "prod", "enabled") + registerWeb(t, m, 128, 256) + m.SeedContainerInstance("prod", "i-1", WithCapacity(2048, 4096)) + + clock.Advance(time.Minute) + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ServiceName: "web", Cluster: "prod", TaskDefinition: "web", DesiredCount: 2}) + require.NoError(t, err) + + cluster := map[string]string{"ClusterName": "prod"} + service := map[string]string{"ClusterName": "prod", "ServiceName": "web"} + + check := func(ns, name string, dims map[string]string, want float64) { + t.Helper() + + got, ok := lastValue(t, cw, clock, ns, name, dims) + require.True(t, ok, "%s has data", name) + assert.InDelta(t, want, got, 0.001, name) + } + + check(insightsNS, "ContainerInstanceCount", cluster, 1) + check(insightsNS, "ServiceCount", cluster, 1) + check(insightsNS, "TaskCount", cluster, 2) + check(insightsNS, "DesiredTaskCount", service, 2) + check(insightsNS, "RunningTaskCount", service, 2) + check(insightsNS, "PendingTaskCount", service, 0) + check(insightsNS, "DeploymentCount", service, 1) + check(insightsNS, "TaskSetCount", service, 0) + + clock.Advance(time.Minute) + + one := 1 + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", DesiredCount: &one}) + require.NoError(t, err) + + check(insightsNS, "DesiredTaskCount", service, 1) + check(insightsNS, "RunningTaskCount", service, 1) + check(insightsNS, "TaskCount", cluster, 1) + + clock.Advance(time.Minute) + + _, err = m.DeleteService(ctx, "prod", "web", true) + require.NoError(t, err) + + check(insightsNS, "ServiceCount", cluster, 0) + check(insightsNS, "TaskCount", cluster, 0) +} + +func TestContainerInsights_TaskSetCount(t *testing.T) { + m, cw, clock := metricsMock(t) + ctx := context.Background() + + insightsCluster(t, m, "prod", "enhanced") + netCfg := fargateFixtureNoCluster(t, m) + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "ext", Cluster: "prod", DeploymentController: deployControllerExternal, DesiredCount: 2, + }) + require.NoError(t, err) + + clock.Advance(time.Minute) + + _, err = m.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + }) + require.NoError(t, err) + + got, ok := lastValue(t, cw, clock, insightsNS, "TaskSetCount", map[string]string{"ClusterName": "prod", "ServiceName": "ext"}) + require.True(t, ok) + assert.InDelta(t, 1, got, 0.001) +} + +func TestContainerInsights_OnlyWhenEnabled(t *testing.T) { + m, cw, clock := metricsMock(t) + ctx := context.Background() + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + registerWeb(t, m, 128, 256) + m.SeedContainerInstance("prod", "i-1", WithCapacity(2048, 4096)) + + _, _, err = m.RunTask(ctx, driver.RunTaskInput{Cluster: "prod", TaskDefinition: "web"}) + require.NoError(t, err) + assert.Empty(t, metricNames(t, cw, insightsNS), "no cluster setting and no account setting: Container Insights is off") + + // Turning it on publishes immediately. + clock.Advance(time.Minute) + + _, err = m.UpdateClusterSettings(ctx, "prod", []driver.Setting{{Name: "containerInsights", Value: "enabled"}}) + require.NoError(t, err) + + got, ok := lastValue(t, cw, clock, insightsNS, "TaskCount", map[string]string{"ClusterName": "prod"}) + require.True(t, ok) + assert.InDelta(t, 1, got, 0.001) + + // Turning it off stops publishing. + clock.Advance(time.Minute) + + _, err = m.UpdateClusterSettings(ctx, "prod", []driver.Setting{{Name: "containerInsights", Value: "disabled"}}) + require.NoError(t, err) + + clock.Advance(time.Minute) + + _, _, err = m.RunTask(ctx, driver.RunTaskInput{Cluster: "prod", TaskDefinition: "web"}) + require.NoError(t, err) + + got, ok = lastValue(t, cw, clock, insightsNS, "TaskCount", map[string]string{"ClusterName": "prod"}) + require.True(t, ok) + assert.InDelta(t, 1, got, 0.001, "the series stopped at the last enabled sample") + + // The account setting turns it on for clusters that set nothing themselves. + _, err = m.CreateCluster(ctx, driver.CreateClusterInput{Name: "inherit"}) + require.NoError(t, err) + _, err = m.PutAccountSetting(ctx, "containerInsights", "enabled") + require.NoError(t, err) + + clock.Advance(time.Minute) + + _, err = m.UpdateCluster(ctx, driver.UpdateClusterInput{Cluster: "inherit"}) + require.NoError(t, err) + + _, ok = lastValue(t, cw, clock, insightsNS, "ServiceCount", map[string]string{"ClusterName": "inherit"}) + assert.True(t, ok, "account-level containerInsights applies to a cluster with no setting of its own") +} + +func TestAWSECS_ReservationAndLiveTaskCount(t *testing.T) { + m, cw, clock := metricsMock(t) + ctx := context.Background() + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + + _, err = m.RegisterTaskDefinition(ctx, driver.RegisterTaskDefinitionInput{ + Family: "web", CPU: "250", Memory: "500", ContainerDefinitions: []driver.ContainerDefinition{{Name: "c", Image: "img"}}, + }) + require.NoError(t, err) + m.SeedContainerInstance("prod", "i-1", WithCapacity(1000, 2000)) + + clock.Advance(time.Minute) + + _, err = m.CreateService(ctx, driver.CreateServiceInput{ServiceName: "web", Cluster: "prod", TaskDefinition: "web", DesiredCount: 1}) + require.NoError(t, err) + + cpu, ok := lastValue(t, cw, clock, awsECSNS, "CPUReservation", map[string]string{"ClusterName": "prod"}) + require.True(t, ok) + assert.InDelta(t, 25, cpu, 0.001) + + mem, ok := lastValue(t, cw, clock, awsECSNS, "MemoryReservation", map[string]string{"ClusterName": "prod"}) + require.True(t, ok) + assert.InDelta(t, 25, mem, 0.001) + + live, ok := lastValue(t, cw, clock, awsECSNS, "LiveTaskCount", map[string]string{"ClusterName": "prod", "ServiceName": "web"}) + require.True(t, ok) + assert.InDelta(t, 1, live, 0.001) + + for _, name := range metricNames(t, cw, awsECSNS) { + assert.NotContains(t, []string{"CPUUtilization", "MemoryUtilization"}, name, + "no workload runs, so no utilization is fabricated") + } +} + +func TestMetrics_EmptyClusterAndNoReservation(t *testing.T) { + m, cw, clock := metricsMock(t) + + insightsCluster(t, m, "prod", "enabled") + + for _, name := range []string{"ContainerInstanceCount", "ServiceCount", "TaskCount"} { + got, ok := lastValue(t, cw, clock, insightsNS, name, map[string]string{"ClusterName": "prod"}) + require.True(t, ok, name) + assert.InDelta(t, 0, got, 0.001, name) + } + + assert.NotContains(t, metricNames(t, cw, awsECSNS), "CPUReservation", "no instances, no reservation series") +} + +// fargateFixtureNoCluster registers the Fargate task definition (the cluster +// already exists) and returns the awsvpc network configuration. +func fargateFixtureNoCluster(t *testing.T, m *Mock) *driver.NetworkConfiguration { + t.Helper() + + _, err := m.RegisterTaskDefinition(context.Background(), driver.RegisterTaskDefinitionInput{ + Family: "fg", + ContainerDefinitions: []driver.ContainerDefinition{{Name: "c", Image: "img", Essential: true}}, + CPU: "256", + Memory: "512", + NetworkMode: networkModeAwsvpc, + RequiresCompatibilities: []string{launchFargate}, + }) + require.NoError(t, err) + + return &driver.NetworkConfiguration{AwsVpcConfiguration: &driver.AwsVpcConfiguration{Subnets: []string{"subnet-1"}}} +} + +// TestMetrics_ConcurrentCreateServices creates services of one cluster in +// parallel with metrics wired: the metric publisher reads every service of the +// cluster while other requests are still converging theirs (a -race regression +// test for the stored name claim being mutated in place). +func TestMetrics_ConcurrentCreateServices(t *testing.T) { + m, _, _ := metricsMock(t) + ctx := context.Background() + + insightsCluster(t, m, "prod", "enabled") + netCfg := fargateFixtureNoCluster(t, m) + + var wg sync.WaitGroup + + for i := range 20 { + wg.Add(1) + + go func() { + defer wg.Done() + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "svc" + string(rune('a'+i)), Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, + NetworkConfiguration: netCfg, DesiredCount: 2, + }) + assert.NoError(t, err) + }() + } + + wg.Wait() + + services, err := m.ListServices(ctx, "prod") + require.NoError(t, err) + assert.Len(t, services, 20) +} diff --git a/providers/aws/ecs/namespace.go b/providers/aws/ecs/namespace.go new file mode 100644 index 000000000..3cd3530c0 --- /dev/null +++ b/providers/aws/ecs/namespace.go @@ -0,0 +1,60 @@ +package ecs + +import ( + "context" + "slices" + "strings" + + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/pagination" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +const ( + defaultNamespaceListResults = 10 + maxNamespaceListResults = 100 +) + +// Compile-time check that Mock implements the ServiceNamespaces capability. +var _ driver.ServiceNamespaces = (*Mock)(nil) + +// ListServicesByNamespace lists the ARNs of the services whose Service Connect +// configuration names the namespace, across all clusters, sorted by ARN and +// paged with maxResults (1-100, default 10) and nextToken. A service is matched +// by the namespace string it was configured with (name or ARN); the emulator has +// no Cloud Map to resolve one to the other, so an unknown namespace lists +// nothing rather than failing. +func (m *Mock) ListServicesByNamespace( + _ context.Context, namespace string, maxResults int, nextToken string, +) (serviceARNs []string, next string, err error) { + if namespace == "" { + return nil, "", apiErrf(errors.InvalidArgument, excInvalidParameter, "namespace is required.") + } + + if maxResults < 0 || maxResults > maxNamespaceListResults { + return nil, "", apiErrf(errors.InvalidArgument, excInvalidParameter, + "maxResults must be between 1 and %d.", maxNamespaceListResults) + } + + var arns []string + + for _, svc := range m.services.All() { + if svc.Status != statusInactive && svc.ServiceConnect != nil && svc.ServiceConnect.Namespace == namespace { + arns = append(arns, svc.ARN) + } + } + + slices.SortFunc(arns, strings.Compare) + + size := maxResults + if size == 0 { + size = defaultNamespaceListResults + } + + page, perr := pagination.Paginate(arns, nextToken, size) + if perr != nil { + return nil, "", apiErrf(errors.InvalidArgument, excInvalidParameter, "invalid nextToken: %v", perr) + } + + return page.Items, page.NextPageToken, nil +} diff --git a/providers/aws/ecs/nitin_review_test.go b/providers/aws/ecs/nitin_review_test.go index 8ffe16ba1..b000d750d 100644 --- a/providers/aws/ecs/nitin_review_test.go +++ b/providers/aws/ecs/nitin_review_test.go @@ -119,7 +119,7 @@ func TestRunTaskRejectsDeletedCluster(t *testing.T) { } // A deleted cluster's name can be recreated (the INACTIVE tombstone is not a -// permanent AlreadyExists), while an ACTIVE cluster of the same name still is. +// permanent conflict), while re-creating an ACTIVE cluster is idempotent. func TestCreateClusterReusesDeletedName(t *testing.T) { m := newTestMock() ctx := context.Background() @@ -127,10 +127,10 @@ func TestCreateClusterReusesDeletedName(t *testing.T) { _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) require.NoError(t, err) - // Re-creating an ACTIVE cluster of the same name is a conflict. - _, err = m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) - require.Error(t, err) - assert.True(t, errors.IsAlreadyExists(err), "got %v", err) + // Re-creating an ACTIVE cluster of the same name returns it (idempotent). + again, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + assert.Equal(t, statusActive, again.Status) // After delete, the name is free to reuse and the new cluster is ACTIVE. _, err = m.DeleteCluster(ctx, "prod") diff --git a/providers/aws/ecs/parity_helpers_test.go b/providers/aws/ecs/parity_helpers_test.go new file mode 100644 index 000000000..f11361fd1 --- /dev/null +++ b/providers/aws/ecs/parity_helpers_test.go @@ -0,0 +1,85 @@ +package ecs + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// publishedEvent is one event captured by recordingPublisher. +type publishedEvent struct { + DetailType string + Detail any + Resources []string +} + +// recordingPublisher is a thread-safe awsevents.Publisher that records every +// published event in order. +type recordingPublisher struct { + mu sync.Mutex + events []publishedEvent + onEmit func() +} + +func (r *recordingPublisher) PublishServiceEvent(_ context.Context, _, detailType string, detail any, resources []string) { + r.mu.Lock() + r.events = append(r.events, publishedEvent{DetailType: detailType, Detail: detail, Resources: resources}) + hook := r.onEmit + r.mu.Unlock() + + if hook != nil { + hook() + } +} + +func (r *recordingPublisher) snapshot() []publishedEvent { + r.mu.Lock() + defer r.mu.Unlock() + + return append([]publishedEvent(nil), r.events...) +} + +// fargateFixture creates cluster "prod" and the Fargate task definition "fg" +// and returns the awsvpc network configuration RunTask/CreateService need. +func fargateFixture(t *testing.T, m *Mock) *driver.NetworkConfiguration { + t.Helper() + + ctx := context.Background() + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + + _, err = m.RegisterTaskDefinition(ctx, driver.RegisterTaskDefinitionInput{ + Family: "fg", + ContainerDefinitions: []driver.ContainerDefinition{{Name: "c", Image: "img", Essential: true}}, + CPU: "256", + Memory: "512", + NetworkMode: networkModeAwsvpc, + RequiresCompatibilities: []string{launchFargate}, + }) + require.NoError(t, err) + + return &driver.NetworkConfiguration{ + AwsVpcConfiguration: &driver.AwsVpcConfiguration{Subnets: []string{"subnet-1"}}, + } +} + +// instanceRemaining returns the remaining CPU and memory of the single +// container instance registered in cluster. +func (m *Mock) instanceRemaining(t *testing.T, cluster string) [2]int { + t.Helper() + + for _, ci := range m.instances.All() { + if instanceClusterName(ci.ARN) == cluster { + return [2]int{ci.RemainingCPU, ci.RemainingMemory} + } + } + + t.Fatalf("no container instance in cluster %q", cluster) + + return [2]int{} +} diff --git a/providers/aws/ecs/service_deployments.go b/providers/aws/ecs/service_deployments.go new file mode 100644 index 000000000..86d1c5088 --- /dev/null +++ b/providers/aws/ecs/service_deployments.go @@ -0,0 +1,499 @@ +package ecs + +import ( + "context" + "slices" + "strconv" + "time" + + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/pagination" + "github.com/stackshy/cloudemu/v2/internal/settle" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// Limits from the ECS service-deployment API reference, and the retained +// history per service (ECS keeps 90 days; the emulator keeps the newest 100). +const ( + maxServiceDeployments = 100 + maxDeploymentDescribeARNs = 20 + defaultDeploymentListResults = 20 + maxDeploymentListResults = 100 + deploymentIDDigits = 15 + idBaseHex = 16 + + rollbackReasonUser = "Service deployment rolled back by user." +) + +// Compile-time check that Mock implements the ServiceDeployments capability. +var _ driver.ServiceDeployments = (*Mock)(nil) + +// numericID mints a decimal id from random hex, the shape of ECS revision ids. +func (m *Mock) numericID() string { + for { + if n, err := strconv.ParseUint(m.hexID()[:deploymentIDDigits], idBaseHex, 64); err == nil { + return strconv.FormatUint(n, 10) + } + } +} + +// recordServiceDeployment records the service deployment (and the service +// revision it rolls out) that a committed CreateService or deploying +// UpdateService started. It is a no-op when the service was deleted meanwhile, +// so a deployment is never recorded for an inactive service. +func (m *Mock) recordServiceDeployment(svc *driver.Service) { + if svc.DeploymentController == deployControllerExternal { + return + } + + cluster := clusterNameFromARN(svc.ClusterARN) + key := serviceKey(cluster, svc.Name) + + unlock := m.deploymentLock.lock(key) + defer unlock() + + if current, ok := m.services.Get(key); !ok || current.Status != statusActive { + return + } + + m.addDeploymentLocked(svc) +} + +// addDeploymentLocked appends a deployment+revision for svc and trims the +// history. The caller holds the service's deploymentLock. +func (m *Mock) addDeploymentLocked(svc *driver.Service) { + prior := m.deploymentsOf(svc) + now := m.now() + region := arnRegion(svc.ARN, m.opts.Region) + cluster := clusterNameFromARN(svc.ClusterARN) + + rev := &driver.ServiceRevision{ + ARN: m.arnIn(region, "service-revision/"+cluster+"/"+svc.Name+"/"+m.numericID()), + ServiceARN: svc.ARN, + ClusterARN: svc.ClusterARN, + TaskDefinition: svc.TaskDefinition, + LaunchType: svc.LaunchType, + PlatformVersion: svc.PlatformVersion, + CapacityProviderStrategy: append([]driver.CapacityProviderStrategyItem(nil), svc.CapacityProviderStrategy...), + NetworkConfiguration: cloneNetworkConfig(svc.NetworkConfiguration), + LoadBalancers: append([]driver.LoadBalancer(nil), svc.LoadBalancers...), + ServiceRegistries: append([]driver.ServiceRegistry(nil), svc.ServiceRegistries...), + ServiceConnect: cloneServiceConnect(svc.ServiceConnect), + CreatedAt: now, + } + m.serviceRevisions.Set(rev.ARN, rev) + + dep := &driver.ServiceDeployment{ + Sequence: nextDeploymentSequence(prior), + ARN: m.arnIn(region, "service-deployment/"+cluster+"/"+svc.Name+"/"+m.hexID()[:21]), + ServiceARN: svc.ARN, + ClusterARN: svc.ClusterARN, + Status: driver.DeploymentStatusSuccessful, + TargetServiceRevision: driver.ServiceRevisionSummary{ + ARN: rev.ARN, RequestedTaskCount: svc.DesiredCount, + RunningTaskCount: svc.RunningCount, PendingTaskCount: svc.PendingCount, + }, + DeploymentConfiguration: cloneDeploymentConfig(svc.DeploymentConfiguration), + CreatedAt: now, + StartedAt: now, + FinishedAt: now, + UpdatedAt: now, + } + + if len(prior) > 0 { + dep.SourceServiceRevisions = []driver.ServiceRevisionSummary{{ARN: prior[0].TargetServiceRevision.ARN}} + } + + if svc.RunningCount < svc.DesiredCount { + dep.Status, dep.FinishedAt = driver.DeploymentStatusInProgress, "" + } + + m.serviceDeployments.Set(dep.ARN, dep) + + if dep.Status == driver.DeploymentStatusSuccessful { + m.deploymentSettle.Begin(dep.ARN, driver.DeploymentStatusInProgress, m.opts.Clock.Now(), + m.opts.SettleDuration(settle.DefaultECSDeploymentSettle)) + } + + m.trimDeploymentsLocked(svc) +} + +func nextDeploymentSequence(prior []*driver.ServiceDeployment) uint64 { + if len(prior) == 0 { + return 1 + } + + return prior[0].Sequence + 1 +} + +// deploymentsOf returns the stored deployments of svc, newest first. +func (m *Mock) deploymentsOf(svc *driver.Service) []*driver.ServiceDeployment { + var out []*driver.ServiceDeployment + + for _, d := range m.serviceDeployments.All() { + if d.ServiceARN == svc.ARN { + out = append(out, d) + } + } + + slices.SortFunc(out, func(a, b *driver.ServiceDeployment) int { + switch { + case a.Sequence > b.Sequence: + return -1 + case a.Sequence < b.Sequence: + return 1 + default: + return 0 + } + }) + + return out +} + +// trimDeploymentsLocked drops the oldest deployments (and revisions no retained +// deployment references) beyond the retained history. +func (m *Mock) trimDeploymentsLocked(svc *driver.Service) { + deps := m.deploymentsOf(svc) + if len(deps) <= maxServiceDeployments { + return + } + + for _, old := range deps[maxServiceDeployments:] { + m.serviceDeployments.Delete(old.ARN) + m.deploymentSettle.Clear(old.ARN) + } + + keep := make(map[string]bool, len(deps)) + + for _, d := range deps[:maxServiceDeployments] { + keep[d.TargetServiceRevision.ARN] = true + + for _, src := range d.SourceServiceRevisions { + keep[src.ARN] = true + } + } + + for arn, rev := range m.serviceRevisions.All() { + if rev.ServiceARN == svc.ARN && !keep[arn] { + m.serviceRevisions.Delete(arn) + } + } +} + +// deleteServiceDeployments removes a deleted service's history. It takes the +// service's deploymentLock, so it serializes with a concurrent record. +func (m *Mock) deleteServiceDeployments(svc *driver.Service) { + unlock := m.deploymentLock.lock(serviceKey(clusterNameFromARN(svc.ClusterARN), svc.Name)) + defer unlock() + + for _, d := range m.deploymentsOf(svc) { + m.serviceDeployments.Delete(d.ARN) + m.deploymentSettle.Clear(d.ARN) + } + + for arn, rev := range m.serviceRevisions.All() { + if rev.ServiceARN == svc.ARN { + m.serviceRevisions.Delete(arn) + } + } +} + +// ensureServiceDeployment gives an ECS-controller service that has no recorded +// deployment (state restored from before deployments were recorded) its current +// one, once. The caller holds the service's deploymentLock. +func (m *Mock) ensureServiceDeployment(svc *driver.Service) { + if svc.DeploymentController == deployControllerExternal || svc.Status != statusActive || len(m.deploymentsOf(svc)) > 0 { + return + } + + m.addDeploymentLocked(svc) +} + +// observedDeployment copies d with its wire-visible status: the settle overlay +// while a transient window is open, in which case finishedAt/stoppedAt are not +// yet set. +func (m *Mock) observedDeployment(d *driver.ServiceDeployment) driver.ServiceDeployment { + out := cloneServiceDeployment(d) + + if observed := m.deploymentSettle.State(d.ARN, m.opts.Clock.Now(), d.Status); observed != d.Status { + out.Status = observed + out.FinishedAt, out.StoppedAt = "", "" + } + + return out +} + +// deploymentService resolves the cluster and service a list call names. +func (m *Mock) deploymentService(cluster, service string) (*driver.Service, error) { + want := resolveClusterName(cluster) + if !m.clusterExists(want) { + return nil, apiErrf(errors.NotFound, excClusterNotFound, "cluster %q not found", want) + } + + svc, ok := m.resolveService(want, service) + if !ok { + return nil, apiErrf(errors.NotFound, excServiceNotFound, "service %q not found", service) + } + + return svc, nil +} + +// ListServiceDeployments lists a service's deployments newest first, filtered by +// status and creation time and paged with maxResults (1-100, default 20) and +// nextToken. +// +//nolint:gocritic // in is passed by value to satisfy the driver.ServiceDeployments interface. +func (m *Mock) ListServiceDeployments( + _ context.Context, in driver.ListServiceDeploymentsInput, +) ([]driver.ServiceDeployment, string, error) { + svc, err := m.deploymentService(in.Cluster, in.Service) + if err != nil { + return nil, "", err + } + + if in.MaxResults < 0 || in.MaxResults > maxDeploymentListResults { + return nil, "", apiErrf(errors.InvalidArgument, excInvalidParameter, + "maxResults must be between 1 and %d.", maxDeploymentListResults) + } + + before, err := parseOptionalTime(in.CreatedAtBefore) + if err != nil { + return nil, "", err + } + + after, err := parseOptionalTime(in.CreatedAtAfter) + if err != nil { + return nil, "", err + } + + stored := m.ensuredDeployments(svc) + + matched := make([]driver.ServiceDeployment, 0, len(stored)) + + for _, d := range stored { + view := m.observedDeployment(d) + if deploymentMatches(&view, in.Statuses, before, after) { + matched = append(matched, view) + } + } + + size := in.MaxResults + if size == 0 { + size = defaultDeploymentListResults + } + + page, err := pagination.Paginate(matched, in.NextToken, size) + if err != nil { + return nil, "", apiErrf(errors.InvalidArgument, excInvalidParameter, "invalid nextToken: %v", err) + } + + return page.Items, page.NextPageToken, nil +} + +// ensuredDeployments returns svc's deployments newest first, synthesizing the +// current one first when state restored from before deployments were recorded +// has none. +func (m *Mock) ensuredDeployments(svc *driver.Service) []*driver.ServiceDeployment { + unlock := m.deploymentLock.lock(serviceKey(clusterNameFromARN(svc.ClusterARN), svc.Name)) + defer unlock() + + m.ensureServiceDeployment(svc) + + return m.deploymentsOf(svc) +} + +func parseOptionalTime(v string) (time.Time, error) { + if v == "" { + return time.Time{}, nil + } + + t, err := time.Parse(time.RFC3339, v) + if err != nil { + return time.Time{}, apiErrf(errors.InvalidArgument, excInvalidParameter, "%q is not an RFC3339 timestamp.", v) + } + + return t, nil +} + +// deploymentMatches applies the list filters: status membership and a creation +// time strictly after/before the given instants. +func deploymentMatches(d *driver.ServiceDeployment, statuses []string, before, after time.Time) bool { + if len(statuses) > 0 && !slices.Contains(statuses, d.Status) { + return false + } + + created, err := time.Parse(time.RFC3339, d.CreatedAt) + if err != nil { + return false + } + + if !after.IsZero() && !created.After(after) { + return false + } + + return before.IsZero() || created.Before(before) +} + +func validateARNBatch(arns []string) error { + if len(arns) == 0 || len(arns) > maxDeploymentDescribeARNs { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "ARNs must contain between 1 and %d entries.", maxDeploymentDescribeARNs) + } + + return nil +} + +// DescribeServiceDeployments describes up to 20 deployments by ARN; an ARN that +// matches no deployment is a MISSING failure. +func (m *Mock) DescribeServiceDeployments(_ context.Context, arns []string) ([]driver.ServiceDeployment, []driver.Failure, error) { + if err := validateARNBatch(arns); err != nil { + return nil, nil, err + } + + out := make([]driver.ServiceDeployment, 0, len(arns)) + failures := make([]driver.Failure, 0, len(arns)) + + for _, arn := range arns { + d, ok := m.serviceDeployments.Get(arn) + if !ok { + failures = append(failures, driver.Failure{ARN: arn, Reason: failureMissing}) + continue + } + + out = append(out, m.observedDeployment(d)) + } + + return out, failures, nil +} + +// DescribeServiceRevisions describes up to 20 service revisions by ARN. +func (m *Mock) DescribeServiceRevisions(_ context.Context, arns []string) ([]driver.ServiceRevision, []driver.Failure, error) { + if err := validateARNBatch(arns); err != nil { + return nil, nil, err + } + + out := make([]driver.ServiceRevision, 0, len(arns)) + failures := make([]driver.Failure, 0, len(arns)) + + for _, arn := range arns { + r, ok := m.serviceRevisions.Get(arn) + if !ok { + failures = append(failures, driver.Failure{ARN: arn, Reason: failureMissing}) + continue + } + + out = append(out, cloneServiceRevision(r)) + } + + return out, failures, nil +} + +// StopServiceDeployment stops a deployment that has not completed. ROLLBACK +// rolls the service back to the deployment's source revision and ends +// ROLLBACK_SUCCESSFUL; ABORT ends STOPPED and leaves the service as it is. With +// --async-settle the transient ROLLBACK_IN_PROGRESS / STOP_REQUESTED status is +// visible first and a repeated stop continues as-is; a completed deployment is a +// ConflictException. +func (m *Mock) StopServiceDeployment(ctx context.Context, arn, stopType string) (string, error) { + if stopType != driver.StopTypeAbort && stopType != driver.StopTypeRollback { + return "", apiErrf(errors.InvalidArgument, excInvalidParameter, + "stopType must be %s or %s.", driver.StopTypeAbort, driver.StopTypeRollback) + } + + stored, ok := m.serviceDeployments.Get(arn) + if !ok { + return "", apiErrf(errors.NotFound, excServiceDeploymentNotFound, "The service deployment %q was not found.", arn) + } + + svcKey := serviceKey(clusterNameFromARN(stored.ClusterARN), serviceNameOf(stored.ServiceARN)) + + revert, err := m.stopDeploymentLocked(arn, stopType, svcKey) + if err != nil { + return "", err + } + + if revert != nil { + m.revertService(ctx, svcKey, revert) + } + + return arn, nil +} + +// stopDeploymentLocked flips the deployment's status under the service's +// deploymentLock and returns the revision to roll the service back to (nil for +// ABORT or a repeated stop). +func (m *Mock) stopDeploymentLocked(arn, stopType, svcKey string) (*driver.ServiceRevision, error) { + unlock := m.deploymentLock.lock(svcKey) + defer unlock() + + stored, ok := m.serviceDeployments.Get(arn) + if !ok { + return nil, apiErrf(errors.NotFound, excServiceDeploymentNotFound, "The service deployment %q was not found.", arn) + } + + switch m.deploymentSettle.State(arn, m.opts.Clock.Now(), stored.Status) { + case driver.DeploymentStatusStopRequested, driver.DeploymentStatusRollbackRequested, driver.DeploymentStatusRollbackInProgress: + return nil, nil // already stopping: continues as-is + case driver.DeploymentStatusPending, driver.DeploymentStatusInProgress: + default: + return nil, apiErrf(errors.FailedPrecondition, excConflict, + "The service deployment has already completed and can't be stopped.") + } + + updated := cloneServiceDeployment(stored) + now := m.now() + updated.UpdatedAt = now + + var ( + revert *driver.ServiceRevision + transient string + ) + + if stopType == driver.StopTypeRollback { + if len(updated.SourceServiceRevisions) == 0 { + return nil, apiErrf(errors.FailedPrecondition, excConflict, + "The service deployment has no previous service revision to roll back to.") + } + + source := updated.SourceServiceRevisions[0].ARN + + rev, found := m.serviceRevisions.Get(source) + if !found { + return nil, apiErrf(errors.FailedPrecondition, excConflict, "The previous service revision is no longer available.") + } + + copied := cloneServiceRevision(rev) + revert = &copied + updated.Status, updated.FinishedAt = driver.DeploymentStatusRollbackSuccessful, now + updated.Rollback = &driver.ServiceDeploymentRollback{Reason: rollbackReasonUser, ServiceRevisionARN: source, StartedAt: now} + transient = driver.DeploymentStatusRollbackInProgress + } else { + updated.Status, updated.StoppedAt = driver.DeploymentStatusStopped, now + transient = driver.DeploymentStatusStopRequested + } + + m.serviceDeployments.Set(arn, &updated) + m.deploymentSettle.Begin(arn, transient, m.opts.Clock.Now(), m.opts.SettleDuration(settle.DefaultECSDeploymentSettle)) + + return revert, nil +} + +// revertService puts the service back on a previous revision's task definition +// and network configuration and redeploys its tasks. It does not record a new +// deployment: the rollback belongs to the deployment that was stopped. +func (m *Mock) revertService(ctx context.Context, svcKey string, rev *driver.ServiceRevision) { + svc, ok := m.services.Get(svcKey) + if !ok || svc.Status != statusActive { + return + } + + updated := cloneService(svc) + updated.TaskDefinition = rev.TaskDefinition + updated.NetworkConfiguration = cloneNetworkConfig(rev.NetworkConfiguration) + + var events pendingTaskEvents + + m.redeployService(ctx, &updated, &driver.UpdateServiceInput{}, &events) + m.services.Set(svcKey, &updated) + m.publish(ctx, &events) +} diff --git a/providers/aws/ecs/service_deployments_test.go b/providers/aws/ecs/service_deployments_test.go new file mode 100644 index 000000000..14424157f --- /dev/null +++ b/providers/aws/ecs/service_deployments_test.go @@ -0,0 +1,533 @@ +package ecs + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// deploymentFixture creates cluster "prod", Fargate task definitions fg:1 and +// fg:2, and an ECS-controller service "web" with no tasks (desired 0) so the +// deployment history can be exercised without placing tasks. +func deploymentFixture(t *testing.T, m *Mock) *driver.NetworkConfiguration { + t.Helper() + + netCfg := fargateFixture(t, m) + + _, err := m.RegisterTaskDefinition(context.Background(), driver.RegisterTaskDefinitionInput{ + Family: "fg", + ContainerDefinitions: []driver.ContainerDefinition{{Name: "c", Image: "img2", Essential: true}}, + CPU: "256", + Memory: "512", + NetworkMode: networkModeAwsvpc, + RequiresCompatibilities: []string{launchFargate}, + }) + require.NoError(t, err) + + _, err = m.CreateService(context.Background(), driver.CreateServiceInput{ + ServiceName: "web", Cluster: "prod", TaskDefinition: "fg:1", LaunchType: launchFargate, NetworkConfiguration: netCfg, + }) + require.NoError(t, err) + + return netCfg +} + +func listDeployments(t *testing.T, m *Mock, in driver.ListServiceDeploymentsInput) ([]driver.ServiceDeployment, string) { + t.Helper() + + in.Cluster, in.Service = "prod", "web" + + out, next, err := m.ListServiceDeployments(context.Background(), in) + require.NoError(t, err) + + return out, next +} + +func forceDeploy(t *testing.T, m *Mock) { + t.Helper() + + _, err := m.UpdateService(context.Background(), driver.UpdateServiceInput{ + Service: "web", Cluster: "prod", ForceNewDeployment: true, + }) + require.NoError(t, err) +} + +func TestCreateService_RecordsDeploymentAndRevision(t *testing.T) { + m := newTestMock() + deploymentFixture(t, m) + + deps, next := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, deps, 1) + assert.Empty(t, next) + + dep := deps[0] + assert.Equal(t, driver.DeploymentStatusSuccessful, dep.Status) + assert.Contains(t, dep.ARN, ":service-deployment/prod/web/") + assert.Contains(t, dep.TargetServiceRevision.ARN, ":service-revision/prod/web/") + assert.NotEmpty(t, dep.StartedAt) + assert.NotEmpty(t, dep.FinishedAt) + assert.Empty(t, dep.SourceServiceRevisions, "the first deployment has no source revision") + + revs, failures, err := m.DescribeServiceRevisions(context.Background(), []string{dep.TargetServiceRevision.ARN}) + require.NoError(t, err) + require.Empty(t, failures) + require.Len(t, revs, 1) + assert.Contains(t, revs[0].TaskDefinition, "task-definition/fg:1") + assert.Equal(t, launchFargate, revs[0].LaunchType) + assert.Equal(t, "DISABLED", revs[0].NetworkConfiguration.AwsVpcConfiguration.AssignPublicIP) +} + +func TestUpdateService_NewDeploymentOnChange(t *testing.T) { + m := newTestMock() + deploymentFixture(t, m) + ctx := context.Background() + + first, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, first, 1) + + _, err := m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", TaskDefinition: "fg:2"}) + require.NoError(t, err) + + deps, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, deps, 2, "a task-definition change is a new deployment") + assert.NotEqual(t, first[0].ARN, deps[0].ARN) + require.Len(t, deps[0].SourceServiceRevisions, 1) + assert.Equal(t, first[0].TargetServiceRevision.ARN, deps[0].SourceServiceRevisions[0].ARN) + + two := 0 + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", DesiredCount: &two}) + require.NoError(t, err) + + deps, _ = listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + assert.Len(t, deps, 2, "a desired-count-only change is not a deployment") + + forceDeploy(t, m) + + deps, _ = listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + assert.Len(t, deps, 3, "force-new-deployment is a deployment") +} + +func TestListServiceDeployments_FilterAndPaging(t *testing.T) { + m := newTestMock() + deploymentFixture(t, m) + + clock, ok := m.opts.Clock.(*config.FakeClock) + require.True(t, ok) + + for range 4 { + clock.Advance(time.Hour) + forceDeploy(t, m) + } + + all, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, all, 5) + + for i := 1; i < len(all); i++ { + assert.GreaterOrEqual(t, all[i-1].CreatedAt, all[i].CreatedAt, "newest first") + } + + var got []string + + token := "" + + for pages := 0; pages < 10; pages++ { + page, next := listDeployments(t, m, driver.ListServiceDeploymentsInput{MaxResults: 2, NextToken: token}) + for i := range page { + got = append(got, page[i].ARN) + } + + if next == "" { + break + } + + token = next + } + + require.Len(t, got, 5) + + for i := range all { + assert.Equal(t, all[i].ARN, got[i], "pages resume exactly in the same order") + } + + stopped, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{Statuses: []string{driver.DeploymentStatusStopped}}) + assert.Empty(t, stopped) + + successful, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{Statuses: []string{driver.DeploymentStatusSuccessful}}) + assert.Len(t, successful, 5) + + cutoff := all[2].CreatedAt + after, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{CreatedAtAfter: cutoff}) + assert.Len(t, after, 2, "strictly after the cutoff") + + before, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{CreatedAtBefore: cutoff}) + assert.Len(t, before, 2, "strictly before the cutoff") + + for _, bad := range []driver.ListServiceDeploymentsInput{ + {MaxResults: 101}, {MaxResults: -1}, {NextToken: "!!not-a-token"}, {CreatedAtAfter: "yesterday"}, + } { + bad.Cluster, bad.Service = "prod", "web" + _, _, err := m.ListServiceDeployments(context.Background(), bad) + require.Error(t, err, bad) + assert.Equal(t, excInvalidParameter, ecsException(t, err), bad) + } + + _, _, err := m.ListServiceDeployments(context.Background(), driver.ListServiceDeploymentsInput{Cluster: "ghost", Service: "web"}) + assert.Equal(t, excClusterNotFound, ecsException(t, err)) + + _, _, err = m.ListServiceDeployments(context.Background(), driver.ListServiceDeploymentsInput{Cluster: "prod", Service: "ghost"}) + assert.Equal(t, excServiceNotFound, ecsException(t, err)) +} + +func TestDescribeServiceDeployments_AndRevisions(t *testing.T) { + m := newTestMock() + deploymentFixture(t, m) + ctx := context.Background() + + _, err := m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", TaskDefinition: "fg:2"}) + require.NoError(t, err) + + deps, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, deps, 2) + + got, failures, err := m.DescribeServiceDeployments(ctx, []string{deps[0].ARN, deps[1].ARN, "arn:aws:ecs:us-east-1:000000000000:service-deployment/prod/web/nope"}) + require.NoError(t, err) + require.Len(t, got, 2) + require.Len(t, failures, 1) + assert.Equal(t, "MISSING", failures[0].Reason) + assert.Equal(t, deps[0].TargetServiceRevision.ARN, got[0].TargetServiceRevision.ARN) + + revs, failures, err := m.DescribeServiceRevisions(ctx, []string{ + deps[0].TargetServiceRevision.ARN, deps[1].TargetServiceRevision.ARN, "bogus", + }) + require.NoError(t, err) + require.Len(t, revs, 2) + require.Len(t, failures, 1) + assert.Contains(t, revs[0].TaskDefinition, "fg:2") + assert.Contains(t, revs[1].TaskDefinition, "fg:1") + + tooMany := make([]string, 21) + for i := range tooMany { + tooMany[i] = deps[0].ARN + } + + _, _, err = m.DescribeServiceDeployments(ctx, tooMany) + assert.Equal(t, excInvalidParameter, ecsException(t, err), "at most 20 ARNs") + + _, _, err = m.DescribeServiceRevisions(ctx, tooMany) + assert.Equal(t, excInvalidParameter, ecsException(t, err), "at most 20 ARNs") + + _, _, err = m.DescribeServiceDeployments(ctx, nil) + assert.Equal(t, excInvalidParameter, ecsException(t, err), "ARNs are required") +} + +// asyncDeploymentFixture is deploymentFixture on a mock with AsyncSettle, plus a +// second deployment (fg:2) still IN_PROGRESS. +func asyncDeploymentFixture(t *testing.T) (*Mock, *config.FakeClock, []driver.ServiceDeployment) { + t.Helper() + + m, clock := newSettleMock() + deploymentFixture(t, m) + + clock.Advance(time.Minute) + + _, err := m.UpdateService(context.Background(), driver.UpdateServiceInput{Service: "web", Cluster: "prod", TaskDefinition: "fg:2"}) + require.NoError(t, err) + + deps, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, deps, 2) + require.Equal(t, driver.DeploymentStatusInProgress, deps[0].Status) + require.Equal(t, driver.DeploymentStatusSuccessful, deps[1].Status) + + return m, clock, deps +} + +func TestStopServiceDeployment_Rollback_AsyncWindow(t *testing.T) { + m, clock, deps := asyncDeploymentFixture(t) + ctx := context.Background() + + arn, err := m.StopServiceDeployment(ctx, deps[0].ARN, driver.StopTypeRollback) + require.NoError(t, err) + assert.Equal(t, deps[0].ARN, arn) + + got, _, err := m.DescribeServiceDeployments(ctx, []string{deps[0].ARN}) + require.NoError(t, err) + assert.Equal(t, driver.DeploymentStatusRollbackInProgress, got[0].Status) + + // A repeated stop while it is rolling back continues as-is. + _, err = m.StopServiceDeployment(ctx, deps[0].ARN, driver.StopTypeRollback) + require.NoError(t, err) + + clock.Advance(time.Minute) + + got, _, err = m.DescribeServiceDeployments(ctx, []string{deps[0].ARN}) + require.NoError(t, err) + assert.Equal(t, driver.DeploymentStatusRollbackSuccessful, got[0].Status) + require.NotNil(t, got[0].Rollback) + assert.Equal(t, deps[1].TargetServiceRevision.ARN, got[0].Rollback.ServiceRevisionARN) + + svc, _, err := m.DescribeServices(ctx, "prod", []string{"web"}) + require.NoError(t, err) + assert.Contains(t, svc[0].TaskDefinition, "task-definition/fg:1", "the service is back on the source revision") + + after, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + assert.Len(t, after, 2, "a rollback does not record a new deployment") +} + +func TestStopServiceDeployment_Abort_AsyncWindow(t *testing.T) { + m, clock, deps := asyncDeploymentFixture(t) + ctx := context.Background() + + _, err := m.StopServiceDeployment(ctx, deps[0].ARN, driver.StopTypeAbort) + require.NoError(t, err) + + got, _, err := m.DescribeServiceDeployments(ctx, []string{deps[0].ARN}) + require.NoError(t, err) + assert.Equal(t, driver.DeploymentStatusStopRequested, got[0].Status) + + clock.Advance(time.Minute) + + got, _, err = m.DescribeServiceDeployments(ctx, []string{deps[0].ARN}) + require.NoError(t, err) + assert.Equal(t, driver.DeploymentStatusStopped, got[0].Status) + assert.NotEmpty(t, got[0].StoppedAt) +} + +func TestStopServiceDeployment_ConflictAndNotFound(t *testing.T) { + m := newTestMock() + deploymentFixture(t, m) + ctx := context.Background() + + deps, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, deps, 1) + + _, err := m.StopServiceDeployment(ctx, deps[0].ARN, driver.StopTypeRollback) + require.Error(t, err) + assert.Equal(t, excConflict, ecsException(t, err), "a completed deployment cannot be stopped") + + _, err = m.StopServiceDeployment(ctx, "arn:aws:ecs:us-east-1:000000000000:service-deployment/prod/web/nope", driver.StopTypeAbort) + assert.Equal(t, excServiceDeploymentNotFound, ecsException(t, err)) + + _, err = m.StopServiceDeployment(ctx, deps[0].ARN, "HALT") + assert.Equal(t, excInvalidParameter, ecsException(t, err)) +} + +func TestDeleteService_RemovesDeploymentsAndCapsHistory(t *testing.T) { + m := newTestMock() + deploymentFixture(t, m) + ctx := context.Background() + + for range maxServiceDeployments + 5 { + forceDeploy(t, m) + } + + deps, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{MaxResults: 100}) + assert.Len(t, deps, maxServiceDeployments, "history is capped, oldest dropped") + // The oldest retained deployment still references its source revision. + assert.LessOrEqual(t, m.serviceRevisions.Len(), maxServiceDeployments+1) + + _, err := m.DeleteService(ctx, "prod", "web", true) + require.NoError(t, err) + + assert.Equal(t, 0, m.serviceDeployments.Len()) + assert.Equal(t, 0, m.serviceRevisions.Len()) +} + +func TestRestore_ServiceWithoutDeployments(t *testing.T) { + m := newTestMock() + deploymentFixture(t, m) + + // A snapshot written before deployment records existed restores a service + // with none: it must still report its current deployment. + m.serviceDeployments.Clear() + m.serviceRevisions.Clear() + + deps, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, deps, 1) + assert.Equal(t, driver.DeploymentStatusSuccessful, deps[0].Status) + + revs, _, err := m.DescribeServiceRevisions(context.Background(), []string{deps[0].TargetServiceRevision.ARN}) + require.NoError(t, err) + assert.Len(t, revs, 1) + + again, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, again, 1) + assert.Equal(t, deps[0].ARN, again[0].ARN, "synthesized once, then stable") +} + +func TestLargeCollections_PagedAndBounded(t *testing.T) { + m := newTestMock() + ctx := context.Background() + netCfg := fargateFixture(t, m) + + const services = 5 + + for i := range services { + name := "svc" + string(rune('a'+i)) + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: name, Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + ServiceConnect: &driver.ServiceConnectConfiguration{Namespace: "bulk"}, + }) + require.NoError(t, err) + + for range maxServiceDeployments + 20 { + _, err := m.UpdateService(ctx, driver.UpdateServiceInput{Service: name, Cluster: "prod", ForceNewDeployment: true}) + require.NoError(t, err) + } + + var seen []string + + token := "" + + for { + page, next, err := m.ListServiceDeployments(ctx, driver.ListServiceDeploymentsInput{ + Cluster: "prod", Service: name, MaxResults: 7, NextToken: token, + }) + require.NoError(t, err) + + for i := range page { + seen = append(seen, page[i].ARN) + } + + if next == "" { + break + } + + token = next + } + + assert.Len(t, seen, maxServiceDeployments) + assert.Len(t, uniqueStrings(seen), maxServiceDeployments, "every deployment appears exactly once") + } + + var all []string + + token := "" + + for { + page, next, err := m.ListServicesByNamespace(ctx, "bulk", 2, token) + require.NoError(t, err) + + all = append(all, page...) + + if next == "" { + break + } + + token = next + } + + assert.Len(t, all, services) + assert.Len(t, uniqueStrings(all), services) +} + +func uniqueStrings(in []string) map[string]bool { + out := make(map[string]bool, len(in)) + for _, s := range in { + out[s] = true + } + + return out +} + +func TestMalformedAndForeignIdentifiers_Deployments(t *testing.T) { + m := newTestMock() + deploymentFixture(t, m) + ctx := context.Background() + + deps, _ := listDeployments(t, m, driver.ListServiceDeploymentsInput{}) + require.Len(t, deps, 1) + + got, failures, err := m.DescribeServiceDeployments(ctx, []string{"not-an-arn", "arn:aws:ecs:us-east-1:111111111111:service-deployment/prod/web/x"}) + require.NoError(t, err) + assert.Empty(t, got) + assert.Len(t, failures, 2, "malformed and other-account ARNs are MISSING, never someone else's data") +} + +func TestListServicesByNamespace(t *testing.T) { + m := newTestMock() + ctx := context.Background() + netCfg := fargateFixture(t, m) + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "second"}) + require.NoError(t, err) + + mk := func(cluster, name, ns string) { + in := driver.CreateServiceInput{ + ServiceName: name, Cluster: cluster, TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + } + if ns != "" { + in.ServiceConnect = &driver.ServiceConnectConfiguration{Namespace: ns, Raw: []byte(`{"enabled":true,"namespace":"` + ns + `"}`)} + } + + _, err := m.CreateService(ctx, in) + require.NoError(t, err) + } + + mk("prod", "b", "ns1") + mk("second", "a", "ns1") + mk("prod", "c", "ns2") + mk("prod", "d", "") + + got, next, err := m.ListServicesByNamespace(ctx, "ns1", 0, "") + require.NoError(t, err) + assert.Empty(t, next) + require.Len(t, got, 2) + assert.Contains(t, got[0], "service/prod/b", "sorted by ARN, across clusters") + assert.Contains(t, got[1], "service/second/a") + + deleted, err := m.DeleteService(ctx, "prod", "b", true) + require.NoError(t, err) + assert.Equal(t, statusInactive, deleted.Status) + + got, _, err = m.ListServicesByNamespace(ctx, "ns1", 0, "") + require.NoError(t, err) + assert.Len(t, got, 1, "deleted services are not listed") +} + +func TestListServicesByNamespace_PagingAndUnknown(t *testing.T) { + m := newTestMock() + ctx := context.Background() + netCfg := fargateFixture(t, m) + + for _, n := range []string{"a", "b", "c"} { + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: n, Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + ServiceConnect: &driver.ServiceConnectConfiguration{Namespace: "ns"}, + }) + require.NoError(t, err) + } + + first, next, err := m.ListServicesByNamespace(ctx, "ns", 2, "") + require.NoError(t, err) + require.Len(t, first, 2) + require.NotEmpty(t, next) + + second, next, err := m.ListServicesByNamespace(ctx, "ns", 2, next) + require.NoError(t, err) + assert.Len(t, second, 1) + assert.Empty(t, next) + + unknown, next, err := m.ListServicesByNamespace(ctx, "no-such-namespace", 0, "") + require.NoError(t, err) + assert.Empty(t, unknown, "an unknown namespace lists nothing (no Cloud Map to consult)") + assert.Empty(t, next) + + for _, bad := range []int{-1, 101} { + _, _, err := m.ListServicesByNamespace(ctx, "ns", bad, "") + require.Error(t, err, bad) + assert.Equal(t, excInvalidParameter, ecsException(t, err), bad) + } + + _, _, err = m.ListServicesByNamespace(ctx, "ns", 1, "!!bad") + assert.Equal(t, excInvalidParameter, ecsException(t, err)) + + _, _, err = m.ListServicesByNamespace(ctx, "", 0, "") + assert.Equal(t, excInvalidParameter, ecsException(t, err), "namespace is required") +} diff --git a/providers/aws/ecs/services.go b/providers/aws/ecs/services.go index faf792b62..ed458fe53 100644 --- a/providers/aws/ecs/services.go +++ b/providers/aws/ecs/services.go @@ -95,34 +95,20 @@ func (m *Mock) CreateService(ctx context.Context, in driver.CreateServiceInput) return nil, apiErrf(errors.NotFound, excClusterNotFound, "cluster %q not found", cluster) } - td, err := m.resolveLaunchableTaskDef(in.TaskDefinition) + td, err := m.prepareCreateService(&in) if err != nil { return nil, err } - launchType := effectiveServiceLaunchType(in.LaunchType) - if err := validateLaunch(td, launchType, in.NetworkConfiguration); err != nil { - return nil, err - } - - // Real ECS normalizes the service's taskDefinition to the full ARN regardless - // of how the caller referenced it (bare family or family:revision), so echo the - // resolved ARN rather than the caller's short form. This is what lets Terraform - // diff-suppress family:revision against the ARN AWS always returns. - in.TaskDefinition = td.ARN - - // A DAEMON service runs exactly one task per container instance, so AWS - // rejects a caller-supplied desiredCount rather than overriding it. - if in.SchedulingStrategy == schedDaemon && in.DesiredCount > 0 { - return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, - "desiredCount must not be specified for a DAEMON service") - } - region := regionctx.RegionOr(ctx, m.opts.Region) arnIn := func(resource string) string { return m.arnIn(region, resource) } svc := serviceFromInput(&in, arnIn, cluster, m.now(), m.rootPrincipalARN()) - if err := m.reserveServiceName(serviceKey(cluster, in.ServiceName), svc); err != nil { + // The name is claimed with a copy: convergence below keeps mutating svc, and + // the stored claim is read concurrently (metrics, other requests) until the + // finished record replaces it. + claim := cloneService(svc) + if err := m.reserveServiceName(serviceKey(cluster, in.ServiceName), &claim); err != nil { return nil, err } @@ -132,16 +118,68 @@ func (m *Mock) CreateService(ctx context.Context, in driver.CreateServiceInput) // SERVICE reads them from the tag store, and a re-created service reuses // its predecessor's ARN, whose stale entry must not leak into its tasks. m.recordTags(svc.ARN, in.Tags) - m.convergeNewService(ctx, svc, td, &events) + + if svc.DeploymentController == deployControllerExternal { + // No tasks and no deployments: an EXTERNAL service runs tasks only + // through its task sets. + svc.Events = []driver.ServiceEvent{m.serviceEvent(fmt.Sprintf("(service %s) has started 0 tasks.", svc.Name))} + } else { + m.convergeNewService(ctx, svc, td, &events) + } + m.services.Set(serviceKey(cluster, svc.Name), svc) + m.recordServiceDeployment(svc) m.publish(ctx, &events) + + if svc.DeploymentController != deployControllerExternal { + m.emitDeploymentEvents(ctx, svc) + } + m.emitServiceSteadyState(ctx, svc) + m.publishClusterMetrics(cluster) out := cloneService(svc) return &out, nil } +// prepareCreateService validates and normalizes a CreateService request and +// returns the task definition its tasks run (nil for an EXTERNAL-controller +// service created without one, since its task sets supply the definition). +func (m *Mock) prepareCreateService(in *driver.CreateServiceInput) (*driver.TaskDefinition, error) { + // A DAEMON service runs exactly one task per container instance, so AWS + // rejects a caller-supplied desiredCount rather than overriding it. + if in.SchedulingStrategy == schedDaemon && in.DesiredCount > 0 { + return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "desiredCount must not be specified for a DAEMON service") + } + + var td *driver.TaskDefinition + + if in.DeploymentController != deployControllerExternal || in.TaskDefinition != "" { + resolved, err := m.resolveLaunchableTaskDef(in.TaskDefinition) + if err != nil { + return nil, err + } + + td = resolved + + if err := validateLaunch(td, effectiveServiceLaunchType(in.LaunchType), in.NetworkConfiguration); err != nil { + return nil, err + } + + // Real ECS normalizes the service's taskDefinition to the full ARN regardless + // of how the caller referenced it (bare family or family:revision), so echo the + // resolved ARN rather than the caller's short form. This is what lets Terraform + // diff-suppress family:revision against the ARN AWS always returns. + in.TaskDefinition = td.ARN + } + + in.NetworkConfiguration = withDefaultAssignPublicIP(in.NetworkConfiguration) + + return td, nil +} + // serviceFromInput builds the service record (without convergence) from the // create input, defaulting the scheduling strategy and deployment controller. // createdBy is the creating principal ECS records on the service. @@ -189,6 +227,7 @@ func serviceFromInput( CapacityProviderStrategy: append([]driver.CapacityProviderStrategyItem(nil), in.CapacityProviderStrategy...), LoadBalancers: append([]driver.LoadBalancer(nil), in.LoadBalancers...), ServiceRegistries: append([]driver.ServiceRegistry(nil), in.ServiceRegistries...), + ServiceConnect: cloneServiceConnect(in.ServiceConnect), Tags: copyTags(in.Tags), } } @@ -241,7 +280,7 @@ func ptrInt(v int) *int { return &v } func (m *Mock) reserveServiceName(key string, svc *driver.Service) error { existing, exists := m.services.Get(key) if exists && existing.Status == statusActive { - return errors.Newf(errors.AlreadyExists, "service %q already exists", svc.Name) + return errServiceNotIdempotent() } if exists { @@ -251,12 +290,18 @@ func (m *Mock) reserveServiceName(key string, svc *driver.Service) error { } if !m.services.SetIfAbsent(key, svc) { - return errors.Newf(errors.AlreadyExists, "service %q already exists", svc.Name) + return errServiceNotIdempotent() } return nil } +// errServiceNotIdempotent is the error CreateService returns when an ACTIVE +// service of the same name already exists in the cluster. +func errServiceNotIdempotent() error { + return apiErrf(errors.AlreadyExists, excInvalidParameter, "Creation of service was not idempotent.") +} + // convergeNewService resolves the target count (DAEMON implies one task per // container instance), launches the tasks, and records the PRIMARY deployment // and the start event on the service. @@ -308,14 +353,19 @@ func (m *Mock) converge( events *pendingTaskEvents, ) (running, pending int) { spec := m.serviceTaskSpec(svc, td, deploymentID) + placementFailure := "" for range target { - task, _ := m.launchTask(ctx, &spec, true) + task, failure := m.launchTask(ctx, &spec, true) if task == nil { continue } - events.add(task, taskEventVersionLaunch) + if failure != nil && placementFailure == "" { + placementFailure = failure.Reason + } + + events.addLaunch(m, task) if task.LastStatus == statusRunning { running++ @@ -326,6 +376,14 @@ func (m *Mock) converge( } } + if placementFailure != "" { + serviceARN, clusterARN := svc.ARN, svc.ClusterARN + + events.addFunc(func(ctx context.Context) { + m.emitServiceAction(ctx, serviceARN, clusterARN, serviceTaskPlacementFailure, serviceEventTypeError, placementFailure) + }) + } + return running, pending } @@ -343,6 +401,7 @@ func (m *Mock) serviceTaskSpec(svc *driver.Service, td *driver.TaskDefinition, d platformVersion: svc.PlatformVersion, netCfg: svc.NetworkConfiguration, tags: m.propagatedTaskTags(svc, td), + enableExec: svc.EnableExecuteCommand, } } @@ -365,24 +424,32 @@ func (m *Mock) propagatedTaskTags(svc *driver.Service, td *driver.TaskDefinition // drainService stops every RUNNING or PENDING task linked to the service in its // cluster, releasing any reserved container-instance capacity. It is used to -// drain a superseded deployment before relaunching and to tear down tasks on -// delete. Each stop is buffered on events, to be published once the caller -// commits the service. -func (m *Mock) drainService(ctx context.Context, svc *driver.Service, events *pendingTaskEvents) { +// drain a superseded deployment before relaunching (keepProtected leaves tasks +// with active scale-in protection running) and to tear down tasks on delete. +// Each stop is buffered on events, to be published once the caller commits the +// service. +func (m *Mock) drainService(ctx context.Context, svc *driver.Service, events *pendingTaskEvents, keepProtected bool) { group := serviceGroup(svc.Name) cluster := clusterNameFromARN(svc.ClusterARN) + now := m.opts.Clock.Now() for _, t := range m.tasks.SortedValues() { if t.Group != group || clusterNameFromARN(t.ClusterARN) != cluster || t.LastStatus == statusStopped { continue } + // A scale-in or redeployment leaves tasks with active scale-in protection + // running; deleting the service stops everything. + if keepProtected && taskProtected(t, now) { + continue + } + m.deregisterTaskTargets(ctx, svc, t) // No reconciliation: this drain already owns and re-converges the whole // service state itself (the caller launches the replacement tasks), so // StopTask's own reconciliation would race it. See stopTaskQuiet. if stopped, stoppedNow, err := m.stopTaskQuiet(ctx, cluster, t.ARN, serviceStoppedReason); err == nil && stoppedNow { - events.add(stopped, taskEventVersionStop) + events.addStop(m, stopped) } } } @@ -496,12 +563,17 @@ func (m *Mock) reconcileServiceAfterStop(ctx context.Context, task *driver.Task) return } + // A task-set task is replaced by its task set, not by the service scheduler. + if m.reconcileTaskSetAfterStop(ctx, task, name) { + return + } + // Replacement launches are published only after reconcileLock is released: // an event target (e.g. a synchronously invoked Lambda) that stops another // task of this same service re-enters this function for the same key, and // the lock is not reentrant. for _, t := range m.reconcileServiceLocked(ctx, task, name) { - m.emitTaskStateChange(ctx, t, taskEventVersionLaunch) + m.emitTaskLaunch(ctx, t) } } @@ -569,14 +641,16 @@ func (m *Mock) newDeployment(id, status string, svc *driver.Service, running, pe RolloutState: rolloutState(running, svc.DesiredCount), CreatedAt: now, UpdatedAt: now, + ServiceConnect: cloneServiceConnect(svc.ServiceConnect), } } -// rolloutState reports COMPLETED once the running count reaches the desired -// count, else IN_PROGRESS. Circuit-breaker FAILED transitions are accepted but +// rolloutState reports COMPLETED once the running count equals the desired +// count, else IN_PROGRESS (running fewer tasks, or more because scale-in +// protection kept surplus ones). Circuit-breaker FAILED transitions are accepted but // not simulated. func rolloutState(running, desired int) string { - if running >= desired { + if running == desired { return rolloutCompleted } @@ -602,11 +676,14 @@ func (m *Mock) UpdateService(ctx context.Context, in driver.UpdateServiceInput) return nil, apiErrf(errors.NotFound, excServiceNotFound, "service %q not found", in.Service) } - // A DAEMON service runs one task per container instance, so ECS rejects a - // caller-supplied desiredCount on update just as it does on create. - if svc.SchedulingStrategy == schedDaemon && in.DesiredCount != nil { - return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, - "desiredCount must not be specified for a DAEMON service") + if err := validateServiceUpdate(svc, &in); err != nil { + return nil, err + } + + in.NetworkConfiguration = withDefaultAssignPublicIP(in.NetworkConfiguration) + + if svc.DeploymentController == deployControllerExternal { + return m.updateExternalService(ctx, svc, &in) } updated := cloneService(svc) @@ -628,17 +705,104 @@ func (m *Mock) UpdateService(ctx context.Context, in driver.UpdateServiceInput) } m.services.Set(serviceKey(cluster, updated.Name), &updated) - m.publish(ctx, &events) - if redeployed { - m.emitServiceSteadyState(ctx, &updated) + deployed := in.ForceNewDeployment || tdChanged || serviceConfigChanged(&in) + if deployed { + m.recordServiceDeployment(&updated) } + m.publish(ctx, &events) + m.emitServiceUpdateEvents(ctx, &updated, countChanged, deployed, redeployed) + m.publishClusterMetrics(cluster) + out := cloneService(&updated) return &out, nil } +// validateServiceUpdate rejects the update inputs ECS refuses up front: a +// caller-supplied desiredCount on a DAEMON service (it runs one task per +// container instance, as on create) and an invalid assignPublicIp. +func validateServiceUpdate(svc *driver.Service, in *driver.UpdateServiceInput) error { + if svc.SchedulingStrategy == schedDaemon && in.DesiredCount != nil { + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "desiredCount must not be specified for a DAEMON service") + } + + return validateAssignPublicIP(in.NetworkConfiguration) +} + +// emitServiceUpdateEvents publishes the events a committed UpdateService caused: +// a desired-count change, a started deployment and the steady state it reached. +func (m *Mock) emitServiceUpdateEvents(ctx context.Context, svc *driver.Service, countChanged, deployed, redeployed bool) { + if countChanged { + m.emitServiceAction(ctx, svc.ARN, svc.ClusterARN, serviceDesiredCountUpdated, serviceEventTypeInfo, "") + } + + if deployed { + m.emitDeploymentEvents(ctx, svc) + } + + if redeployed { + m.emitServiceSteadyState(ctx, svc) + } +} + +// updateExternalService updates a service that uses the EXTERNAL deployment +// controller. Only the desired count and the scalar/reference settings change +// (the task definition comes from task sets), and a new desired count rescales +// every task set. It runs under the service's task-set lock. +func (m *Mock) updateExternalService(ctx context.Context, svc *driver.Service, in *driver.UpdateServiceInput) (*driver.Service, error) { + cluster := clusterNameFromARN(svc.ClusterARN) + key := serviceKey(cluster, svc.Name) + + var events pendingTaskEvents + + out, err := func() (*driver.Service, error) { + unlock := m.taskSetLock.lock(key) + defer unlock() + + current, ok := m.services.Get(key) + if !ok || current.Status != statusActive { + return nil, apiErrf(errors.FailedPrecondition, excServiceNotActive, + "The specified service is not active. You can't update a service that is inactive.") + } + + updated := cloneService(current) + applyServiceScalars(&updated, in) + applyServiceRefs(&updated, in) + + countChanged := in.DesiredCount != nil && *in.DesiredCount != current.DesiredCount + + if in.DesiredCount != nil { + updated.DesiredCount = *in.DesiredCount + } + + m.services.Set(key, &updated) + + if countChanged { + m.rescaleTaskSets(ctx, &updated, &events) + } + + out := cloneService(&updated) + + return &out, nil + }() + if err != nil { + return nil, err + } + + m.publish(ctx, &events) + + if in.DesiredCount != nil && *in.DesiredCount != svc.DesiredCount { + m.emitServiceAction(ctx, out.ARN, out.ClusterARN, serviceDesiredCountUpdated, serviceEventTypeInfo, "") + } + + m.publishClusterMetrics(cluster) + + return out, nil +} + // applyTaskDefChange applies a task-definition change to the pending service // update, reporting whether the definition actually changed. An unchanged (or // unset) task definition is a no-op. A deregistered (INACTIVE) definition can't @@ -687,10 +851,14 @@ func (m *Mock) redeployService( return } - m.drainService(ctx, svc, events) + m.drainService(ctx, svc, events, true) + // Tasks kept alive by scale-in protection count toward the service, so only + // the shortfall is launched; the service reports IN_PROGRESS while it runs + // more tasks than desired. id := m.deploymentID() - running, pending := m.converge(ctx, svc, td, id, target, events) + m.converge(ctx, svc, td, id, max(0, target-len(m.keptProtectedTasks(svc))), events) + running, pending := m.liveServiceTaskCounts(cluster, serviceGroup(svc.Name)) svc.RunningCount = running svc.PendingCount = pending @@ -703,6 +871,14 @@ func (m *Mock) redeployService( svc.Events = append(svc.Events, m.serviceEvent(fmt.Sprintf("(service %s) has started %d tasks.", svc.Name, running))) } +// serviceConfigChanged reports whether an update supplies a setting that is +// part of a service revision (network, load balancers, registries, capacity +// providers, platform version, Service Connect), which starts a deployment. +func serviceConfigChanged(in *driver.UpdateServiceInput) bool { + return in.NetworkConfiguration != nil || in.LoadBalancers != nil || in.ServiceRegistries != nil || + in.CapacityProviderStrategy != nil || in.PlatformVersion != "" || in.ServiceConnect != nil +} + // applyServiceScalars stores the supplied scalar/pointer update fields, leaving // unset (empty or nil) fields unchanged. func applyServiceScalars(svc *driver.Service, in *driver.UpdateServiceInput) { @@ -751,6 +927,10 @@ func applyServiceRefs(svc *driver.Service, in *driver.UpdateServiceInput) { if in.ServiceRegistries != nil { svc.ServiceRegistries = append([]driver.ServiceRegistry(nil), in.ServiceRegistries...) } + + if in.ServiceConnect != nil { + svc.ServiceConnect = cloneServiceConnect(in.ServiceConnect) + } } // ListServices returns services in a cluster in deterministic order. @@ -790,6 +970,7 @@ func (m *Mock) DescribeServices(ctx context.Context, cluster string, ids []strin if s, ok := m.resolveService(want, id); ok { out := cloneService(s) out.Tags = m.liveTags(s.ARN, s.Tags) + out.TaskSets = m.describedTaskSets(want, s) found = append(found, out) continue } @@ -803,6 +984,19 @@ func (m *Mock) DescribeServices(ctx context.Context, cluster string, ids []strin return found, failures, nil } +// describedTaskSets returns the task sets DescribeServices reports for svc: the +// refreshed sets of an ACTIVE EXTERNAL-controller service, none otherwise. +func (m *Mock) describedTaskSets(cluster string, svc *driver.Service) []driver.TaskSet { + if svc.DeploymentController != deployControllerExternal || svc.Status != statusActive { + return nil + } + + unlock := m.taskSetLock.lock(serviceKey(cluster, svc.Name)) + defer unlock() + + return m.taskSetViews(svc) +} + // DeleteService marks a service INACTIVE and stops its tasks (releasing // capacity). AWS refuses to delete a service whose desired or running count is // non-zero unless force is set; with force the service is deleted regardless. @@ -825,11 +1019,21 @@ func (m *Mock) DeleteService(ctx context.Context, cluster, service string, force var events pendingTaskEvents + // The task-set lock makes the cascade atomic against a concurrent + // CreateTaskSet: either the new set exists before the service is marked + // deleted (and is removed here) or it is refused as ServiceNotActive. + unlock := m.taskSetLock.lock(serviceKey(want, svc.Name)) + updated := cloneService(svc) - m.drainService(ctx, &updated, &events) + m.drainService(ctx, &updated, &events, false) + m.deleteTaskSetsOf(ctx, &updated, &events) m.markServiceDeleted(&updated) m.services.Set(serviceKey(want, updated.Name), &updated) + m.deleteServiceDeployments(&updated) + unlock() + m.publish(ctx, &events) + m.publishClusterMetrics(want) out := cloneService(&updated) diff --git a/providers/aws/ecs/snapshot.go b/providers/aws/ecs/snapshot.go index 9044f15a3..110377ffd 100644 --- a/providers/aws/ecs/snapshot.go +++ b/providers/aws/ecs/snapshot.go @@ -31,6 +31,9 @@ type ecsSnapshot struct { Attributes json.RawMessage `json:"attributes,omitempty"` EngineHandles json.RawMessage `json:"engineHandles,omitempty"` CapacityProviders json.RawMessage `json:"capacityProviders,omitempty"` + TaskSets json.RawMessage `json:"taskSets,omitempty"` + Deployments json.RawMessage `json:"serviceDeployments,omitempty"` + Revisions json.RawMessage `json:"serviceRevisions,omitempty"` PortCounter uint32 `json:"portCounter,omitempty"` } @@ -62,6 +65,9 @@ func (m *Mock) snapshotStores(snap *ecsSnapshot) error { {&snap.Attributes, m.attributes.Snapshot}, {&snap.EngineHandles, m.engineHandles.Snapshot}, {&snap.CapacityProviders, m.capacityProviders.Snapshot}, + {&snap.TaskSets, m.taskSets.Snapshot}, + {&snap.Deployments, m.serviceDeployments.Snapshot}, + {&snap.Revisions, m.serviceRevisions.Snapshot}, } for _, d := range dumps { @@ -108,6 +114,9 @@ func (m *Mock) restoreStores(snap *ecsSnapshot) error { {snap.Attributes, m.attributes.LoadSnapshot}, {snap.EngineHandles, m.engineHandles.LoadSnapshot}, {snap.CapacityProviders, m.capacityProviders.LoadSnapshot}, + {snap.TaskSets, m.taskSets.LoadSnapshot}, + {snap.Deployments, m.serviceDeployments.LoadSnapshot}, + {snap.Revisions, m.serviceRevisions.LoadSnapshot}, } for _, l := range loads { diff --git a/providers/aws/ecs/snapshot_test.go b/providers/aws/ecs/snapshot_test.go index bea5561a1..f9d6ca334 100644 --- a/providers/aws/ecs/snapshot_test.go +++ b/providers/aws/ecs/snapshot_test.go @@ -40,3 +40,74 @@ func TestSnapshotRoundTripECS(t *testing.T) { require.NoError(t, err) assert.Equal(t, "web", td.Family) } + +// TestSnapshot_NewStoresRoundTrip proves task sets, service deployments and +// revisions, task protection, exec settings and event/instance versions survive +// a snapshot/restore under their original ARNs. +func TestSnapshot_NewStoresRoundTrip(t *testing.T) { + ctx := context.Background() + src := newTestMock() + netCfg := fargateFixture(t, src) + + _, err := src.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "web", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + DesiredCount: 1, EnableExecuteCommand: true, + ServiceConnect: &driver.ServiceConnectConfiguration{Namespace: "mesh", Raw: []byte(`{"enabled":true,"namespace":"mesh"}`)}, + }) + require.NoError(t, err) + + _, err = src.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "ext", Cluster: "prod", DeploymentController: deployControllerExternal, DesiredCount: 2, + }) + require.NoError(t, err) + + ts, err := src.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + }) + require.NoError(t, err) + + tasks, err := src.ListTasks(ctx, "prod", "", "RUNNING", "web") + require.NoError(t, err) + require.Len(t, tasks, 1) + + _, failures := protect(t, src, nil, tasks[0].ARN) + require.Empty(t, failures) + + deps, _, err := src.ListServiceDeployments(ctx, driver.ListServiceDeploymentsInput{Cluster: "prod", Service: "web"}) + require.NoError(t, err) + require.Len(t, deps, 1) + + raw, err := src.Snapshot(ctx, true) + require.NoError(t, err) + + dst := newTestMock() + require.NoError(t, dst.Restore(ctx, raw)) + + sets, _, err := dst.DescribeTaskSets(ctx, "prod", "ext", []string{ts.ARN}) + require.NoError(t, err) + require.Len(t, sets, 1) + assert.Equal(t, ts.ID, sets[0].ID) + assert.Equal(t, 2, sets[0].RunningCount) + + restored, _, err := dst.ListServiceDeployments(ctx, driver.ListServiceDeploymentsInput{Cluster: "prod", Service: "web"}) + require.NoError(t, err) + require.Len(t, restored, 1) + assert.Equal(t, deps[0].ARN, restored[0].ARN) + + revs, _, err := dst.DescribeServiceRevisions(ctx, []string{deps[0].TargetServiceRevision.ARN}) + require.NoError(t, err) + assert.Len(t, revs, 1) + + got, _, err := dst.GetTaskProtection(ctx, "prod", []string{tasks[0].ARN}) + require.NoError(t, err) + assert.True(t, got[0].ProtectionEnabled) + + byNS, _, err := dst.ListServicesByNamespace(ctx, "mesh", 0, "") + require.NoError(t, err) + assert.Len(t, byNS, 1) + + described, _, err := dst.DescribeTasks(ctx, "prod", []string{tasks[0].ARN}) + require.NoError(t, err) + assert.True(t, described[0].EnableExecuteCommand) + assert.Equal(t, tasks[0].EventVersion, described[0].EventVersion) +} diff --git a/providers/aws/ecs/tags.go b/providers/aws/ecs/tags.go index 4cf5fd9e7..bf2e79063 100644 --- a/providers/aws/ecs/tags.go +++ b/providers/aws/ecs/tags.go @@ -232,6 +232,7 @@ const ( arnTypeTask = "task" arnTypeContainerInst = "container-instance" arnTypeCapacityProvider = "capacity-provider" + arnTypeTaskSet = "task-set" ) // storedARN returns the stored ARN of the resource an ECS ARN names. @@ -266,6 +267,11 @@ func (m *Mock) storedARN(resourceARN, resourceType, rest string) (string, bool) return arnOf(ci, ok, func(ci *driver.ContainerInstance) string { return ci.ARN }) }, + arnTypeTaskSet: func() (string, bool) { + ts, ok := m.taskSets.Get(rest) + + return arnOf(ts, ok, func(ts *driver.TaskSet) string { return ts.ARN }) + }, arnTypeCapacityProvider: func() (string, bool) { if isBuiltinCapacityProvider(rest) { return resourceARN, true diff --git a/providers/aws/ecs/task_protection.go b/providers/aws/ecs/task_protection.go new file mode 100644 index 000000000..fd3168dc4 --- /dev/null +++ b/providers/aws/ecs/task_protection.go @@ -0,0 +1,231 @@ +package ecs + +import ( + "context" + "time" + + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// Task scale-in protection limits and failure reasons, from the ECS +// GetTaskProtection / UpdateTaskProtection API reference. +const ( + protectionDefaultMinutes = 120 + protectionMinMinutes = 1 + protectionMaxMinutes = 2880 + + maxUpdateProtectionTasks = 10 + maxGetProtectionTasks = 100 + + failureTaskNotValid = "TASK_NOT_VALID" + failureDeploymentBlocked = "DEPLOYMENT_BLOCKED" +) + +// Compile-time check that Mock implements the TaskProtection capability. +var _ driver.TaskProtection = (*Mock)(nil) + +// taskProtected reports whether t's scale-in protection is active at now: +// enabled and not past its expiry. +func taskProtected(t *driver.Task, now time.Time) bool { + if !t.ProtectionEnabled { + return false + } + + if t.ProtectionExpiresAt == "" { + return true + } + + expires, err := time.Parse(time.RFC3339, t.ProtectionExpiresAt) + + return err != nil || now.Before(expires) +} + +func protectedView(t *driver.Task, now time.Time) driver.ProtectedTask { + if !taskProtected(t, now) { + return driver.ProtectedTask{TaskARN: t.ARN} + } + + return driver.ProtectedTask{TaskARN: t.ARN, ProtectionEnabled: true, ExpirationDate: t.ProtectionExpiresAt} +} + +// GetTaskProtection reports the scale-in protection of up to 100 service +// tasks. A task that does not exist in the cluster is a MISSING failure; a task +// that is not part of a service is TASK_NOT_VALID. +func (m *Mock) GetTaskProtection(_ context.Context, cluster string, tasks []string) ([]driver.ProtectedTask, []driver.Failure, error) { + want, err := m.protectionCluster(cluster, tasks, maxGetProtectionTasks) + if err != nil { + return nil, nil, err + } + + now := m.opts.Clock.Now() + out := make([]driver.ProtectedTask, 0, len(tasks)) + failures := make([]driver.Failure, 0, len(tasks)) + + for _, id := range tasks { + t, failure := m.protectableTask(want, id) + if failure != nil { + failures = append(failures, *failure) + continue + } + + out = append(out, protectedView(t, now)) + } + + return out, failures, nil +} + +// UpdateTaskProtection enables or disables scale-in protection on up to 10 +// service tasks. Enabling lasts ExpiresInMinutes (1-2880, default 120) from the +// clock and a repeat call resets the expiry. Enabling is refused with +// DEPLOYMENT_BLOCKED when it would leave more protected tasks than the service's +// desired count. The whole call runs under protectMu so the count check and the +// writes cannot interleave with another protection change. +func (m *Mock) UpdateTaskProtection( + _ context.Context, in driver.UpdateTaskProtectionInput, +) ([]driver.ProtectedTask, []driver.Failure, error) { + want, err := m.protectionCluster(in.Cluster, in.Tasks, maxUpdateProtectionTasks) + if err != nil { + return nil, nil, err + } + + minutes := protectionDefaultMinutes + + if in.ProtectionEnabled && in.ExpiresInMinutes != nil { + minutes = *in.ExpiresInMinutes + if minutes < protectionMinMinutes || minutes > protectionMaxMinutes { + return nil, nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "expiresInMinutes must be between %d and %d.", protectionMinMinutes, protectionMaxMinutes) + } + } + + m.protectMu.Lock() + defer m.protectMu.Unlock() + + now := m.opts.Clock.Now() + expires := "" + + if in.ProtectionEnabled { + expires = now.Add(time.Duration(minutes) * time.Minute).UTC().Format(time.RFC3339) + } + + out := make([]driver.ProtectedTask, 0, len(in.Tasks)) + failures := make([]driver.Failure, 0, len(in.Tasks)) + + for _, id := range in.Tasks { + view, failure := m.protectOne(want, id, in.ProtectionEnabled, expires, now) + if failure != nil { + failures = append(failures, *failure) + continue + } + + out = append(out, *view) + } + + return out, failures, nil +} + +// protectOne applies a protection change to one task under protectMu and +// returns its new state, or the failure that kept it from applying. +func (m *Mock) protectOne( + cluster, id string, enable bool, expires string, now time.Time, +) (*driver.ProtectedTask, *driver.Failure) { + t, failure := m.protectableTask(cluster, id) + if failure != nil { + return nil, failure + } + + if enable && m.protectionBlocked(t, now) { + return nil, &driver.Failure{ + ARN: id, Reason: failureDeploymentBlocked, + Detail: "Protecting this task would leave more protected tasks than the service's desired count.", + } + } + + updated := cloneTask(t) + updated.ProtectionEnabled = enable + updated.ProtectionExpiresAt = expires + m.tasks.Set(updated.ARN, &updated) + + view := protectedView(&updated, now) + + return &view, nil +} + +// protectionCluster validates the cluster and task-id list shared by both +// protection operations and returns the bare cluster name. +func (m *Mock) protectionCluster(cluster string, tasks []string, maxTasks int) (string, error) { + want := resolveClusterName(cluster) + if !m.clusterExists(want) { + return "", apiErrf(errors.NotFound, excClusterNotFound, "cluster %q not found", want) + } + + if len(tasks) == 0 || len(tasks) > maxTasks { + return "", apiErrf(errors.InvalidArgument, excInvalidParameter, + "tasks must contain between 1 and %d task ids or ARNs.", maxTasks) + } + + return want, nil +} + +// protectableTask resolves a task id inside cluster want. It returns a MISSING +// failure for an unknown task (or one in another cluster) and TASK_NOT_VALID for +// a task that no service manages, since only service tasks can be protected. +func (m *Mock) protectableTask(want, id string) (*driver.Task, *driver.Failure) { + t, ok := m.resolveTask(id) + if !ok || clusterNameFromARN(t.ClusterARN) != want { + return nil, &driver.Failure{ARN: id, Reason: failureMissing} + } + + if _, managed := serviceNameFromGroup(t.Group); !managed { + return nil, &driver.Failure{ + ARN: t.ARN, Reason: failureTaskNotValid, + Detail: "Only tasks started by a service can be protected from scale-in.", + } + } + + return t, nil +} + +// protectionBlocked reports whether protecting t would leave its service with +// more protected tasks than its desired count. +func (m *Mock) protectionBlocked(t *driver.Task, now time.Time) bool { + name, _ := serviceNameFromGroup(t.Group) + + svc, ok := m.services.Get(serviceKey(clusterNameFromARN(t.ClusterARN), name)) + if !ok { + return false + } + + protected := 1 + + for _, other := range m.tasks.All() { + if other.ARN == t.ARN || other.Group != t.Group || other.ClusterARN != t.ClusterARN || other.LastStatus == statusStopped { + continue + } + + if taskProtected(other, now) { + protected++ + } + } + + return protected > svc.DesiredCount +} + +// keptProtectedTasks returns the service's non-stopped tasks whose protection is +// active; a scale-in or redeployment must leave these running. +func (m *Mock) keptProtectedTasks(svc *driver.Service) []*driver.Task { + group := serviceGroup(svc.Name) + cluster := clusterNameFromARN(svc.ClusterARN) + now := m.opts.Clock.Now() + + var kept []*driver.Task + + for _, t := range m.tasks.SortedValues() { + if t.Group == group && clusterNameFromARN(t.ClusterARN) == cluster && t.LastStatus != statusStopped && taskProtected(t, now) { + kept = append(kept, t) + } + } + + return kept +} diff --git a/providers/aws/ecs/task_protection_test.go b/providers/aws/ecs/task_protection_test.go new file mode 100644 index 000000000..916b37e34 --- /dev/null +++ b/providers/aws/ecs/task_protection_test.go @@ -0,0 +1,278 @@ +package ecs + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// protectionFixture creates a Fargate service with the given desired count and +// returns its task ARNs (sorted). +func protectionFixture(t *testing.T, m *Mock, desired int) []string { + t.Helper() + + netCfg := fargateFixture(t, m) + + _, err := m.CreateService(context.Background(), driver.CreateServiceInput{ + ServiceName: "web", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, + NetworkConfiguration: netCfg, DesiredCount: desired, + }) + require.NoError(t, err) + + tasks, err := m.ListTasks(context.Background(), "prod", "", "", "web") + require.NoError(t, err) + require.Len(t, tasks, desired) + + arns := make([]string, 0, len(tasks)) + for i := range tasks { + arns = append(arns, tasks[i].ARN) + } + + return arns +} + +func protect(t *testing.T, m *Mock, expires *int, arns ...string) ([]driver.ProtectedTask, []driver.Failure) { + t.Helper() + + out, failures, err := m.UpdateTaskProtection(context.Background(), driver.UpdateTaskProtectionInput{ + Cluster: "prod", Tasks: arns, ProtectionEnabled: true, ExpiresInMinutes: expires, + }) + require.NoError(t, err) + + return out, failures +} + +func TestUpdateTaskProtection_EnableDefaultExpiry(t *testing.T) { + m := newTestMock() + arns := protectionFixture(t, m, 1) + + out, failures := protect(t, m, nil, arns...) + require.Empty(t, failures) + require.Len(t, out, 1) + assert.Equal(t, arns[0], out[0].TaskARN) + assert.True(t, out[0].ProtectionEnabled) + assert.Equal(t, "2025-01-01T02:00:00Z", out[0].ExpirationDate, "default 120 minutes from the clock") + + got, failures, err := m.GetTaskProtection(context.Background(), "prod", arns) + require.NoError(t, err) + require.Empty(t, failures) + assert.Equal(t, out, got) +} + +func TestUpdateTaskProtection_ExpiresBounds(t *testing.T) { + m := newTestMock() + arns := protectionFixture(t, m, 1) + + for _, ok := range []int{1, 2880} { + v := ok + _, failures := protect(t, m, &v, arns...) + assert.Empty(t, failures, ok) + } + + for _, bad := range []int{0, 2881, -1} { + v := bad + _, _, err := m.UpdateTaskProtection(context.Background(), driver.UpdateTaskProtectionInput{ + Cluster: "prod", Tasks: arns, ProtectionEnabled: true, ExpiresInMinutes: &v, + }) + require.Error(t, err, bad) + assert.Equal(t, excInvalidParameter, ecsException(t, err), bad) + } +} + +func TestTaskProtection_Failures(t *testing.T) { + m := newTestMock() + ctx := context.Background() + arns := protectionFixture(t, m, 1) + + standalone := runFargate(t, m, &driver.NetworkConfiguration{ + AwsVpcConfiguration: &driver.AwsVpcConfiguration{Subnets: []string{"subnet-1"}}, + }, false) + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "other"}) + require.NoError(t, err) + + _, failures := protect(t, m, nil, standalone.ARN, "does-not-exist", arns[0]) + require.Len(t, failures, 2) + assert.Equal(t, driver.Failure{ARN: standalone.ARN, Reason: "TASK_NOT_VALID", Detail: failures[0].Detail}, failures[0]) + assert.NotEmpty(t, failures[0].Detail) + assert.Equal(t, "MISSING", failures[1].Reason) + assert.Equal(t, "does-not-exist", failures[1].ARN) + + // A task of another cluster is invisible, never reported. + _, failures, err = m.GetTaskProtection(ctx, "other", arns) + require.NoError(t, err) + require.Len(t, failures, 1) + assert.Equal(t, "MISSING", failures[0].Reason) + + _, _, err = m.GetTaskProtection(ctx, "ghost", arns) + assert.Equal(t, excClusterNotFound, ecsException(t, err)) + + tooMany := make([]string, 11) + for i := range tooMany { + tooMany[i] = arns[0] + } + + _, _, err = m.UpdateTaskProtection(ctx, driver.UpdateTaskProtectionInput{ + Cluster: "prod", Tasks: tooMany, ProtectionEnabled: true, + }) + assert.Equal(t, excInvalidParameter, ecsException(t, err), "update accepts at most 10 tasks") + + _, _, err = m.UpdateTaskProtection(ctx, driver.UpdateTaskProtectionInput{Cluster: "prod", ProtectionEnabled: true}) + assert.Equal(t, excInvalidParameter, ecsException(t, err), "tasks is required") + + hundredOne := make([]string, 101) + for i := range hundredOne { + hundredOne[i] = arns[0] + } + + _, _, err = m.GetTaskProtection(ctx, "prod", hundredOne) + assert.Equal(t, excInvalidParameter, ecsException(t, err), "get accepts at most 100 tasks") +} + +func TestTaskProtection_ExpiryByClock(t *testing.T) { + m := newTestMock() + arns := protectionFixture(t, m, 1) + ten := 10 + + protect(t, m, &ten, arns...) + + clock, ok := m.opts.Clock.(*config.FakeClock) + require.True(t, ok) + + clock.Advance(9 * time.Minute) + + got, _, err := m.GetTaskProtection(context.Background(), "prod", arns) + require.NoError(t, err) + assert.True(t, got[0].ProtectionEnabled) + + clock.Advance(2 * time.Minute) + + got, _, err = m.GetTaskProtection(context.Background(), "prod", arns) + require.NoError(t, err) + assert.False(t, got[0].ProtectionEnabled) + assert.Empty(t, got[0].ExpirationDate) +} + +func TestTaskProtection_DisableOmitsExpiryAndRepeatResets(t *testing.T) { + m := newTestMock() + ctx := context.Background() + arns := protectionFixture(t, m, 1) + ten := 10 + + first, _ := protect(t, m, &ten, arns...) + + clock, ok := m.opts.Clock.(*config.FakeClock) + require.True(t, ok) + + clock.Advance(5 * time.Minute) + + second, _ := protect(t, m, &ten, arns...) + assert.NotEqual(t, first[0].ExpirationDate, second[0].ExpirationDate, "repeat enable resets the expiry") + + off, failures, err := m.UpdateTaskProtection(ctx, driver.UpdateTaskProtectionInput{ + Cluster: "prod", Tasks: arns, ProtectionEnabled: false, + }) + require.NoError(t, err) + require.Empty(t, failures) + assert.False(t, off[0].ProtectionEnabled) + assert.Empty(t, off[0].ExpirationDate) +} + +func TestScaleIn_SkipsProtectedTasks(t *testing.T) { + m := newTestMock() + ctx := context.Background() + arns := protectionFixture(t, m, 3) + + protect(t, m, nil, arns[0], arns[1]) + + one := 1 + svc, err := m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", DesiredCount: &one}) + require.NoError(t, err) + + running, err := m.ListTasks(ctx, "prod", "", "RUNNING", "web") + require.NoError(t, err) + + got := map[string]bool{} + for i := range running { + got[running[i].ARN] = true + } + + assert.True(t, got[arns[0]] && got[arns[1]], "protected tasks survive scale-in") + assert.False(t, got[arns[2]], "the unprotected task is stopped") + assert.Len(t, running, 2) + assert.Equal(t, 2, svc.RunningCount) + assert.Equal(t, 1, svc.DesiredCount) + assert.Equal(t, rolloutInProgress, svc.Deployments[0].RolloutState, "more running than desired is not steady state") + + // Once protection is removed the next deployment stops the surplus tasks. + _, _, err = m.UpdateTaskProtection(ctx, driver.UpdateTaskProtectionInput{ + Cluster: "prod", Tasks: []string{arns[0], arns[1]}, ProtectionEnabled: false, + }) + require.NoError(t, err) + + svc, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", ForceNewDeployment: true}) + require.NoError(t, err) + assert.Equal(t, 1, svc.RunningCount) + assert.Equal(t, rolloutCompleted, svc.Deployments[0].RolloutState) +} + +func TestUpdateTaskProtection_DeploymentBlocked(t *testing.T) { + m := newTestMock() + arns := protectionFixture(t, m, 2) + + _, failures := protect(t, m, nil, arns[0]) + require.Empty(t, failures) + + m.services.Update(serviceKey("prod", "web"), func(s *driver.Service) *driver.Service { + out := cloneService(s) + out.DesiredCount = 1 + + return &out + }) + + _, failures = protect(t, m, nil, arns[1]) + require.Len(t, failures, 1) + assert.Equal(t, "DEPLOYMENT_BLOCKED", failures[0].Reason) + assert.Equal(t, arns[1], failures[0].ARN) +} + +func TestTaskProtection_ParallelWithStop(t *testing.T) { + m := newTestMock() + ctx := context.Background() + arns := protectionFixture(t, m, 4) + + var wg sync.WaitGroup + + for i := range 40 { + wg.Add(1) + + go func() { + defer wg.Done() + + arn := arns[i%len(arns)] + + switch i % 3 { + case 0: + _, _, err := m.UpdateTaskProtection(ctx, driver.UpdateTaskProtectionInput{ + Cluster: "prod", Tasks: []string{arn}, ProtectionEnabled: true, + }) + assert.NoError(t, err) + case 1: + _, _, err := m.GetTaskProtection(ctx, "prod", []string{arn}) + assert.NoError(t, err) + default: + _, err := m.StopTask(ctx, "prod", arn, "race") + assert.NoError(t, err) + } + }() + } + + wg.Wait() +} diff --git a/providers/aws/ecs/task_sets.go b/providers/aws/ecs/task_sets.go new file mode 100644 index 000000000..f9e6edc3b --- /dev/null +++ b/providers/aws/ecs/task_sets.go @@ -0,0 +1,623 @@ +package ecs + +import ( + "context" + "math" + "slices" + "strconv" + "strings" + + "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// Task-set statuses and stability values, and the bounds ECS documents for +// CreateTaskSet. +const ( + deployControllerExternal = "EXTERNAL" + + taskSetPrimary = "PRIMARY" + taskSetActive = "ACTIVE" + taskSetDraining = "DRAINING" + + stabilitySteadyState = "STEADY_STATE" + stabilityStabilizing = "STABILIZING" + maxTaskSetClientToken = 36 + maxScalePercent = 100 + taskSetIDDigits = 15 + taskSetIDBase = 16 + + // scaleRoundingEpsilon keeps float noise (3 x 100%% / 100 = 3.0000000000000004) + // from rounding a whole count up. + scaleRoundingEpsilon = 1e-9 +) + +// Compile-time check that Mock implements the TaskSets capability. +var _ driver.TaskSets = (*Mock)(nil) + +func taskSetKey(cluster, service, id string) string { + return cluster + "/" + service + "/" + id +} + +// computedDesiredCount is the service desired count scaled by percent and +// rounded up, ECS's computedDesiredCount. +func computedDesiredCount(desired int, percent float64) int { + return int(math.Ceil(float64(desired)*percent/maxScalePercent - scaleRoundingEpsilon)) +} + +func validateScale(s driver.Scale) error { + if s.Unit != driver.ScaleUnitPercent { + return apiErrf(errors.InvalidArgument, excInvalidParameter, "scale unit must be %s.", driver.ScaleUnitPercent) + } + + if s.Value < 0 || s.Value > maxScalePercent { + return apiErrf(errors.InvalidArgument, excInvalidParameter, "scale value must be between 0 and %d.", maxScalePercent) + } + + return nil +} + +// externalService resolves the ACTIVE service a task-set call targets: the +// cluster must exist, the service must exist, use the EXTERNAL deployment +// controller and be ACTIVE. +func (m *Mock) externalService(cluster, name string) (*driver.Service, error) { + want := resolveClusterName(cluster) + if !m.clusterExists(want) { + return nil, apiErrf(errors.NotFound, excClusterNotFound, "cluster %q not found", want) + } + + svc, ok := m.resolveService(want, name) + if !ok { + return nil, apiErrf(errors.NotFound, excServiceNotFound, "service %q not found", name) + } + + if svc.DeploymentController != deployControllerExternal { + return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "Task sets can only be used with services that use the EXTERNAL deployment controller.") + } + + if svc.Status != statusActive { + return nil, apiErrf(errors.FailedPrecondition, excServiceNotActive, + "The specified service is not active. You can't update a service that is inactive.") + } + + return svc, nil +} + +// CreateTaskSet creates a task set in an EXTERNAL-controller service and +// launches computedDesiredCount tasks for it. The set starts ACTIVE (use +// UpdateServicePrimaryTaskSet to promote it). A repeated clientToken returns the +// task set the token already created. +// +//nolint:gocritic // in is passed by value to satisfy the driver.TaskSets interface. +func (m *Mock) CreateTaskSet(ctx context.Context, in driver.CreateTaskSetInput) (*driver.TaskSet, error) { + scale := driver.Scale{Unit: driver.ScaleUnitPercent, Value: maxScalePercent} + if in.Scale != nil { + scale = *in.Scale + } + + if err := validateScale(scale); err != nil { + return nil, err + } + + if len(in.ClientToken) > maxTaskSetClientToken { + return nil, apiErrf(errors.InvalidArgument, excInvalidParameter, + "clientToken must be at most %d characters.", maxTaskSetClientToken) + } + + if _, err := m.externalService(in.Cluster, in.Service); err != nil { + return nil, err + } + + td, err := m.resolveLaunchableTaskDef(in.TaskDefinition) + if err != nil { + return nil, err + } + + if err = validateLaunch(td, effectiveServiceLaunchType(in.LaunchType), in.NetworkConfiguration); err != nil { + return nil, err + } + + var events pendingTaskEvents + + out, err := m.createTaskSetLocked(ctx, &in, td, scale, &events) + if err != nil { + return nil, err + } + + m.publish(ctx, &events) + m.publishClusterMetrics(resolveClusterName(in.Cluster)) + + return out, nil +} + +// createTaskSetLocked runs under the service's task-set lock: it re-resolves the +// service (a concurrent DeleteService may have removed it), honors an existing +// clientToken, claims the task-set id and launches the set's tasks. +func (m *Mock) createTaskSetLocked( + ctx context.Context, in *driver.CreateTaskSetInput, td *driver.TaskDefinition, scale driver.Scale, events *pendingTaskEvents, +) (*driver.TaskSet, error) { + want := resolveClusterName(in.Cluster) + svcKey := serviceKey(want, serviceNameOf(in.Service)) + + unlock := m.taskSetLock.lock(svcKey) + defer unlock() + + svc, err := m.externalService(in.Cluster, in.Service) + if err != nil { + return nil, err + } + + if in.ClientToken != "" { + for _, existing := range m.taskSetsOf(svc) { + if existing.ClientToken == in.ClientToken { + out := cloneTaskSet(existing) + + return &out, nil + } + } + } + + now := m.now() + id := m.newTaskSetID(svcKey) + region := arnRegion(svc.ARN, m.opts.Region) + + ts := &driver.TaskSet{ + ID: id, + ARN: m.arnIn(region, "task-set/"+want+"/"+svc.Name+"/"+id), + ServiceARN: svc.ARN, + ClusterARN: svc.ClusterARN, + ExternalID: in.ExternalID, + TaskDefinition: td.ARN, + Status: taskSetActive, + LaunchType: in.LaunchType, + PlatformVersion: in.PlatformVersion, + CapacityProviderStrategy: append([]driver.CapacityProviderStrategyItem(nil), in.CapacityProviderStrategy...), + NetworkConfiguration: withDefaultAssignPublicIP(in.NetworkConfiguration), + LoadBalancers: append([]driver.LoadBalancer(nil), in.LoadBalancers...), + ServiceRegistries: append([]driver.ServiceRegistry(nil), in.ServiceRegistries...), + Scale: scale, + StartedBy: id, + ClientToken: in.ClientToken, + CreatedAt: now, + UpdatedAt: now, + Tags: copyTags(in.Tags), + } + + ts.ComputedDesiredCount = computedDesiredCount(svc.DesiredCount, scale.Value) + m.launchTaskSetTasks(ctx, svc, ts, td, ts.ComputedDesiredCount, events) + m.refreshTaskSet(ts, svc.DesiredCount) + m.taskSets.Set(taskSetKey(want, svc.Name, id), ts) + m.recordTags(ts.ARN, in.Tags) + + out := cloneTaskSet(ts) + + return &out, nil +} + +// newTaskSetID mints an "ecs-svc/" task-set id unique within the +// service. +func (m *Mock) newTaskSetID(svcKey string) string { + for { + n, err := strconv.ParseUint(m.hexID()[:taskSetIDDigits], taskSetIDBase, 64) + if err != nil { + continue + } + + id := "ecs-svc/" + strconv.FormatUint(n, 10) + if !m.taskSets.Has(svcKey + "/" + id) { + return id + } + } +} + +// taskSetsOf returns the stored task sets of svc, oldest first. +func (m *Mock) taskSetsOf(svc *driver.Service) []*driver.TaskSet { + var out []*driver.TaskSet + + for _, ts := range m.taskSets.All() { + if ts.ServiceARN == svc.ARN { + out = append(out, ts) + } + } + + slices.SortFunc(out, func(a, b *driver.TaskSet) int { + return strings.Compare(a.CreatedAt+a.ID, b.CreatedAt+b.ID) + }) + + return out +} + +// findTaskSet resolves a task set of svc by id or ARN. +func (m *Mock) findTaskSet(svc *driver.Service, id string) (*driver.TaskSet, bool) { + for _, ts := range m.taskSetsOf(svc) { + if ts.ID == id || ts.ARN == id { + return ts, true + } + } + + return nil, false +} + +// taskSetTasks returns the non-stopped tasks a task set owns, oldest first. +func (m *Mock) taskSetTasks(svc *driver.Service, ts *driver.TaskSet) []*driver.Task { + group := serviceGroup(svc.Name) + cluster := clusterNameFromARN(svc.ClusterARN) + + var out []*driver.Task + + for _, t := range m.tasks.SortedValues() { + if t.StartedBy == ts.ID && t.Group == group && clusterNameFromARN(t.ClusterARN) == cluster && t.LastStatus != statusStopped { + out = append(out, t) + } + } + + return out +} + +// launchTaskSetTasks launches n tasks for the set, buffering their launch +// events for the caller to publish after it releases the task-set lock. +func (m *Mock) launchTaskSetTasks( + ctx context.Context, svc *driver.Service, ts *driver.TaskSet, td *driver.TaskDefinition, n int, events *pendingTaskEvents, +) { + spec := taskSpec{ + cluster: clusterNameFromARN(svc.ClusterARN), clusterARN: svc.ClusterARN, td: td, + launchType: effectiveServiceLaunchType(ts.LaunchType), group: serviceGroup(svc.Name), startedBy: ts.ID, + platformVersion: ts.PlatformVersion, netCfg: ts.NetworkConfiguration, enableExec: svc.EnableExecuteCommand, + } + + for range n { + if task, _ := m.launchTask(ctx, &spec, true); task != nil { + events.addLaunch(m, task) + } + } +} + +// refreshTaskSet recomputes a task set's live counts and stability from the +// task store, stamping stabilityStatusAt when the stability changes. +func (m *Mock) refreshTaskSet(ts *driver.TaskSet, serviceDesired int) { + ts.ComputedDesiredCount = computedDesiredCount(serviceDesired, ts.Scale.Value) + ts.RunningCount, ts.PendingCount = 0, 0 + + cluster := clusterNameFromARN(ts.ClusterARN) + + for _, t := range m.tasks.All() { + if t.StartedBy != ts.ID || clusterNameFromARN(t.ClusterARN) != cluster { + continue + } + + switch t.LastStatus { + case statusRunning: + ts.RunningCount++ + case statusPending: + ts.PendingCount++ + } + } + + stability := stabilityStabilizing + if ts.RunningCount == ts.ComputedDesiredCount && ts.PendingCount == 0 { + stability = stabilitySteadyState + } + + if stability != ts.StabilityStatus { + ts.StabilityStatus = stability + ts.StabilityStatusAt = m.now() + } +} + +// scaleTaskSet brings the set's task count to its computed desired count: +// launches the shortfall, or stops surplus tasks newest first, never the ones +// protected from scale-in. +func (m *Mock) scaleTaskSet(ctx context.Context, svc *driver.Service, ts *driver.TaskSet, events *pendingTaskEvents) { + live := m.taskSetTasks(svc, ts) + + switch { + case len(live) < ts.ComputedDesiredCount: + if td, ok := m.resolveTaskDef(ts.TaskDefinition); ok { + m.launchTaskSetTasks(ctx, svc, ts, td, ts.ComputedDesiredCount-len(live), events) + } + case len(live) > ts.ComputedDesiredCount: + m.stopSurplus(ctx, svc, live, len(live)-ts.ComputedDesiredCount, events) + } +} + +// stopSurplus stops up to n of the given tasks, newest first, skipping tasks +// with active scale-in protection. +func (m *Mock) stopSurplus(ctx context.Context, svc *driver.Service, live []*driver.Task, n int, events *pendingTaskEvents) { + now := m.opts.Clock.Now() + cluster := clusterNameFromARN(svc.ClusterARN) + + for i := len(live) - 1; i >= 0 && n > 0; i-- { + if taskProtected(live[i], now) { + continue + } + + if stopped, stoppedNow, err := m.stopTaskQuiet(ctx, cluster, live[i].ARN, serviceStoppedReason); err == nil && stoppedNow { + events.addStop(m, stopped) + + n-- + } + } +} + +// UpdateTaskSet changes a task set's scale and rescales its tasks. +func (m *Mock) UpdateTaskSet(ctx context.Context, in driver.UpdateTaskSetInput) (*driver.TaskSet, error) { + if err := validateScale(in.Scale); err != nil { + return nil, err + } + + var events pendingTaskEvents + + out, err := m.mutateTaskSet(ctx, in.Cluster, in.Service, in.TaskSet, &events, func(svc *driver.Service, ts *driver.TaskSet) { + ts.Scale = in.Scale + ts.UpdatedAt = m.now() + m.refreshTaskSet(ts, svc.DesiredCount) + m.scaleTaskSet(ctx, svc, ts, &events) + m.refreshTaskSet(ts, svc.DesiredCount) + }) + if err != nil { + return nil, err + } + + m.publish(ctx, &events) + m.publishClusterMetrics(resolveClusterName(in.Cluster)) + + return out, nil +} + +// mutateTaskSet resolves a task set under the service's task-set lock, applies +// mutate to a copy and stores it, returning a clone. +func (m *Mock) mutateTaskSet( + _ context.Context, cluster, service, id string, _ *pendingTaskEvents, mutate func(*driver.Service, *driver.TaskSet), +) (*driver.TaskSet, error) { + want := resolveClusterName(cluster) + unlock := m.taskSetLock.lock(serviceKey(want, serviceNameOf(service))) + + defer unlock() + + svc, err := m.externalService(cluster, service) + if err != nil { + return nil, err + } + + stored, ok := m.findTaskSet(svc, id) + if !ok { + return nil, apiErrf(errors.NotFound, excTaskSetNotFound, "The specified task set %q was not found.", id) + } + + ts := cloneTaskSet(stored) + mutate(svc, &ts) + m.taskSets.Set(taskSetKey(want, svc.Name, ts.ID), &ts) + + out := cloneTaskSet(&ts) + + return &out, nil +} + +// DeleteTaskSet deletes a task set and stops its tasks. Unless force is set the +// set must already be scaled down to zero. The returned copy is DRAINING. +func (m *Mock) DeleteTaskSet(ctx context.Context, in driver.DeleteTaskSetInput) (*driver.TaskSet, error) { + want := resolveClusterName(in.Cluster) + + var events pendingTaskEvents + + out, err := func() (*driver.TaskSet, error) { + unlock := m.taskSetLock.lock(serviceKey(want, serviceNameOf(in.Service))) + defer unlock() + + svc, err := m.externalService(in.Cluster, in.Service) + if err != nil { + return nil, err + } + + stored, ok := m.findTaskSet(svc, in.TaskSet) + if !ok { + return nil, apiErrf(errors.NotFound, excTaskSetNotFound, "The specified task set %q was not found.", in.TaskSet) + } + + if !in.Force && stored.Scale.Value > 0 { + return nil, apiErrf(errors.FailedPrecondition, excInvalidParameter, + "The task set must be scaled down to 0 before it is deleted. Scale it down or set force.") + } + + m.stopTaskSetTasks(ctx, svc, stored, &events) + m.taskSets.Delete(taskSetKey(want, svc.Name, stored.ID)) + m.tags.Delete(stored.ARN) + + out := cloneTaskSet(stored) + out.Status = taskSetDraining + out.RunningCount, out.PendingCount, out.ComputedDesiredCount = 0, 0, 0 + + return &out, nil + }() + if err != nil { + return nil, err + } + + m.publish(ctx, &events) + m.publishClusterMetrics(want) + + return out, nil +} + +// stopTaskSetTasks stops every task a set owns, buffering the stop events. +func (m *Mock) stopTaskSetTasks(ctx context.Context, svc *driver.Service, ts *driver.TaskSet, events *pendingTaskEvents) { + cluster := clusterNameFromARN(svc.ClusterARN) + + for _, t := range m.taskSetTasks(svc, ts) { + if stopped, stoppedNow, err := m.stopTaskQuiet(ctx, cluster, t.ARN, serviceStoppedReason); err == nil && stoppedNow { + events.addStop(m, stopped) + } + } +} + +// DescribeTaskSets returns the service's task sets (all of them when ids is +// empty). An id that matches no task set of the service is a MISSING failure. +func (m *Mock) DescribeTaskSets(_ context.Context, cluster, service string, ids []string) ([]driver.TaskSet, []driver.Failure, error) { + svc, err := m.externalService(cluster, service) + if err != nil { + return nil, nil, err + } + + unlock := m.taskSetLock.lock(serviceKey(resolveClusterName(cluster), svc.Name)) + defer unlock() + + if len(ids) == 0 { + return m.taskSetViews(svc), nil, nil + } + + out := make([]driver.TaskSet, 0, len(ids)) + failures := make([]driver.Failure, 0, len(ids)) + + for _, id := range ids { + stored, ok := m.findTaskSet(svc, id) + if !ok { + failures = append(failures, driver.Failure{ARN: id, Reason: failureMissing}) + continue + } + + out = append(out, m.viewTaskSet(svc, stored)) + } + + return out, failures, nil +} + +// taskSetViews returns refreshed copies of every task set of svc. +func (m *Mock) taskSetViews(svc *driver.Service) []driver.TaskSet { + sets := m.taskSetsOf(svc) + out := make([]driver.TaskSet, 0, len(sets)) + + for _, ts := range sets { + out = append(out, m.viewTaskSet(svc, ts)) + } + + return out +} + +// viewTaskSet returns a copy of ts with live counts and stability, persisting +// the refresh so stabilityStatusAt is stable across reads. +func (m *Mock) viewTaskSet(svc *driver.Service, stored *driver.TaskSet) driver.TaskSet { + ts := cloneTaskSet(stored) + m.refreshTaskSet(&ts, svc.DesiredCount) + + if ts.RunningCount != stored.RunningCount || ts.PendingCount != stored.PendingCount || + ts.ComputedDesiredCount != stored.ComputedDesiredCount || ts.StabilityStatus != stored.StabilityStatus { + persisted := cloneTaskSet(&ts) + m.taskSets.Set(taskSetKey(clusterNameFromARN(svc.ClusterARN), svc.Name, ts.ID), &persisted) + } + + return ts +} + +// UpdateServicePrimaryTaskSet promotes a task set to PRIMARY (the previous +// PRIMARY becomes ACTIVE) and has the service adopt its task definition and +// network configuration. +func (m *Mock) UpdateServicePrimaryTaskSet(_ context.Context, cluster, service, primary string) (*driver.TaskSet, error) { + want := resolveClusterName(cluster) + unlock := m.taskSetLock.lock(serviceKey(want, serviceNameOf(service))) + + defer unlock() + + svc, err := m.externalService(cluster, service) + if err != nil { + return nil, err + } + + target, ok := m.findTaskSet(svc, primary) + if !ok { + return nil, apiErrf(errors.NotFound, excTaskSetNotFound, "The specified task set %q was not found.", primary) + } + + now := m.now() + + for _, ts := range m.taskSetsOf(svc) { + if ts.Status != taskSetPrimary || ts.ID == target.ID { + continue + } + + demoted := cloneTaskSet(ts) + demoted.Status, demoted.UpdatedAt = taskSetActive, now + m.taskSets.Set(taskSetKey(want, svc.Name, demoted.ID), &demoted) + } + + promoted := cloneTaskSet(target) + promoted.Status, promoted.UpdatedAt = taskSetPrimary, now + m.taskSets.Set(taskSetKey(want, svc.Name, promoted.ID), &promoted) + + m.services.Update(serviceKey(want, svc.Name), func(s *driver.Service) *driver.Service { + updated := cloneService(s) + updated.TaskDefinition = promoted.TaskDefinition + updated.NetworkConfiguration = cloneNetworkConfig(promoted.NetworkConfiguration) + + return &updated + }) + + out := m.viewTaskSet(svc, &promoted) + + return &out, nil +} + +// rescaleTaskSets recomputes and rescales every task set of svc after the +// service's desired count changed. It returns the buffered task events. +func (m *Mock) rescaleTaskSets(ctx context.Context, svc *driver.Service, events *pendingTaskEvents) { + for _, stored := range m.taskSetsOf(svc) { + ts := cloneTaskSet(stored) + m.refreshTaskSet(&ts, svc.DesiredCount) + m.scaleTaskSet(ctx, svc, &ts, events) + m.refreshTaskSet(&ts, svc.DesiredCount) + ts.UpdatedAt = m.now() + m.taskSets.Set(taskSetKey(clusterNameFromARN(svc.ClusterARN), svc.Name, ts.ID), &ts) + } +} + +// deleteTaskSetsOf removes every task set of svc with its tasks and tag entries +// (DeleteService's cascade). It runs under the service's task-set lock. +func (m *Mock) deleteTaskSetsOf(ctx context.Context, svc *driver.Service, events *pendingTaskEvents) { + for _, ts := range m.taskSetsOf(svc) { + m.stopTaskSetTasks(ctx, svc, ts, events) + m.taskSets.Delete(taskSetKey(clusterNameFromARN(svc.ClusterARN), svc.Name, ts.ID)) + m.tags.Delete(ts.ARN) + } +} + +// reconcileTaskSetAfterStop relaunches a replacement for a stopped task-set +// task so the set converges back to its computed desired count, as the ECS +// scheduler does. It reports false when the task's service does not use the +// EXTERNAL controller. +func (m *Mock) reconcileTaskSetAfterStop(ctx context.Context, task *driver.Task, name string) bool { + cluster := clusterNameFromARN(task.ClusterARN) + key := serviceKey(cluster, name) + + svc, ok := m.services.Get(key) + if !ok || svc.DeploymentController != deployControllerExternal { + return false + } + + var events pendingTaskEvents + + func() { + unlock := m.taskSetLock.lock(key) + defer unlock() + + current, ok := m.services.Get(key) + if !ok || current.Status != statusActive { + return + } + + stored, ok := m.findTaskSet(current, task.StartedBy) + if !ok { + return + } + + ts := cloneTaskSet(stored) + m.refreshTaskSet(&ts, current.DesiredCount) + m.scaleTaskSet(ctx, current, &ts, &events) + m.refreshTaskSet(&ts, current.DesiredCount) + m.taskSets.Set(taskSetKey(cluster, name, ts.ID), &ts) + }() + + m.publish(ctx, &events) + m.publishClusterMetrics(cluster) + + return true +} diff --git a/providers/aws/ecs/task_sets_test.go b/providers/aws/ecs/task_sets_test.go new file mode 100644 index 000000000..cce68d4c6 --- /dev/null +++ b/providers/aws/ecs/task_sets_test.go @@ -0,0 +1,472 @@ +package ecs + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// externalFixture creates an EXTERNAL-controller service named "ext" with the +// given desired count (no task definition, as real ECS allows) and returns the +// network configuration task sets need. +func externalFixture(t *testing.T, m *Mock, desired int) *driver.NetworkConfiguration { + t.Helper() + + netCfg := fargateFixture(t, m) + + _, err := m.CreateService(context.Background(), driver.CreateServiceInput{ + ServiceName: "ext", Cluster: "prod", DeploymentController: deployControllerExternal, DesiredCount: desired, + }) + require.NoError(t, err) + + return netCfg +} + +func newTaskSet(t *testing.T, m *Mock, netCfg *driver.NetworkConfiguration, scale *driver.Scale) *driver.TaskSet { + t.Helper() + + ts, err := m.CreateTaskSet(context.Background(), driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, + NetworkConfiguration: netCfg, Scale: scale, + }) + require.NoError(t, err) + + return ts +} + +func TestCreateTaskSet_RequiresExternalController(t *testing.T) { + m := newTestMock() + netCfg := fargateFixture(t, m) + ctx := context.Background() + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "rolling", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + }) + require.NoError(t, err) + + _, err = m.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "rolling", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + }) + require.Error(t, err) + assert.Equal(t, excInvalidParameter, ecsException(t, err)) +} + +func TestTaskSet_NotFoundErrors(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 1) + ctx := context.Background() + + _, err := m.CreateTaskSet(ctx, driver.CreateTaskSetInput{Cluster: "ghost", Service: "ext", TaskDefinition: "fg"}) + assert.Equal(t, excClusterNotFound, ecsException(t, err)) + + _, err = m.CreateTaskSet(ctx, driver.CreateTaskSetInput{Cluster: "prod", Service: "ghost", TaskDefinition: "fg"}) + assert.Equal(t, excServiceNotFound, ecsException(t, err)) + + ts := newTaskSet(t, m, netCfg, nil) + + _, err = m.UpdateTaskSet(ctx, driver.UpdateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskSet: "ecs-svc/0", Scale: driver.Scale{Unit: driver.ScaleUnitPercent, Value: 50}, + }) + assert.Equal(t, excTaskSetNotFound, ecsException(t, err)) + + _, err = m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: "ecs-svc/0"}) + assert.Equal(t, excTaskSetNotFound, ecsException(t, err)) + + _, err = m.UpdateServicePrimaryTaskSet(ctx, "prod", "ext", "ecs-svc/0") + assert.Equal(t, excTaskSetNotFound, ecsException(t, err)) + + _, err = m.DeleteService(ctx, "prod", "ext", true) + require.NoError(t, err) + + _, err = m.CreateTaskSet(ctx, driver.CreateTaskSetInput{Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg}) + assert.Equal(t, excServiceNotActive, ecsException(t, err)) + + _, err = m.UpdateTaskSet(ctx, driver.UpdateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskSet: ts.ID, Scale: driver.Scale{Unit: driver.ScaleUnitPercent, Value: 10}, + }) + assert.Contains(t, []string{excServiceNotActive, excTaskSetNotFound}, ecsException(t, err)) +} + +func TestCreateTaskSet_ScaleAndComputedDesiredCount(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 5) + + half := newTaskSet(t, m, netCfg, &driver.Scale{Unit: driver.ScaleUnitPercent, Value: 50}) + assert.Equal(t, 3, half.ComputedDesiredCount, "5 x 50%% rounds up to 3") + assert.Equal(t, 3, half.RunningCount) + assert.Equal(t, "ACTIVE", half.Status) + assert.Contains(t, half.ARN, "task-set/prod/ext/"+half.ID) + + full := newTaskSet(t, m, netCfg, nil) + assert.Equal(t, driver.Scale{Unit: driver.ScaleUnitPercent, Value: 100}, full.Scale, "default scale is 100 percent") + assert.Equal(t, 5, full.ComputedDesiredCount) + + tasks, err := m.ListTasks(context.Background(), "prod", "", "RUNNING", "ext") + require.NoError(t, err) + assert.Len(t, tasks, 8, "task sets launch their own tasks, the service itself runs none") + + svc, _, err := m.DescribeServices(context.Background(), "prod", []string{"ext"}) + require.NoError(t, err) + assert.Empty(t, svc[0].Deployments, "an EXTERNAL service has no deployments") + assert.Len(t, svc[0].TaskSets, 2) +} + +func TestTaskSet_ScaleBounds(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 4) + ctx := context.Background() + + for _, v := range []float64{0, 100} { + _, err := m.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + Scale: &driver.Scale{Unit: driver.ScaleUnitPercent, Value: v}, + }) + require.NoError(t, err, v) + } + + for _, bad := range []driver.Scale{ + {Unit: driver.ScaleUnitPercent, Value: -1}, {Unit: driver.ScaleUnitPercent, Value: 100.5}, {Unit: "COUNT", Value: 1}, + } { + _, err := m.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, Scale: &bad, + }) + require.Error(t, err, bad) + assert.Equal(t, excInvalidParameter, ecsException(t, err), bad) + } +} + +func TestTaskSet_UpdateScaleAndStability(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 4) + ctx := context.Background() + + ts := newTaskSet(t, m, netCfg, &driver.Scale{Unit: driver.ScaleUnitPercent, Value: 50}) + assert.Equal(t, "STEADY_STATE", ts.StabilityStatus) + assert.NotEmpty(t, ts.StabilityStatusAt) + + up, err := m.UpdateTaskSet(ctx, driver.UpdateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskSet: ts.ID, Scale: driver.Scale{Unit: driver.ScaleUnitPercent, Value: 100}, + }) + require.NoError(t, err) + assert.Equal(t, 4, up.ComputedDesiredCount) + assert.Equal(t, 4, up.RunningCount) + + down, err := m.UpdateTaskSet(ctx, driver.UpdateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskSet: ts.ARN, Scale: driver.Scale{Unit: driver.ScaleUnitPercent, Value: 25}, + }) + require.NoError(t, err) + assert.Equal(t, 1, down.ComputedDesiredCount) + assert.Equal(t, 1, down.RunningCount) + + running, err := m.ListTasks(ctx, "prod", "", "RUNNING", "ext") + require.NoError(t, err) + assert.Len(t, running, 1) + + // A service desired-count change rescales every task set. + eight := 8 + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "ext", Cluster: "prod", DesiredCount: &eight}) + require.NoError(t, err) + + got, _, err := m.DescribeTaskSets(ctx, "prod", "ext", []string{ts.ID}) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, 2, got[0].ComputedDesiredCount) + assert.Equal(t, 2, got[0].RunningCount) +} + +func TestTaskSet_StabilizingWhenTasksStopped(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 2) + ctx := context.Background() + ts := newTaskSet(t, m, netCfg, nil) + + tasks, err := m.ListTasks(ctx, "prod", "", "RUNNING", "ext") + require.NoError(t, err) + require.Len(t, tasks, 2) + + // The service scheduler relaunches a stopped task-set task, restoring steady state. + _, err = m.StopTask(ctx, "prod", tasks[0].ARN, "test") + require.NoError(t, err) + + got, _, err := m.DescribeTaskSets(ctx, "prod", "ext", []string{ts.ID}) + require.NoError(t, err) + assert.Equal(t, 2, got[0].RunningCount) + assert.Equal(t, "STEADY_STATE", got[0].StabilityStatus) +} + +func TestUpdateServicePrimaryTaskSet(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 1) + ctx := context.Background() + + a := newTaskSet(t, m, netCfg, nil) + b := newTaskSet(t, m, netCfg, nil) + + got, err := m.UpdateServicePrimaryTaskSet(ctx, "prod", "ext", a.ID) + require.NoError(t, err) + assert.Equal(t, "PRIMARY", got.Status) + + _, err = m.UpdateServicePrimaryTaskSet(ctx, "prod", "ext", b.ARN) + require.NoError(t, err) + + sets, _, err := m.DescribeTaskSets(ctx, "prod", "ext", nil) + require.NoError(t, err) + require.Len(t, sets, 2) + + primaries := 0 + + for _, s := range sets { + switch s.ID { + case b.ID: + assert.Equal(t, "PRIMARY", s.Status) + + primaries++ + case a.ID: + assert.Equal(t, "ACTIVE", s.Status) + } + } + + assert.Equal(t, 1, primaries, "exactly one PRIMARY") + + svc, _, err := m.DescribeServices(ctx, "prod", []string{"ext"}) + require.NoError(t, err) + require.Len(t, svc[0].TaskSets, 2) +} + +func TestDeleteTaskSet_ScaleGuardAndForce(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 2) + ctx := context.Background() + + ts := newTaskSet(t, m, netCfg, nil) + + _, err := m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: ts.ID}) + require.Error(t, err) + assert.Equal(t, excInvalidParameter, ecsException(t, err)) + + deleted, err := m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: ts.ID, Force: true}) + require.NoError(t, err) + assert.Equal(t, "DRAINING", deleted.Status) + + running, err := m.ListTasks(ctx, "prod", "", "RUNNING", "ext") + require.NoError(t, err) + assert.Empty(t, running) + + _, err = m.UpdateServicePrimaryTaskSet(ctx, "prod", "ext", ts.ID) + assert.Equal(t, excTaskSetNotFound, ecsException(t, err)) + + // A set scaled to zero deletes without force. + zero := newTaskSet(t, m, netCfg, &driver.Scale{Unit: driver.ScaleUnitPercent, Value: 0}) + _, err = m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: zero.ID}) + require.NoError(t, err) + + // Deleting the PRIMARY leaves no primary. + p := newTaskSet(t, m, netCfg, nil) + _, err = m.UpdateServicePrimaryTaskSet(ctx, "prod", "ext", p.ID) + require.NoError(t, err) + _, err = m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: p.ID, Force: true}) + require.NoError(t, err) + + left, _, err := m.DescribeTaskSets(ctx, "prod", "ext", nil) + require.NoError(t, err) + assert.Empty(t, left) +} + +func TestDescribeTaskSets_MissingAndDeepCopy(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 1) + ctx := context.Background() + + ts, err := m.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + Tags: []driver.Tag{{Key: "k", Value: "v"}}, + }) + require.NoError(t, err) + + got, failures, err := m.DescribeTaskSets(ctx, "prod", "ext", []string{ts.ID, "ecs-svc/missing"}) + require.NoError(t, err) + require.Len(t, got, 1) + require.Len(t, failures, 1) + assert.Equal(t, "MISSING", failures[0].Reason) + assert.Equal(t, "ecs-svc/missing", failures[0].ARN) + + got[0].Tags[0].Value = "mutated" + got[0].NetworkConfiguration.AwsVpcConfiguration.Subnets[0] = "mutated" + + again, _, err := m.DescribeTaskSets(ctx, "prod", "ext", []string{ts.ARN}) + require.NoError(t, err) + assert.Equal(t, "v", again[0].Tags[0].Value) + assert.Equal(t, "subnet-1", again[0].NetworkConfiguration.AwsVpcConfiguration.Subnets[0]) +} + +func TestCreateTaskSet_ClientTokenIdempotent(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 1) + ctx := context.Background() + + in := driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, + NetworkConfiguration: netCfg, ClientToken: "token-1", + } + + a, err := m.CreateTaskSet(ctx, in) + require.NoError(t, err) + + b, err := m.CreateTaskSet(ctx, in) + require.NoError(t, err) + assert.Equal(t, a.ID, b.ID) + + sets, _, err := m.DescribeTaskSets(ctx, "prod", "ext", nil) + require.NoError(t, err) + assert.Len(t, sets, 1) + + long := in + long.ClientToken = "0123456789012345678901234567890123456" + _, err = m.CreateTaskSet(ctx, long) + assert.Equal(t, excInvalidParameter, ecsException(t, err), "clientToken is at most 36 characters") +} + +func TestCreateTaskSet_ParallelSameToken(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 1) + ctx := context.Background() + + ids := make([]string, 50) + + var wg sync.WaitGroup + + for i := range ids { + wg.Add(1) + + go func() { + defer wg.Done() + + ts, err := m.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, + NetworkConfiguration: netCfg, ClientToken: "same", + }) + if assert.NoError(t, err) { + ids[i] = ts.ID + } + }() + } + + wg.Wait() + + for _, id := range ids { + assert.Equal(t, ids[0], id) + } + + sets, _, err := m.DescribeTaskSets(ctx, "prod", "ext", nil) + require.NoError(t, err) + assert.Len(t, sets, 1) +} + +func TestDeleteService_CascadesTaskSets(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 2) + ctx := context.Background() + + ts, err := m.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + Tags: []driver.Tag{{Key: "k", Value: "v"}}, + }) + require.NoError(t, err) + + _, err = m.DeleteService(ctx, "prod", "ext", true) + require.NoError(t, err) + + assert.Equal(t, 0, m.taskSets.Len(), "no orphan task sets") + + running, err := m.ListTasks(ctx, "prod", "", "RUNNING", "ext") + require.NoError(t, err) + assert.Empty(t, running) + + _, err = m.ListTagsForResource(ctx, ts.ARN) + require.Error(t, err, "the task set's tag entry is gone with it") +} + +func TestDeleteService_ParallelWithCreateTaskSetLeavesNoOrphans(t *testing.T) { + for range 20 { + m := newTestMock() + netCfg := externalFixture(t, m, 1) + ctx := context.Background() + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + _, _ = m.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + }) + }() + + go func() { + defer wg.Done() + + _, _ = m.DeleteService(ctx, "prod", "ext", true) + }() + + wg.Wait() + + svc, _, err := m.DescribeServices(ctx, "prod", []string{"ext"}) + require.NoError(t, err) + require.Len(t, svc, 1) + + if svc[0].Status == statusInactive { + assert.Equal(t, 0, m.taskSets.Len(), "an inactive service must own no task sets") + + running, err := m.ListTasks(ctx, "prod", "", "RUNNING", "ext") + require.NoError(t, err) + assert.Empty(t, running) + } + } +} + +func TestTaskSet_TagsAndTagResource(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 1) + ctx := context.Background() + ts := newTaskSet(t, m, netCfg, nil) + + require.NoError(t, m.TagResource(ctx, ts.ARN, []driver.Tag{{Key: "a", Value: "b"}})) + + tags, err := m.ListTagsForResource(ctx, ts.ARN) + require.NoError(t, err) + assert.Equal(t, []driver.Tag{{Key: "a", Value: "b"}}, tags) + + require.NoError(t, m.UntagResource(ctx, ts.ARN, []string{"a"})) +} + +func TestMalformedAndForeignIdentifiers_TaskSets(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 1) + ctx := context.Background() + ts := newTaskSet(t, m, netCfg, nil) + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "other"}) + require.NoError(t, err) + + _, err = m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "ext", Cluster: "other", DeploymentController: deployControllerExternal, + }) + require.NoError(t, err) + + got, failures, err := m.DescribeTaskSets(ctx, "other", "ext", []string{ts.ID, ts.ARN}) + require.NoError(t, err) + assert.Empty(t, got, "another cluster's task set is invisible") + assert.Len(t, failures, 2) + + _, err = m.UpdateTaskSet(ctx, driver.UpdateTaskSetInput{ + Cluster: "other", Service: "ext", TaskSet: ts.ARN, Scale: driver.Scale{Unit: driver.ScaleUnitPercent, Value: 1}, + }) + assert.Equal(t, excTaskSetNotFound, ecsException(t, err)) +} diff --git a/providers/aws/ecs/tasks.go b/providers/aws/ecs/tasks.go index ff4abbea8..eb27613bf 100644 --- a/providers/aws/ecs/tasks.go +++ b/providers/aws/ecs/tasks.go @@ -73,6 +73,7 @@ func (m *Mock) RunTask(ctx context.Context, in driver.RunTaskInput) ([]driver.Ta cluster: cluster, clusterARN: clusterARN, td: td, launchType: launchType, group: in.Group, startedBy: in.StartedBy, platformVersion: in.PlatformVersion, netCfg: in.NetworkConfiguration, tags: in.Tags, runToCompletion: true, + enableExec: in.EnableExecuteCommand, } // Capacity is the validated constant maximum, not the user value, so the @@ -87,7 +88,7 @@ func (m *Mock) RunTask(ctx context.Context, in driver.RunTaskInput) ([]driver.Ta continue } - m.emitTaskStateChange(ctx, task, taskEventVersionLaunch) + m.emitTaskLaunch(ctx, task) // The response reflects what a caller polling DescribeTasks would see // right now: the launch-settle transient (PROVISIONING/PENDING) rather @@ -96,6 +97,8 @@ func (m *Mock) RunTask(ctx context.Context, in driver.RunTaskInput) ([]driver.Ta tasks = append(tasks, *task) } + m.publishClusterMetrics(cluster) + return tasks, failures, nil } @@ -105,6 +108,14 @@ func (m *Mock) RunTask(ctx context.Context, in driver.RunTaskInput) ([]driver.Ta // the request) plus task-level cpu and memory. It is shared by RunTask and the // service scheduler. func validateLaunch(td *driver.TaskDefinition, launchType string, netCfg *driver.NetworkConfiguration) error { + if err := validateLaunchType(launchType); err != nil { + return err + } + + if err := validateAssignPublicIP(netCfg); err != nil { + return err + } + if len(td.RequiresCompatibilities) > 0 && !containsLaunchType(td.RequiresCompatibilities, launchType) { return apiErrf(errors.InvalidArgument, excInvalidParameter, "task definition does not support the %s launch type", launchType) @@ -114,6 +125,12 @@ func validateLaunch(td *driver.TaskDefinition, launchType string, netCfg *driver return nil } + return validateFargateLaunch(td, netCfg) +} + +// validateFargateLaunch enforces the Fargate-only rules: awsvpc networking on the +// task definition and in the request, and a supported task-level cpu/memory pair. +func validateFargateLaunch(td *driver.TaskDefinition, netCfg *driver.NetworkConfiguration) error { if td.NetworkMode != networkModeAwsvpc { return apiErrf(errors.InvalidArgument, excInvalidParameter, "Fargate requires the awsvpc network mode") @@ -137,6 +154,56 @@ func validateLaunch(td *driver.TaskDefinition, launchType string, netCfg *driver return nil } +// launchManagedInstances is the fourth ECS launch type the SDK models. +const launchManagedInstances = "MANAGED_INSTANCES" + +// validateLaunchType rejects a launch type outside the ECS LaunchType enum +// (EC2, FARGATE, EXTERNAL, MANAGED_INSTANCES, case-sensitive) up front, as a +// request-level InvalidParameterException rather than a placement failure. +func validateLaunchType(launchType string) error { + switch launchType { + case launchEC2, launchFargate, launchExternal, launchManagedInstances: + return nil + default: + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "Unsupported launch type %q; it must be one of EC2, FARGATE, EXTERNAL, MANAGED_INSTANCES.", launchType) + } +} + +// Public-IP assignment values of an awsvpc configuration. +const ( + assignPublicIPEnabled = "ENABLED" + assignPublicIPDisabled = "DISABLED" +) + +// validateAssignPublicIP rejects an assignPublicIp value other than ENABLED or +// DISABLED. An empty value is allowed (it defaults to DISABLED). +func validateAssignPublicIP(nc *driver.NetworkConfiguration) error { + if nc == nil || nc.AwsVpcConfiguration == nil { + return nil + } + + switch nc.AwsVpcConfiguration.AssignPublicIP { + case "", assignPublicIPEnabled, assignPublicIPDisabled: + return nil + default: + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "Invalid assignPublicIp %q; it must be ENABLED or DISABLED.", nc.AwsVpcConfiguration.AssignPublicIP) + } +} + +// withDefaultAssignPublicIP returns a copy of nc with an empty assignPublicIp +// defaulted to DISABLED, the value real ECS reports. The caller's value is never +// mutated. +func withDefaultAssignPublicIP(nc *driver.NetworkConfiguration) *driver.NetworkConfiguration { + out := cloneNetworkConfig(nc) + if out != nil && out.AwsVpcConfiguration != nil && out.AwsVpcConfiguration.AssignPublicIP == "" { + out.AwsVpcConfiguration.AssignPublicIP = assignPublicIPDisabled + } + + return out +} + // fargateMemRange is the supported task memory (MiB) for one Fargate task-cpu // value: either an explicit set of sizes or an inclusive [min,max] with a fixed // step between allowed sizes. The .25 vCPU (256) tier is the one non-uniform @@ -203,6 +270,8 @@ type taskSpec struct { platformVersion string netCfg *driver.NetworkConfiguration tags []driver.Tag + // enableExec is the task's ECS Exec setting, fixed when the task launches. + enableExec bool // runToCompletion selects the engine run mode: standalone RunTask runs its // containers to completion (blocking for exit codes), while the service // scheduler launches them detached. @@ -214,7 +283,8 @@ type taskSpec struct { // EXTERNAL runs unplaced when no instance fits. For EC2 with no fitting // instance the behavior depends on pendingOnShortfall: RunTask (false) returns a // placement failure and stores nothing, while the service scheduler (true) -// stores the task PENDING so the service reports RunningCount 0 { + out.ServiceConnectConfig = r.ServiceConnect.Raw + } + + return out +} + +func (h *Handler) routeServiceDeployments(w http.ResponseWriter, r *http.Request, op string) bool { + handlers := map[string]func(http.ResponseWriter, *http.Request, driver.ServiceDeployments){ + "ListServiceDeployments": listServiceDeployments, + "DescribeServiceDeployments": describeServiceDeployments, + "DescribeServiceRevisions": describeServiceRevisions, + "StopServiceDeployment": stopServiceDeployment, + } + + if handle, known := handlers[op]; known { + if sd, ok := capabilityOf[driver.ServiceDeployments](h.ecs, w, op); ok { + handle(w, r, sd) + } + + return true + } + + if op == "ListServicesByNamespace" { + if sn, ok := capabilityOf[driver.ServiceNamespaces](h.ecs, w, op); ok { + listServicesByNamespace(w, r, sn) + } + + return true + } + + return false +} + +// epochToRFC3339 converts a wire epoch-seconds filter to the RFC3339 instant the +// driver takes ("" when unset). +func epochToRFC3339(sec *float64) string { + if sec == nil { + return "" + } + + return time.Unix(int64(*sec), 0).UTC().Format(time.RFC3339) +} + +func listServiceDeployments(w http.ResponseWriter, r *http.Request, sd driver.ServiceDeployments) { + var req struct { + Service string `json:"service"` + Cluster string `json:"cluster"` + Status []string `json:"status"` + CreatedAt *struct { + Before *float64 `json:"before"` + After *float64 `json:"after"` + } `json:"createdAt"` + MaxResults int `json:"maxResults"` + NextToken string `json:"nextToken"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + in := driver.ListServiceDeploymentsInput{ + Cluster: req.Cluster, Service: req.Service, Statuses: req.Status, MaxResults: req.MaxResults, NextToken: req.NextToken, + } + + if req.CreatedAt != nil { + in.CreatedAtBefore, in.CreatedAtAfter = epochToRFC3339(req.CreatedAt.Before), epochToRFC3339(req.CreatedAt.After) + } + + deps, next, err := sd.ListServiceDeployments(r.Context(), in) + if err != nil { + writeErr(w, err) + + return + } + + out := make([]wireServiceDeploymentBrief, 0, len(deps)) + + for i := range deps { + d := &deps[i] + out = append(out, wireServiceDeploymentBrief{ + ServiceDeploymentArn: d.ARN, ServiceArn: d.ServiceARN, ClusterArn: d.ClusterARN, + TargetServiceRevisionArn: d.TargetServiceRevision.ARN, Status: d.Status, StatusReason: d.StatusReason, + CreatedAt: epoch(d.CreatedAt), StartedAt: epoch(d.StartedAt), FinishedAt: epoch(d.FinishedAt), + }) + } + + resp := map[string]any{"serviceDeployments": out} + if next != "" { + resp["nextToken"] = next + } + + wire.WriteJSON(w, resp) +} + +func describeServiceDeployments(w http.ResponseWriter, r *http.Request, sd driver.ServiceDeployments) { + describeByARNs(w, r, "serviceDeploymentArns", "serviceDeployments", sd.DescribeServiceDeployments, + func(d *driver.ServiceDeployment) any { return serviceDeploymentToWire(d) }) +} + +func describeServiceRevisions(w http.ResponseWriter, r *http.Request, sd driver.ServiceDeployments) { + describeByARNs(w, r, "serviceRevisionArns", "serviceRevisions", sd.DescribeServiceRevisions, + func(rev *driver.ServiceRevision) any { return serviceRevisionToWire(rev) }) +} + +// describeByARNs serves a Describe* call that takes a list of ARNs under arnsKey +// and answers with the converted items under itemsKey plus failures. +func describeByARNs[T any]( + w http.ResponseWriter, r *http.Request, arnsKey, itemsKey string, + call func(context.Context, []string) ([]T, []driver.Failure, error), toWire func(*T) any, +) { + var req map[string][]string + + if !wire.DecodeJSON(w, r, &req) { + return + } + + items, failures, err := call(r.Context(), req[arnsKey]) + if err != nil { + writeErr(w, err) + + return + } + + out := make([]any, 0, len(items)) + + for i := range items { + out = append(out, toWire(&items[i])) + } + + wire.WriteJSON(w, map[string]any{itemsKey: out, "failures": fromFailures(failures)}) +} + +func stopServiceDeployment(w http.ResponseWriter, r *http.Request, sd driver.ServiceDeployments) { + var req struct { + ServiceDeploymentArn string `json:"serviceDeploymentArn"` + StopType string `json:"stopType"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + arn, err := sd.StopServiceDeployment(r.Context(), req.ServiceDeploymentArn, req.StopType) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, map[string]any{"serviceDeploymentArn": arn}) +} + +func listServicesByNamespace(w http.ResponseWriter, r *http.Request, sn driver.ServiceNamespaces) { + var req struct { + Namespace string `json:"namespace"` + MaxResults int `json:"maxResults"` + NextToken string `json:"nextToken"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + arns, next, err := sn.ListServicesByNamespace(r.Context(), req.Namespace, req.MaxResults, req.NextToken) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, listResponse("serviceArns", arns, next)) +} + +// toServiceConnect converts the wire serviceConnectConfiguration to the driver +// shape, keeping the raw JSON and extracting the namespace. +func toServiceConnect(raw json.RawMessage) *driver.ServiceConnectConfiguration { + if len(raw) == 0 || string(raw) == "null" { + return nil + } + + var cfg struct { + Namespace string `json:"namespace"` + } + + _ = json.Unmarshal(raw, &cfg) + + return &driver.ServiceConnectConfiguration{Namespace: cfg.Namespace, Raw: append([]byte(nil), raw...)} +} diff --git a/server/aws/ecs/service_deployments_sdk_test.go b/server/aws/ecs/service_deployments_sdk_test.go new file mode 100644 index 000000000..d5c05c99f --- /dev/null +++ b/server/aws/ecs/service_deployments_sdk_test.go @@ -0,0 +1,141 @@ +package ecs_test + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awsecs "github.com/aws/aws-sdk-go-v2/service/ecs" + ecstypes "github.com/aws/aws-sdk-go-v2/service/ecs/types" +) + +func TestSDK_ServiceDeployments(t *testing.T) { + client := newECSClient(t) + ctx := context.Background() + netCfg := fargateSetup(t, client) + + if _, err := client.CreateService(ctx, &awsecs.CreateServiceInput{ + Cluster: aws.String("prod"), ServiceName: aws.String("web"), TaskDefinition: aws.String("fg"), + LaunchType: ecstypes.LaunchTypeFargate, NetworkConfiguration: netCfg, + }); err != nil { + t.Fatalf("CreateService: %v", err) + } + + for range 2 { + if _, err := client.UpdateService(ctx, &awsecs.UpdateServiceInput{ + Cluster: aws.String("prod"), Service: aws.String("web"), ForceNewDeployment: true, + }); err != nil { + t.Fatalf("UpdateService: %v", err) + } + } + + list, err := client.ListServiceDeployments(ctx, &awsecs.ListServiceDeploymentsInput{ + Cluster: aws.String("prod"), Service: aws.String("web"), MaxResults: aws.Int32(2), + }) + if err != nil || len(list.ServiceDeployments) != 2 || list.NextToken == nil { + t.Fatalf("ListServiceDeployments page 1: %v %+v", err, list) + } + + rest, err := client.ListServiceDeployments(ctx, &awsecs.ListServiceDeploymentsInput{ + Cluster: aws.String("prod"), Service: aws.String("web"), MaxResults: aws.Int32(2), NextToken: list.NextToken, + }) + if err != nil || len(rest.ServiceDeployments) != 1 || rest.NextToken != nil { + t.Fatalf("ListServiceDeployments page 2: %v %+v", err, rest) + } + + if st := list.ServiceDeployments[0].Status; st != ecstypes.ServiceDeploymentStatusSuccessful { + t.Fatalf("status = %s", st) + } + + filtered, err := client.ListServiceDeployments(ctx, &awsecs.ListServiceDeploymentsInput{ + Cluster: aws.String("prod"), Service: aws.String("web"), Status: []ecstypes.ServiceDeploymentStatus{ecstypes.ServiceDeploymentStatusStopped}, + }) + if err != nil || len(filtered.ServiceDeployments) != 0 { + t.Fatalf("status filter: %v %+v", err, filtered) + } + + arn := aws.ToString(list.ServiceDeployments[0].ServiceDeploymentArn) + + desc, err := client.DescribeServiceDeployments(ctx, &awsecs.DescribeServiceDeploymentsInput{ + ServiceDeploymentArns: []string{arn, "arn:aws:ecs:us-east-1:000000000000:service-deployment/prod/web/none"}, + }) + if err != nil || len(desc.ServiceDeployments) != 1 || len(desc.Failures) != 1 || + len(desc.ServiceDeployments[0].SourceServiceRevisions) != 1 { + t.Fatalf("DescribeServiceDeployments: %v %+v", err, desc) + } + + rev := desc.ServiceDeployments[0].TargetServiceRevision + if aws.ToString(rev.Arn) != aws.ToString(list.ServiceDeployments[0].TargetServiceRevisionArn) { + t.Fatalf("target revision mismatch: %+v", rev) + } + + revs, err := client.DescribeServiceRevisions(ctx, &awsecs.DescribeServiceRevisionsInput{ServiceRevisionArns: []string{aws.ToString(rev.Arn)}}) + if err != nil || len(revs.ServiceRevisions) != 1 || aws.ToString(revs.ServiceRevisions[0].TaskDefinition) == "" { + t.Fatalf("DescribeServiceRevisions: %v %+v", err, revs) + } + + _, err = client.StopServiceDeployment(ctx, &awsecs.StopServiceDeploymentInput{ + ServiceDeploymentArn: aws.String(arn), StopType: ecstypes.StopServiceDeploymentStopTypeRollback, + }) + + var conflict *ecstypes.ConflictException + if !errorsAs(err, &conflict) { + t.Fatalf("stopping a completed deployment: want *ConflictException, got %T: %v", err, err) + } + + _, err = client.StopServiceDeployment(ctx, &awsecs.StopServiceDeploymentInput{ + ServiceDeploymentArn: aws.String("arn:aws:ecs:us-east-1:000000000000:service-deployment/prod/web/none"), + StopType: ecstypes.StopServiceDeploymentStopTypeAbort, + }) + + var notFound *ecstypes.ServiceDeploymentNotFoundException + if !errorsAs(err, ¬Found) { + t.Fatalf("want *ServiceDeploymentNotFoundException, got %T: %v", err, err) + } +} + +func TestSDK_ServiceConnectNamespace(t *testing.T) { + client := newECSClient(t) + ctx := context.Background() + netCfg := fargateSetup(t, client) + + for _, name := range []string{"api", "web"} { + if _, err := client.CreateService(ctx, &awsecs.CreateServiceInput{ + Cluster: aws.String("prod"), ServiceName: aws.String(name), TaskDefinition: aws.String("fg"), + LaunchType: ecstypes.LaunchTypeFargate, NetworkConfiguration: netCfg, + ServiceConnectConfiguration: &ecstypes.ServiceConnectConfiguration{Enabled: true, Namespace: aws.String("mesh")}, + }); err != nil { + t.Fatalf("CreateService %s: %v", name, err) + } + } + + got, err := client.ListServicesByNamespace(ctx, &awsecs.ListServicesByNamespaceInput{Namespace: aws.String("mesh"), MaxResults: aws.Int32(1)}) + if err != nil || len(got.ServiceArns) != 1 || got.NextToken == nil { + t.Fatalf("ListServicesByNamespace page 1: %v %+v", err, got) + } + + rest, err := client.ListServicesByNamespace(ctx, &awsecs.ListServicesByNamespaceInput{ + Namespace: aws.String("mesh"), MaxResults: aws.Int32(1), NextToken: got.NextToken, + }) + if err != nil || len(rest.ServiceArns) != 1 || rest.NextToken != nil { + t.Fatalf("ListServicesByNamespace page 2: %v %+v", err, rest) + } + + desc, err := client.DescribeServices(ctx, &awsecs.DescribeServicesInput{Cluster: aws.String("prod"), Services: []string{"api"}}) + if err != nil || len(desc.Services[0].Deployments) == 0 || desc.Services[0].Deployments[0].ServiceConnectConfiguration == nil || + aws.ToString(desc.Services[0].Deployments[0].ServiceConnectConfiguration.Namespace) != "mesh" { + t.Fatalf("deployment must echo serviceConnectConfiguration: %v", err) + } + + empty, err := client.ListServicesByNamespace(ctx, &awsecs.ListServicesByNamespaceInput{Namespace: aws.String("nobody")}) + if err != nil || len(empty.ServiceArns) != 0 { + t.Fatalf("unknown namespace must list nothing: %v %+v", err, empty) + } + + _, err = client.ListServicesByNamespace(ctx, &awsecs.ListServicesByNamespaceInput{Namespace: aws.String("mesh"), MaxResults: aws.Int32(101)}) + + var invalid *ecstypes.InvalidParameterException + if !errorsAs(err, &invalid) { + t.Fatalf("want *InvalidParameterException, got %T: %v", err, err) + } +} diff --git a/server/aws/ecs/services.go b/server/aws/ecs/services.go index 0a18b4f24..ca121db41 100644 --- a/server/aws/ecs/services.go +++ b/server/aws/ecs/services.go @@ -1,6 +1,7 @@ package ecs import ( + "encoding/json" "net/http" "github.com/stackshy/cloudemu/v2/server/wire" @@ -45,6 +46,7 @@ func (h *Handler) createService(w http.ResponseWriter, r *http.Request) { CapacityProviderStrategy []wireCapacityProviderStrategyItem `json:"capacityProviderStrategy"` LoadBalancers []wireLoadBalancer `json:"loadBalancers"` ServiceRegistries []wireServiceRegistry `json:"serviceRegistries"` + ServiceConnectConfiguration json.RawMessage `json:"serviceConnectConfiguration"` Tags []wireTag `json:"tags"` AvailabilityZoneRebalancing string `json:"availabilityZoneRebalancing"` } @@ -71,6 +73,7 @@ func (h *Handler) createService(w http.ResponseWriter, r *http.Request) { CapacityProviderStrategy: toCapacityProviderStrategy(req.CapacityProviderStrategy), LoadBalancers: toLoadBalancers(req.LoadBalancers), ServiceRegistries: toServiceRegistries(req.ServiceRegistries), + ServiceConnect: toServiceConnect(req.ServiceConnectConfiguration), Tags: toTags(req.Tags), AvailabilityZoneRebalancing: req.AvailabilityZoneRebalancing, }) @@ -99,6 +102,7 @@ func (h *Handler) updateService(w http.ResponseWriter, r *http.Request) { CapacityProviderStrategy []wireCapacityProviderStrategyItem `json:"capacityProviderStrategy"` LoadBalancers []wireLoadBalancer `json:"loadBalancers"` ServiceRegistries []wireServiceRegistry `json:"serviceRegistries"` + ServiceConnectConfiguration json.RawMessage `json:"serviceConnectConfiguration"` AvailabilityZoneRebalancing string `json:"availabilityZoneRebalancing"` } @@ -121,6 +125,7 @@ func (h *Handler) updateService(w http.ResponseWriter, r *http.Request) { CapacityProviderStrategy: toCapacityProviderStrategy(req.CapacityProviderStrategy), LoadBalancers: toLoadBalancers(req.LoadBalancers), ServiceRegistries: toServiceRegistries(req.ServiceRegistries), + ServiceConnect: toServiceConnect(req.ServiceConnectConfiguration), AvailabilityZoneRebalancing: req.AvailabilityZoneRebalancing, }) if err != nil { diff --git a/server/aws/ecs/task_protection.go b/server/aws/ecs/task_protection.go new file mode 100644 index 000000000..3cfc5546e --- /dev/null +++ b/server/aws/ecs/task_protection.go @@ -0,0 +1,104 @@ +package ecs + +import ( + "net/http" + + "github.com/stackshy/cloudemu/v2/server/wire" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// unsupportedFeature answers an operation the backing driver does not provide +// with the ECS UnsupportedFeatureException. +func unsupportedFeature(w http.ResponseWriter, op string) { + wire.WriteJSONError(w, http.StatusBadRequest, "UnsupportedFeatureException", + "The operation "+op+" is not supported by this ECS backend.") +} + +// capabilityOf returns the optional capability T of the backing driver, or +// answers the call with UnsupportedFeatureException when it has none. +func capabilityOf[T any](d driver.ECS, w http.ResponseWriter, op string) (T, bool) { + c, ok := d.(T) + if !ok { + unsupportedFeature(w, op) + } + + return c, ok +} + +func (h *Handler) routeTaskProtection(w http.ResponseWriter, r *http.Request, op string) bool { + switch op { + case "GetTaskProtection": + if tp, ok := capabilityOf[driver.TaskProtection](h.ecs, w, op); ok { + getTaskProtection(w, r, tp) + } + case "UpdateTaskProtection": + if tp, ok := capabilityOf[driver.TaskProtection](h.ecs, w, op); ok { + updateTaskProtection(w, r, tp) + } + default: + return false + } + + return true +} + +type wireProtectedTask struct { + TaskArn string `json:"taskArn"` + ProtectionEnabled bool `json:"protectionEnabled"` + ExpirationDate float64 `json:"expirationDate,omitempty"` +} + +func getTaskProtection(w http.ResponseWriter, r *http.Request, tp driver.TaskProtection) { + var req struct { + Cluster string `json:"cluster"` + Tasks []string `json:"tasks"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + tasks, failures, err := tp.GetTaskProtection(r.Context(), req.Cluster, req.Tasks) + if err != nil { + writeErr(w, err) + + return + } + + writeProtectedTasks(w, tasks, failures) +} + +func updateTaskProtection(w http.ResponseWriter, r *http.Request, tp driver.TaskProtection) { + var req struct { + Cluster string `json:"cluster"` + Tasks []string `json:"tasks"` + ProtectionEnabled bool `json:"protectionEnabled"` + ExpiresInMinutes *int `json:"expiresInMinutes"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + tasks, failures, err := tp.UpdateTaskProtection(r.Context(), driver.UpdateTaskProtectionInput{ + Cluster: req.Cluster, Tasks: req.Tasks, ProtectionEnabled: req.ProtectionEnabled, ExpiresInMinutes: req.ExpiresInMinutes, + }) + if err != nil { + writeErr(w, err) + + return + } + + writeProtectedTasks(w, tasks, failures) +} + +func writeProtectedTasks(w http.ResponseWriter, tasks []driver.ProtectedTask, failures []driver.Failure) { + out := make([]wireProtectedTask, 0, len(tasks)) + for i := range tasks { + out = append(out, wireProtectedTask{ + TaskArn: tasks[i].TaskARN, ProtectionEnabled: tasks[i].ProtectionEnabled, ExpirationDate: epoch(tasks[i].ExpirationDate), + }) + } + + wire.WriteJSON(w, map[string]any{"protectedTasks": out, "failures": fromFailures(failures)}) +} diff --git a/server/aws/ecs/task_protection_sdk_test.go b/server/aws/ecs/task_protection_sdk_test.go new file mode 100644 index 000000000..a2095b387 --- /dev/null +++ b/server/aws/ecs/task_protection_sdk_test.go @@ -0,0 +1,52 @@ +package ecs_test + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awsecs "github.com/aws/aws-sdk-go-v2/service/ecs" + ecstypes "github.com/aws/aws-sdk-go-v2/service/ecs/types" +) + +func TestSDK_TaskProtection(t *testing.T) { + client := newECSClient(t) + ctx := context.Background() + netCfg := fargateSetup(t, client) + + if _, err := client.CreateService(ctx, &awsecs.CreateServiceInput{ + Cluster: aws.String("prod"), ServiceName: aws.String("web"), TaskDefinition: aws.String("fg"), + LaunchType: ecstypes.LaunchTypeFargate, NetworkConfiguration: netCfg, DesiredCount: aws.Int32(1), + }); err != nil { + t.Fatalf("CreateService: %v", err) + } + + list, err := client.ListTasks(ctx, &awsecs.ListTasksInput{Cluster: aws.String("prod"), ServiceName: aws.String("web")}) + if err != nil || len(list.TaskArns) != 1 { + t.Fatalf("ListTasks: %v", err) + } + + upd, err := client.UpdateTaskProtection(ctx, &awsecs.UpdateTaskProtectionInput{ + Cluster: aws.String("prod"), Tasks: list.TaskArns, ProtectionEnabled: true, ExpiresInMinutes: aws.Int32(30), + }) + if err != nil || len(upd.ProtectedTasks) != 1 || !upd.ProtectedTasks[0].ProtectionEnabled || upd.ProtectedTasks[0].ExpirationDate == nil { + t.Fatalf("UpdateTaskProtection: %v %+v", err, upd) + } + + get, err := client.GetTaskProtection(ctx, &awsecs.GetTaskProtectionInput{Cluster: aws.String("prod"), Tasks: list.TaskArns}) + if err != nil || len(get.ProtectedTasks) != 1 || !get.ProtectedTasks[0].ProtectionEnabled { + t.Fatalf("GetTaskProtection: %v %+v", err, get) + } + + missing, err := client.GetTaskProtection(ctx, &awsecs.GetTaskProtectionInput{Cluster: aws.String("prod"), Tasks: []string{"nope"}}) + if err != nil || len(missing.Failures) != 1 || aws.ToString(missing.Failures[0].Reason) != "MISSING" { + t.Fatalf("missing task must be a MISSING failure: %v %+v", err, missing) + } + + _, err = client.GetTaskProtection(ctx, &awsecs.GetTaskProtectionInput{Cluster: aws.String("ghost"), Tasks: list.TaskArns}) + + var notFound *ecstypes.ClusterNotFoundException + if !errorsAs(err, ¬Found) { + t.Fatalf("want *ClusterNotFoundException, got %T: %v", err, err) + } +} diff --git a/server/aws/ecs/task_sets.go b/server/aws/ecs/task_sets.go new file mode 100644 index 000000000..1c405207b --- /dev/null +++ b/server/aws/ecs/task_sets.go @@ -0,0 +1,238 @@ +package ecs + +import ( + "net/http" + + "github.com/stackshy/cloudemu/v2/server/wire" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +type wireScale struct { + Unit string `json:"unit,omitempty"` + Value float64 `json:"value"` +} + +type wireTaskSet struct { + ID string `json:"id"` + TaskSetArn string `json:"taskSetArn"` + ServiceArn string `json:"serviceArn"` + ClusterArn string `json:"clusterArn"` + ExternalID string `json:"externalId,omitempty"` + StartedBy string `json:"startedBy,omitempty"` + Status string `json:"status"` + TaskDefinition string `json:"taskDefinition"` + ComputedDesiredCount int `json:"computedDesiredCount"` + PendingCount int `json:"pendingCount"` + RunningCount int `json:"runningCount"` + CreatedAt float64 `json:"createdAt,omitempty"` + UpdatedAt float64 `json:"updatedAt,omitempty"` + LaunchType string `json:"launchType,omitempty"` + PlatformVersion string `json:"platformVersion,omitempty"` + PlatformFamily string `json:"platformFamily,omitempty"` + CapacityProviderStrategy []wireCapacityProviderStrategyItem `json:"capacityProviderStrategy,omitempty"` + NetworkConfiguration *wireNetworkConfiguration `json:"networkConfiguration,omitempty"` + LoadBalancers []wireLoadBalancer `json:"loadBalancers,omitempty"` + ServiceRegistries []wireServiceRegistry `json:"serviceRegistries,omitempty"` + Scale wireScale `json:"scale"` + StabilityStatus string `json:"stabilityStatus,omitempty"` + StabilityStatusAt float64 `json:"stabilityStatusAt,omitempty"` + Tags []wireTag `json:"tags,omitempty"` +} + +func taskSetToWire(ts *driver.TaskSet) wireTaskSet { + return wireTaskSet{ + ID: ts.ID, TaskSetArn: ts.ARN, ServiceArn: ts.ServiceARN, ClusterArn: ts.ClusterARN, + ExternalID: ts.ExternalID, StartedBy: ts.StartedBy, Status: ts.Status, TaskDefinition: ts.TaskDefinition, + ComputedDesiredCount: ts.ComputedDesiredCount, PendingCount: ts.PendingCount, RunningCount: ts.RunningCount, + CreatedAt: epoch(ts.CreatedAt), UpdatedAt: epoch(ts.UpdatedAt), LaunchType: ts.LaunchType, + PlatformVersion: ts.PlatformVersion, PlatformFamily: ts.PlatformFamily, + CapacityProviderStrategy: fromCapacityProviderStrategy(ts.CapacityProviderStrategy), + NetworkConfiguration: fromNetworkConfiguration(ts.NetworkConfiguration), + LoadBalancers: fromLoadBalancers(ts.LoadBalancers), + ServiceRegistries: fromServiceRegistries(ts.ServiceRegistries), + Scale: wireScale{Unit: ts.Scale.Unit, Value: ts.Scale.Value}, + StabilityStatus: ts.StabilityStatus, StabilityStatusAt: epoch(ts.StabilityStatusAt), + Tags: fromTags(ts.Tags), + } +} + +func fromTaskSets(in []driver.TaskSet, withTags bool) []wireTaskSet { + if len(in) == 0 { + return nil + } + + out := make([]wireTaskSet, 0, len(in)) + + for i := range in { + ts := taskSetToWire(&in[i]) + if !withTags { + ts.Tags = nil + } + + out = append(out, ts) + } + + return out +} + +func (h *Handler) routeTaskSets(w http.ResponseWriter, r *http.Request, op string) bool { + handlers := map[string]func(http.ResponseWriter, *http.Request, driver.TaskSets){ + "CreateTaskSet": createTaskSet, + "UpdateTaskSet": updateTaskSet, + "DeleteTaskSet": deleteTaskSet, + "DescribeTaskSets": describeTaskSets, + "UpdateServicePrimaryTaskSet": updatePrimaryTaskSet, + } + + handle, known := handlers[op] + if !known { + return false + } + + if ts, ok := capabilityOf[driver.TaskSets](h.ecs, w, op); ok { + handle(w, r, ts) + } + + return true +} + +func createTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { + var req struct { + Cluster string `json:"cluster"` + Service string `json:"service"` + TaskDefinition string `json:"taskDefinition"` + ExternalID string `json:"externalId"` + LaunchType string `json:"launchType"` + PlatformVersion string `json:"platformVersion"` + CapacityProviderStrategy []wireCapacityProviderStrategyItem `json:"capacityProviderStrategy"` + NetworkConfiguration *wireNetworkConfiguration `json:"networkConfiguration"` + LoadBalancers []wireLoadBalancer `json:"loadBalancers"` + ServiceRegistries []wireServiceRegistry `json:"serviceRegistries"` + Scale *wireScale `json:"scale"` + ClientToken string `json:"clientToken"` + Tags []wireTag `json:"tags"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + in := driver.CreateTaskSetInput{ + Cluster: req.Cluster, Service: req.Service, TaskDefinition: req.TaskDefinition, ExternalID: req.ExternalID, + LaunchType: req.LaunchType, PlatformVersion: req.PlatformVersion, + CapacityProviderStrategy: toCapacityProviderStrategy(req.CapacityProviderStrategy), + NetworkConfiguration: toNetworkConfiguration(req.NetworkConfiguration), + LoadBalancers: toLoadBalancers(req.LoadBalancers), ServiceRegistries: toServiceRegistries(req.ServiceRegistries), + ClientToken: req.ClientToken, Tags: toTags(req.Tags), + } + + if req.Scale != nil { + in.Scale = &driver.Scale{Unit: req.Scale.Unit, Value: req.Scale.Value} + } + + out, err := ts.CreateTaskSet(r.Context(), in) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, map[string]any{"taskSet": taskSetToWire(out)}) +} + +func updateTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { + var req struct { + Cluster string `json:"cluster"` + Service string `json:"service"` + TaskSet string `json:"taskSet"` + Scale wireScale `json:"scale"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + out, err := ts.UpdateTaskSet(r.Context(), driver.UpdateTaskSetInput{ + Cluster: req.Cluster, Service: req.Service, TaskSet: req.TaskSet, + Scale: driver.Scale{Unit: req.Scale.Unit, Value: req.Scale.Value}, + }) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, map[string]any{"taskSet": taskSetToWire(out)}) +} + +func deleteTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { + var req struct { + Cluster string `json:"cluster"` + Service string `json:"service"` + TaskSet string `json:"taskSet"` + Force bool `json:"force"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + out, err := ts.DeleteTaskSet(r.Context(), driver.DeleteTaskSetInput{ + Cluster: req.Cluster, Service: req.Service, TaskSet: req.TaskSet, Force: req.Force, + }) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, map[string]any{"taskSet": taskSetToWire(out)}) +} + +func describeTaskSets(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { + var req struct { + Cluster string `json:"cluster"` + Service string `json:"service"` + TaskSets []string `json:"taskSets"` + Include []string `json:"include"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + sets, failures, err := ts.DescribeTaskSets(r.Context(), req.Cluster, req.Service, req.TaskSets) + if err != nil { + writeErr(w, err) + + return + } + + out := fromTaskSets(sets, includes(req.Include, "TAGS")) + if out == nil { + out = []wireTaskSet{} + } + + wire.WriteJSON(w, map[string]any{"taskSets": out, "failures": fromFailures(failures)}) +} + +func updatePrimaryTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { + var req struct { + Cluster string `json:"cluster"` + Service string `json:"service"` + PrimaryTaskSet string `json:"primaryTaskSet"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + out, err := ts.UpdateServicePrimaryTaskSet(r.Context(), req.Cluster, req.Service, req.PrimaryTaskSet) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, map[string]any{"taskSet": taskSetToWire(out)}) +} diff --git a/server/aws/ecs/task_sets_sdk_test.go b/server/aws/ecs/task_sets_sdk_test.go new file mode 100644 index 000000000..68723f742 --- /dev/null +++ b/server/aws/ecs/task_sets_sdk_test.go @@ -0,0 +1,140 @@ +package ecs_test + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awsecs "github.com/aws/aws-sdk-go-v2/service/ecs" + ecstypes "github.com/aws/aws-sdk-go-v2/service/ecs/types" +) + +func TestSDK_TaskSets_FullLifecycle(t *testing.T) { + client := newECSClient(t) + ctx := context.Background() + netCfg := fargateSetup(t, client) + + if _, err := client.CreateService(ctx, &awsecs.CreateServiceInput{ + Cluster: aws.String("prod"), ServiceName: aws.String("ext"), DesiredCount: aws.Int32(4), + DeploymentController: &ecstypes.DeploymentController{Type: ecstypes.DeploymentControllerTypeExternal}, + }); err != nil { + t.Fatalf("CreateService EXTERNAL: %v", err) + } + + created, err := client.CreateTaskSet(ctx, &awsecs.CreateTaskSetInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), TaskDefinition: aws.String("fg"), + LaunchType: ecstypes.LaunchTypeFargate, NetworkConfiguration: netCfg, + Scale: &ecstypes.Scale{Unit: ecstypes.ScaleUnitPercent, Value: 50}, + Tags: []ecstypes.Tag{{Key: aws.String("k"), Value: aws.String("v")}}, + }) + if err != nil { + t.Fatalf("CreateTaskSet: %v", err) + } + + set := created.TaskSet + if set.ComputedDesiredCount != 2 || set.RunningCount != 2 || aws.ToString(set.Status) != "ACTIVE" || + set.StabilityStatus != ecstypes.StabilityStatusSteadyState || aws.ToString(set.Id) == "" { + t.Fatalf("unexpected task set: %+v", set) + } + + plain, err := client.DescribeTaskSets(ctx, &awsecs.DescribeTaskSetsInput{Cluster: aws.String("prod"), Service: aws.String("ext")}) + if err != nil || len(plain.TaskSets) != 1 || len(plain.TaskSets[0].Tags) != 0 { + t.Fatalf("DescribeTaskSets must omit tags without include=TAGS: %v %+v", err, plain) + } + + withTags, err := client.DescribeTaskSets(ctx, &awsecs.DescribeTaskSetsInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), TaskSets: []string{aws.ToString(set.Id), "ecs-svc/1"}, + Include: []ecstypes.TaskSetField{ecstypes.TaskSetFieldTags}, + }) + if err != nil || len(withTags.TaskSets) != 1 || len(withTags.TaskSets[0].Tags) != 1 || len(withTags.Failures) != 1 || + aws.ToString(withTags.Failures[0].Reason) != "MISSING" { + t.Fatalf("include=TAGS/failures: %v %+v", err, withTags) + } + + if _, err := client.UpdateTaskSet(ctx, &awsecs.UpdateTaskSetInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), TaskSet: set.TaskSetArn, + Scale: &ecstypes.Scale{Unit: ecstypes.ScaleUnitPercent, Value: 100}, + }); err != nil { + t.Fatalf("UpdateTaskSet: %v", err) + } + + prim, err := client.UpdateServicePrimaryTaskSet(ctx, &awsecs.UpdateServicePrimaryTaskSetInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), PrimaryTaskSet: set.Id, + }) + if err != nil || aws.ToString(prim.TaskSet.Status) != "PRIMARY" { + t.Fatalf("UpdateServicePrimaryTaskSet: %v %+v", err, prim) + } + + svc, err := client.DescribeServices(ctx, &awsecs.DescribeServicesInput{Cluster: aws.String("prod"), Services: []string{"ext"}}) + if err != nil || len(svc.Services[0].TaskSets) != 1 || aws.ToString(svc.Services[0].TaskSets[0].Status) != "PRIMARY" { + t.Fatalf("DescribeServices taskSets: %v", err) + } + + _, err = client.DeleteTaskSet(ctx, &awsecs.DeleteTaskSetInput{Cluster: aws.String("prod"), Service: aws.String("ext"), TaskSet: set.Id}) + + var invalid *ecstypes.InvalidParameterException + if !errorsAs(err, &invalid) { + t.Fatalf("DeleteTaskSet without force on a scaled set: want *InvalidParameterException, got %T: %v", err, err) + } + + del, err := client.DeleteTaskSet(ctx, &awsecs.DeleteTaskSetInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), TaskSet: set.Id, Force: aws.Bool(true), + }) + if err != nil || aws.ToString(del.TaskSet.Status) != "DRAINING" { + t.Fatalf("DeleteTaskSet force: %v %+v", err, del) + } + + _, err = client.UpdateTaskSet(ctx, &awsecs.UpdateTaskSetInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), TaskSet: set.Id, + Scale: &ecstypes.Scale{Unit: ecstypes.ScaleUnitPercent, Value: 1}, + }) + + var notFound *ecstypes.TaskSetNotFoundException + if !errorsAs(err, ¬Found) { + t.Fatalf("want *TaskSetNotFoundException, got %T: %v", err, err) + } + + if _, err := client.DeleteService(ctx, &awsecs.DeleteServiceInput{Cluster: aws.String("prod"), Service: aws.String("ext"), Force: aws.Bool(true)}); err != nil { + t.Fatal(err) + } + + _, err = client.CreateTaskSet(ctx, &awsecs.CreateTaskSetInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), TaskDefinition: aws.String("fg"), + LaunchType: ecstypes.LaunchTypeFargate, NetworkConfiguration: netCfg, + }) + + var inactive *ecstypes.ServiceNotActiveException + if !errorsAs(err, &inactive) { + t.Fatalf("want *ServiceNotActiveException, got %T: %v", err, err) + } +} + +func TestSDK_TaskSets_RequireExternalController(t *testing.T) { + client := newECSClient(t) + ctx := context.Background() + netCfg := fargateSetup(t, client) + + if _, err := client.CreateService(ctx, &awsecs.CreateServiceInput{ + Cluster: aws.String("prod"), ServiceName: aws.String("rolling"), TaskDefinition: aws.String("fg"), + LaunchType: ecstypes.LaunchTypeFargate, NetworkConfiguration: netCfg, + }); err != nil { + t.Fatal(err) + } + + _, err := client.CreateTaskSet(ctx, &awsecs.CreateTaskSetInput{ + Cluster: aws.String("prod"), Service: aws.String("rolling"), TaskDefinition: aws.String("fg"), + LaunchType: ecstypes.LaunchTypeFargate, NetworkConfiguration: netCfg, + }) + + var invalid *ecstypes.InvalidParameterException + if !errorsAs(err, &invalid) { + t.Fatalf("want *InvalidParameterException, got %T: %v", err, err) + } + + _, err = client.DescribeTaskSets(ctx, &awsecs.DescribeTaskSetsInput{Cluster: aws.String("prod"), Service: aws.String("ghost")}) + + var noService *ecstypes.ServiceNotFoundException + if !errorsAs(err, &noService) { + t.Fatalf("want *ServiceNotFoundException, got %T: %v", err, err) + } +} diff --git a/server/aws/ecs/tasks.go b/server/aws/ecs/tasks.go index c9c27e878..aad10d70b 100644 --- a/server/aws/ecs/tasks.go +++ b/server/aws/ecs/tasks.go @@ -77,6 +77,7 @@ func (h *Handler) runTask(w http.ResponseWriter, r *http.Request) { StartedBy string `json:"startedBy"` NetworkConfiguration *wireNetworkConfiguration `json:"networkConfiguration"` CapacityProviderStrategy []wireCapacityProviderStrategyItem `json:"capacityProviderStrategy"` + EnableExecuteCommand bool `json:"enableExecuteCommand"` Tags []wireTag `json:"tags"` } @@ -94,6 +95,7 @@ func (h *Handler) runTask(w http.ResponseWriter, r *http.Request) { StartedBy: req.StartedBy, NetworkConfiguration: toNetworkConfiguration(req.NetworkConfiguration), CapacityProviderStrategy: toCapacityProviderStrategy(req.CapacityProviderStrategy), + EnableExecuteCommand: req.EnableExecuteCommand, Tags: toTags(req.Tags), }) if err != nil { diff --git a/server/aws/ecs/types.go b/server/aws/ecs/types.go index fe2790023..b883f6603 100644 --- a/server/aws/ecs/types.go +++ b/server/aws/ecs/types.go @@ -218,15 +218,33 @@ type wireContainer struct { LastStatus string `json:"lastStatus,omitempty"` // ExitCode is a pointer so a real exit 0 on a STOPPED container serializes // (real ECS reports it), while a running container omits it entirely. - ExitCode *int `json:"exitCode,omitempty"` - Reason string `json:"reason,omitempty"` - RuntimeID string `json:"runtimeId,omitempty"` - NetworkBindings []wireNetworkBinding `json:"networkBindings,omitempty"` + ExitCode *int `json:"exitCode,omitempty"` + Reason string `json:"reason,omitempty"` + RuntimeID string `json:"runtimeId,omitempty"` + NetworkBindings []wireNetworkBinding `json:"networkBindings,omitempty"` + NetworkInterfaces []wireNetworkInterface `json:"networkInterfaces,omitempty"` + ManagedAgents []wireManagedAgent `json:"managedAgents,omitempty"` +} + +// wireNetworkInterface is the ECS NetworkInterface shape of a container. +type wireNetworkInterface struct { + AttachmentID string `json:"attachmentId,omitempty"` + PrivateIPv4Address string `json:"privateIpv4Address,omitempty"` + IPv6Address string `json:"ipv6Address,omitempty"` +} + +// wireManagedAgent is the ECS ManagedAgent shape of a container. +type wireManagedAgent struct { + Name string `json:"name,omitempty"` + LastStatus string `json:"lastStatus,omitempty"` + LastStartedAt float64 `json:"lastStartedAt,omitempty"` + Reason string `json:"reason,omitempty"` } // wireAttachment mirrors the ECS Attachment shape (type/status/details), where // details is a list of name/value pairs (KeyValuePair). type wireAttachment struct { + ID string `json:"id,omitempty"` Type string `json:"type,omitempty"` Status string `json:"status,omitempty"` Details []wireKeyValue `json:"details,omitempty"` @@ -275,6 +293,7 @@ type wireTask struct { StoppedAt float64 `json:"stoppedAt,omitempty"` StoppedReason string `json:"stoppedReason,omitempty"` StopCode string `json:"stopCode,omitempty"` + EnableExecuteCommand bool `json:"enableExecuteCommand"` CPU string `json:"cpu,omitempty"` Memory string `json:"memory,omitempty"` AvailabilityZone string `json:"availabilityZone,omitempty"` @@ -346,6 +365,8 @@ type wireDeployment struct { RolloutStateReason string `json:"rolloutStateReason,omitempty"` CreatedAt float64 `json:"createdAt,omitempty"` UpdatedAt float64 `json:"updatedAt,omitempty"` + // ServiceConnectConfiguration echoes the service's Service Connect setting. + ServiceConnectConfiguration json.RawMessage `json:"serviceConnectConfiguration,omitempty"` } type wireServiceEvent struct { @@ -380,6 +401,7 @@ type wireService struct { Events []wireServiceEvent `json:"events,omitempty"` CreatedAt float64 `json:"createdAt,omitempty"` Tags []wireTag `json:"tags,omitempty"` + TaskSets []wireTaskSet `json:"taskSets,omitempty"` // AvailabilityZoneRebalancing has no omitempty: real ECS always reports // this field ("ENABLED"/"DISABLED"), even for services that never opted in. AvailabilityZoneRebalancing string `json:"availabilityZoneRebalancing"` @@ -1221,6 +1243,36 @@ func fromNetworkBindings(in []driver.NetworkBinding) []wireNetworkBinding { return out } +func fromNetworkInterfaces(in []driver.ContainerNetworkInterface) []wireNetworkInterface { + if len(in) == 0 { + return nil + } + + out := make([]wireNetworkInterface, 0, len(in)) + for _, ni := range in { + out = append(out, wireNetworkInterface{ + AttachmentID: ni.AttachmentID, PrivateIPv4Address: ni.PrivateIPv4Address, IPv6Address: ni.IPv6Address, + }) + } + + return out +} + +func fromManagedAgents(in []driver.ManagedAgent) []wireManagedAgent { + if len(in) == 0 { + return nil + } + + out := make([]wireManagedAgent, 0, len(in)) + for _, a := range in { + out = append(out, wireManagedAgent{ + Name: a.Name, LastStatus: a.LastStatus, LastStartedAt: epoch(a.LastStartedAt), Reason: a.Reason, + }) + } + + return out +} + func taskToWire(t *driver.Task) wireTask { containers := make([]wireContainer, 0, len(t.Containers)) @@ -1230,7 +1282,9 @@ func taskToWire(t *driver.Task) wireTask { ContainerArn: c.ARN, TaskArn: t.ARN, Name: c.Name, Image: c.Image, LastStatus: c.LastStatus, Reason: c.Reason, RuntimeID: c.RuntimeID, - NetworkBindings: fromNetworkBindings(c.NetworkBindings), + NetworkBindings: fromNetworkBindings(c.NetworkBindings), + NetworkInterfaces: fromNetworkInterfaces(c.NetworkInterfaces), + ManagedAgents: fromManagedAgents(c.ManagedAgents), } // Surface the exit code only once the container has stopped, so a genuine // exit 0 is reported while a running container has none. @@ -1259,6 +1313,7 @@ func taskToWire(t *driver.Task) wireTask { StoppedAt: epoch(t.StoppedAt), StoppedReason: t.StoppedReason, StopCode: t.StopCode, + EnableExecuteCommand: t.EnableExecuteCommand, CPU: t.CPU, Memory: t.Memory, AvailabilityZone: t.AvailabilityZone, @@ -1277,6 +1332,7 @@ func fromAttachments(in []driver.Attachment) []wireAttachment { out := make([]wireAttachment, 0, len(in)) for i := range in { out = append(out, wireAttachment{ + ID: in[i].ID, Type: in[i].Type, Status: in[i].Status, Details: fromKeyValues(in[i].Details), @@ -1321,6 +1377,7 @@ func serviceToWire(s *driver.Service) wireService { Events: fromServiceEvents(s.Events), CreatedAt: epoch(s.CreatedAt), Tags: fromTags(s.Tags), + TaskSets: fromTaskSets(s.TaskSets, false), AvailabilityZoneRebalancing: s.AvailabilityZoneRebalancing, } if s.DeploymentController != "" { @@ -1378,12 +1435,22 @@ func fromDeployments(in []driver.Deployment) []wireDeployment { DesiredCount: in[i].DesiredCount, RunningCount: in[i].RunningCount, PendingCount: in[i].PendingCount, LaunchType: in[i].LaunchType, RolloutState: in[i].RolloutState, RolloutStateReason: in[i].RolloutStateReason, CreatedAt: epoch(in[i].CreatedAt), UpdatedAt: epoch(in[i].UpdatedAt), + ServiceConnectConfiguration: serviceConnectRaw(in[i].ServiceConnect), }) } return out } +// serviceConnectRaw returns the verbatim Service Connect configuration, or nil. +func serviceConnectRaw(in *driver.ServiceConnectConfiguration) json.RawMessage { + if in == nil || len(in.Raw) == 0 { + return nil + } + + return in.Raw +} + func fromServiceEvents(in []driver.ServiceEvent) []wireServiceEvent { if len(in) == 0 { return nil diff --git a/services/ecs/capabilities.go b/services/ecs/capabilities.go new file mode 100644 index 000000000..e250d6216 --- /dev/null +++ b/services/ecs/capabilities.go @@ -0,0 +1,196 @@ +package ecs + +import ( + "context" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +// The methods in this file expose the optional driver capabilities (task +// protection, task sets, service deployments, Service Connect namespaces) +// through the same error-injection, rate-limit, latency, metrics and recording +// pipeline as the primary operations. A driver that does not implement a +// capability answers FailedPrecondition. + +// capabilityOf returns the optional capability T of the wrapped driver. +func capabilityOf[T any](e *ECS, op string) (T, error) { + c, ok := e.driver.(T) + if !ok { + var zero T + + return zero, cerrors.Newf(cerrors.FailedPrecondition, "the ECS driver does not implement %s", op) + } + + return c, nil +} + +// GetTaskProtection reports the scale-in protection of service tasks. +func (e *ECS) GetTaskProtection(ctx context.Context, cluster string, tasks []string) ( + []driver.ProtectedTask, []driver.Failure, error, +) { + tp, err := capabilityOf[driver.TaskProtection](e, "GetTaskProtection") + if err != nil { + return nil, nil, err + } + + return doBatch(ctx, e, "GetTaskProtection", tasks, func() ([]driver.ProtectedTask, []driver.Failure, error) { + return tp.GetTaskProtection(ctx, cluster, tasks) + }) +} + +// UpdateTaskProtection enables or disables scale-in protection on service tasks. +func (e *ECS) UpdateTaskProtection(ctx context.Context, in driver.UpdateTaskProtectionInput) ( + []driver.ProtectedTask, []driver.Failure, error, +) { + tp, err := capabilityOf[driver.TaskProtection](e, "UpdateTaskProtection") + if err != nil { + return nil, nil, err + } + + return doBatch(ctx, e, "UpdateTaskProtection", in, func() ([]driver.ProtectedTask, []driver.Failure, error) { + return tp.UpdateTaskProtection(ctx, in) + }) +} + +// callCapability runs one single-result capability operation through do(). +func callCapability[C, R any](ctx context.Context, e *ECS, op string, input any, call func(C) (R, error)) (R, error) { + var zero R + + c, err := capabilityOf[C](e, op) + if err != nil { + return zero, err + } + + out, err := e.do(ctx, op, input, func() (any, error) { return call(c) }) + if err != nil { + return zero, err + } + + return out.(R), nil +} + +// CreateTaskSet creates a task set in an EXTERNAL-controller service. +// +//nolint:gocritic // in is passed by value to mirror the driver interface; the copy is cheap for a mock. +func (e *ECS) CreateTaskSet(ctx context.Context, in driver.CreateTaskSetInput) (*driver.TaskSet, error) { + return callCapability(ctx, e, "CreateTaskSet", in, func(c driver.TaskSets) (*driver.TaskSet, error) { + return c.CreateTaskSet(ctx, in) + }) +} + +// UpdateTaskSet changes a task set's scale. +func (e *ECS) UpdateTaskSet(ctx context.Context, in driver.UpdateTaskSetInput) (*driver.TaskSet, error) { + return callCapability(ctx, e, "UpdateTaskSet", in, func(c driver.TaskSets) (*driver.TaskSet, error) { + return c.UpdateTaskSet(ctx, in) + }) +} + +// DeleteTaskSet deletes a task set. +func (e *ECS) DeleteTaskSet(ctx context.Context, in driver.DeleteTaskSetInput) (*driver.TaskSet, error) { + return callCapability(ctx, e, "DeleteTaskSet", in, func(c driver.TaskSets) (*driver.TaskSet, error) { + return c.DeleteTaskSet(ctx, in) + }) +} + +// DescribeTaskSets describes a service's task sets, returning failures for unresolved ids. +func (e *ECS) DescribeTaskSets(ctx context.Context, cluster, service string, ids []string) ( + []driver.TaskSet, []driver.Failure, error, +) { + c, err := capabilityOf[driver.TaskSets](e, "DescribeTaskSets") + if err != nil { + return nil, nil, err + } + + return doBatch(ctx, e, "DescribeTaskSets", ids, func() ([]driver.TaskSet, []driver.Failure, error) { + return c.DescribeTaskSets(ctx, cluster, service, ids) + }) +} + +// UpdateServicePrimaryTaskSet promotes a task set to PRIMARY. +func (e *ECS) UpdateServicePrimaryTaskSet(ctx context.Context, cluster, service, primary string) (*driver.TaskSet, error) { + return callCapability(ctx, e, "UpdateServicePrimaryTaskSet", primary, func(c driver.TaskSets) (*driver.TaskSet, error) { + return c.UpdateServicePrimaryTaskSet(ctx, cluster, service, primary) + }) +} + +// deploymentPage is a page of ListServiceDeployments results threaded through do(). +type deploymentPage struct { + items []driver.ServiceDeployment + next string +} + +// ListServiceDeployments lists a service's deployments, newest first. +// +//nolint:gocritic // in is passed by value to mirror the driver interface; the copy is cheap for a mock. +func (e *ECS) ListServiceDeployments(ctx context.Context, in driver.ListServiceDeploymentsInput) ( + items []driver.ServiceDeployment, next string, err error, +) { + page, err := callCapability(ctx, e, "ListServiceDeployments", in, func(c driver.ServiceDeployments) (deploymentPage, error) { + got, token, cerr := c.ListServiceDeployments(ctx, in) + + return deploymentPage{items: got, next: token}, cerr + }) + if err != nil { + return nil, "", err + } + + return page.items, page.next, nil +} + +// DescribeServiceDeployments describes deployments by ARN, returning failures for unresolved ARNs. +func (e *ECS) DescribeServiceDeployments(ctx context.Context, arns []string) ( + []driver.ServiceDeployment, []driver.Failure, error, +) { + c, err := capabilityOf[driver.ServiceDeployments](e, "DescribeServiceDeployments") + if err != nil { + return nil, nil, err + } + + return doBatch(ctx, e, "DescribeServiceDeployments", arns, func() ([]driver.ServiceDeployment, []driver.Failure, error) { + return c.DescribeServiceDeployments(ctx, arns) + }) +} + +// DescribeServiceRevisions describes service revisions by ARN, returning failures for unresolved ARNs. +func (e *ECS) DescribeServiceRevisions(ctx context.Context, arns []string) ( + []driver.ServiceRevision, []driver.Failure, error, +) { + c, err := capabilityOf[driver.ServiceDeployments](e, "DescribeServiceRevisions") + if err != nil { + return nil, nil, err + } + + return doBatch(ctx, e, "DescribeServiceRevisions", arns, func() ([]driver.ServiceRevision, []driver.Failure, error) { + return c.DescribeServiceRevisions(ctx, arns) + }) +} + +// StopServiceDeployment stops a deployment that has not completed. +func (e *ECS) StopServiceDeployment(ctx context.Context, arn, stopType string) (string, error) { + return callCapability(ctx, e, "StopServiceDeployment", arn, func(c driver.ServiceDeployments) (string, error) { + return c.StopServiceDeployment(ctx, arn, stopType) + }) +} + +// ListServicesByNamespace lists the services that joined a Service Connect namespace. +func (e *ECS) ListServicesByNamespace(ctx context.Context, namespace string, maxResults int, nextToken string) ( + arns []string, next string, err error, +) { + page, err := callCapability(ctx, e, "ListServicesByNamespace", namespace, func(c driver.ServiceNamespaces) (arnPage, error) { + got, token, cerr := c.ListServicesByNamespace(ctx, namespace, maxResults, nextToken) + + return arnPage{items: got, next: token}, cerr + }) + if err != nil { + return nil, "", err + } + + return page.items, page.next, nil +} + +// arnPage is a page of ARNs threaded through do(). +type arnPage struct { + items []string + next string +} diff --git a/services/ecs/driver/capabilities.go b/services/ecs/driver/capabilities.go new file mode 100644 index 000000000..a40a68f3e --- /dev/null +++ b/services/ecs/driver/capabilities.go @@ -0,0 +1,225 @@ +package driver + +import "context" + +// The interfaces in this file are optional capabilities of an ECS provider: +// the wire handler discovers them by type assertion, so they extend the ECS +// interface without widening it. + +// ProtectedTask is a task's scale-in protection state. ExpirationDate is an +// RFC3339 instant, empty when protection is disabled or has expired. +type ProtectedTask struct { + TaskARN string + ProtectionEnabled bool + ExpirationDate string +} + +// UpdateTaskProtectionInput describes a scale-in protection change. A nil +// ExpiresInMinutes means the default (120 minutes) when enabling. +type UpdateTaskProtectionInput struct { + Cluster string + Tasks []string + ProtectionEnabled bool + ExpiresInMinutes *int +} + +// TaskProtection reads and changes the scale-in protection of service tasks. +type TaskProtection interface { + GetTaskProtection(ctx context.Context, cluster string, tasks []string) ([]ProtectedTask, []Failure, error) + UpdateTaskProtection(ctx context.Context, in UpdateTaskProtectionInput) ([]ProtectedTask, []Failure, error) +} + +// ScaleUnitPercent is the only task-set scale unit ECS models. +const ScaleUnitPercent = "PERCENT" + +// Scale is the share of a service's desired count a task set runs. Value is +// 0-100 and Unit is ScaleUnitPercent. +type Scale struct { + Unit string + Value float64 +} + +// TaskSet is a set of tasks in a service that uses the EXTERNAL deployment +// controller. ComputedDesiredCount is the service desired count scaled by Scale +// and rounded up. Status is PRIMARY, ACTIVE or DRAINING; StabilityStatus is +// STEADY_STATE or STABILIZING. ClientToken is the creation idempotency token. +type TaskSet struct { + ID string + ARN string + ServiceARN string + ClusterARN string + ExternalID string + TaskDefinition string + Status string + StabilityStatus string + StabilityStatusAt string + LaunchType string + PlatformVersion string + PlatformFamily string + CapacityProviderStrategy []CapacityProviderStrategyItem + NetworkConfiguration *NetworkConfiguration + LoadBalancers []LoadBalancer + ServiceRegistries []ServiceRegistry + Scale Scale + ComputedDesiredCount int + PendingCount int + RunningCount int + StartedBy string + ClientToken string + CreatedAt string + UpdatedAt string + Tags []Tag +} + +// CreateTaskSetInput describes a task set to create. A nil Scale means 100 +// percent. +type CreateTaskSetInput struct { + Cluster string + Service string + TaskDefinition string + ExternalID string + LaunchType string + PlatformVersion string + CapacityProviderStrategy []CapacityProviderStrategyItem + NetworkConfiguration *NetworkConfiguration + LoadBalancers []LoadBalancer + ServiceRegistries []ServiceRegistry + Scale *Scale + ClientToken string + Tags []Tag +} + +// UpdateTaskSetInput changes a task set's scale, the only mutable field. +type UpdateTaskSetInput struct { + Cluster string + Service string + TaskSet string + Scale Scale +} + +// DeleteTaskSetInput identifies a task set to delete. Without Force the set must +// already be scaled down to zero. +type DeleteTaskSetInput struct { + Cluster string + Service string + TaskSet string + Force bool +} + +// TaskSets manages task sets, the task groups of EXTERNAL-controller services. +type TaskSets interface { + CreateTaskSet(ctx context.Context, in CreateTaskSetInput) (*TaskSet, error) + UpdateTaskSet(ctx context.Context, in UpdateTaskSetInput) (*TaskSet, error) + DeleteTaskSet(ctx context.Context, in DeleteTaskSetInput) (*TaskSet, error) + DescribeTaskSets(ctx context.Context, cluster, service string, ids []string) ([]TaskSet, []Failure, error) + UpdateServicePrimaryTaskSet(ctx context.Context, cluster, service, primaryTaskSet string) (*TaskSet, error) +} + +// Service deployment statuses (the ECS ServiceDeploymentStatus enum). +const ( + DeploymentStatusPending = "PENDING" + DeploymentStatusSuccessful = "SUCCESSFUL" + DeploymentStatusStopped = "STOPPED" + DeploymentStatusStopRequested = "STOP_REQUESTED" + DeploymentStatusInProgress = "IN_PROGRESS" + DeploymentStatusRollbackRequested = "ROLLBACK_REQUESTED" + DeploymentStatusRollbackInProgress = "ROLLBACK_IN_PROGRESS" + DeploymentStatusRollbackSuccessful = "ROLLBACK_SUCCESSFUL" + DeploymentStatusRollbackFailed = "ROLLBACK_FAILED" +) + +// Stop types accepted by StopServiceDeployment. +const ( + StopTypeAbort = "ABORT" + StopTypeRollback = "ROLLBACK" +) + +// ServiceConnectConfiguration is a service's Service Connect setting. Namespace +// is the Cloud Map namespace (name or ARN) the service joins; Raw is the whole +// configuration as supplied, echoed verbatim. +type ServiceConnectConfiguration struct { + Namespace string + Raw []byte +} + +// ServiceRevision is an immutable snapshot of the service configuration one +// deployment rolled out. +type ServiceRevision struct { + ARN string + ServiceARN string + ClusterARN string + TaskDefinition string + LaunchType string + PlatformVersion string + CapacityProviderStrategy []CapacityProviderStrategyItem + NetworkConfiguration *NetworkConfiguration + LoadBalancers []LoadBalancer + ServiceRegistries []ServiceRegistry + ServiceConnect *ServiceConnectConfiguration + CreatedAt string +} + +// ServiceRevisionSummary is a deployment's view of a revision with its task +// counts. +type ServiceRevisionSummary struct { + ARN string + RequestedTaskCount int + RunningTaskCount int + PendingTaskCount int +} + +// ServiceDeploymentRollback records why and to which revision a deployment +// rolled back. +type ServiceDeploymentRollback struct { + Reason string + ServiceRevisionARN string + StartedAt string +} + +// ServiceDeployment is one rollout of a service to a target service revision. +type ServiceDeployment struct { + // Sequence orders a service's deployments (higher = newer); it breaks ties + // between deployments created in the same instant and is not on the wire. + Sequence uint64 + ARN string + ServiceARN string + ClusterARN string + Status string + StatusReason string + LifecycleStage string + TargetServiceRevision ServiceRevisionSummary + SourceServiceRevisions []ServiceRevisionSummary + DeploymentConfiguration *DeploymentConfiguration + Rollback *ServiceDeploymentRollback + CreatedAt string + StartedAt string + FinishedAt string + StoppedAt string + UpdatedAt string +} + +// ListServiceDeploymentsInput filters and pages a service's deployments. +// MaxResults 0 means the default (20); the maximum is 100. CreatedAtBefore and +// CreatedAtAfter are RFC3339 instants (empty = unset). +type ListServiceDeploymentsInput struct { + Cluster string + Service string + Statuses []string + CreatedAtBefore string + CreatedAtAfter string + MaxResults int + NextToken string +} + +// ServiceDeployments exposes the deployment history of ECS-controller services. +type ServiceDeployments interface { + ListServiceDeployments(ctx context.Context, in ListServiceDeploymentsInput) ([]ServiceDeployment, string, error) + DescribeServiceDeployments(ctx context.Context, arns []string) ([]ServiceDeployment, []Failure, error) + DescribeServiceRevisions(ctx context.Context, arns []string) ([]ServiceRevision, []Failure, error) + StopServiceDeployment(ctx context.Context, arn, stopType string) (string, error) +} + +// ServiceNamespaces lists the services that joined a Service Connect namespace. +type ServiceNamespaces interface { + ListServicesByNamespace(ctx context.Context, namespace string, maxResults int, nextToken string) ([]string, string, error) +} diff --git a/services/ecs/driver/driver.go b/services/ecs/driver/driver.go index f1a1599e9..133725cf4 100644 --- a/services/ecs/driver/driver.go +++ b/services/ecs/driver/driver.go @@ -288,12 +288,35 @@ type Container struct { Reason string RuntimeID string NetworkBindings []NetworkBinding + // NetworkInterfaces lists the task ENI addresses of an awsvpc container. Its + // AttachmentID equals the ID of the task's ElasticNetworkInterface attachment. + NetworkInterfaces []ContainerNetworkInterface + // ManagedAgents lists the agents ECS manages in the container. A task started + // with enableExecuteCommand reports the ExecuteCommandAgent here. + ManagedAgents []ManagedAgent +} + +// ContainerNetworkInterface is one awsvpc network interface of a container. +type ContainerNetworkInterface struct { + AttachmentID string + PrivateIPv4Address string + IPv6Address string +} + +// ManagedAgent is an agent ECS runs inside a container (for example the +// ExecuteCommandAgent). LastStartedAt is an RFC3339 instant. +type ManagedAgent struct { + Name string + LastStatus string + LastStartedAt string + Reason string } // Attachment is a resource attached to a task, such as the elastic network // interface an awsvpc/Fargate task is placed behind. Details carries the // attachment-specific key/value pairs (e.g. networkInterfaceId, privateIPv4Address). type Attachment struct { + ID string Type string Status string Details []KeyValue @@ -397,6 +420,17 @@ type Task struct { StoppedAt string StoppedReason string StopCode string + // EnableExecuteCommand records whether the task was started with ECS Exec + // enabled. It is fixed at launch and cannot be changed on a running task. + EnableExecuteCommand bool + // ProtectionEnabled and ProtectionExpiresAt (RFC3339) are the task's + // scale-in protection, set by UpdateTaskProtection. Protection counts as + // active only until ProtectionExpiresAt. + ProtectionEnabled bool + ProtectionExpiresAt string + // EventVersion counts the task state changes published so far for the task, + // the detail.version of its last ECS Task State Change event. + EventVersion int // CPU and Memory are the task-level size (from the task definition, empty // when it sets none). CPU string @@ -426,6 +460,9 @@ type Deployment struct { RolloutStateReason string CreatedAt string UpdatedAt string + // ServiceConnect echoes the service's Service Connect configuration at the + // time of the deployment. + ServiceConnect *ServiceConnectConfiguration } // ServiceEvent is one entry of a service's event log. @@ -500,6 +537,11 @@ type Service struct { Deployments []Deployment Events []ServiceEvent Tags []Tag + // TaskSets is filled by DescribeServices for a service that uses the + // EXTERNAL deployment controller; it is not stored on the service. + TaskSets []TaskSet + // ServiceConnect is the service's Service Connect configuration, if any. + ServiceConnect *ServiceConnectConfiguration // AvailabilityZoneRebalancing is "ENABLED" or "DISABLED". Real ECS always // reports this field (defaulting new services to "DISABLED" when the // caller doesn't specify it), so it is never left empty on a stored @@ -523,7 +565,10 @@ type ContainerInstance struct { RegisteredMemory int RemainingCPU int RemainingMemory int - Tags []Tag + // Version counts the changes to the instance, the detail.version of its + // last ECS Container Instance State Change event. + Version int + Tags []Tag } // Failure describes a resource that could not be resolved in a batch @@ -576,6 +621,7 @@ type RunTaskInput struct { StartedBy string NetworkConfiguration *NetworkConfiguration CapacityProviderStrategy []CapacityProviderStrategyItem + EnableExecuteCommand bool Tags []Tag } @@ -599,6 +645,7 @@ type CreateServiceInput struct { CapacityProviderStrategy []CapacityProviderStrategyItem LoadBalancers []LoadBalancer ServiceRegistries []ServiceRegistry + ServiceConnect *ServiceConnectConfiguration Tags []Tag // AvailabilityZoneRebalancing is "ENABLED" or "DISABLED"; an empty value // defaults to "DISABLED" (matching real ECS). @@ -623,6 +670,7 @@ type UpdateServiceInput struct { CapacityProviderStrategy []CapacityProviderStrategyItem LoadBalancers []LoadBalancer ServiceRegistries []ServiceRegistry + ServiceConnect *ServiceConnectConfiguration // AvailabilityZoneRebalancing is "ENABLED" or "DISABLED"; an empty value // leaves the service's current setting unchanged. AvailabilityZoneRebalancing string diff --git a/services/ecs/ecs_test.go b/services/ecs/ecs_test.go index 9688d1454..300154e15 100644 --- a/services/ecs/ecs_test.go +++ b/services/ecs/ecs_test.go @@ -177,11 +177,13 @@ func TestWrapperWave4bFlow(t *testing.T) { require.NoError(t, err) mock2.SeedContainerInstance("prod", "i-exec") - tasks, _, err := e2.RunTask(ctx, driver.RunTaskInput{Cluster: "prod", TaskDefinition: "web"}) + tasks, _, err := e2.RunTask(ctx, driver.RunTaskInput{Cluster: "prod", TaskDefinition: "web", EnableExecuteCommand: true}) require.NoError(t, err) require.Len(t, tasks, 1) - exec, err := e2.ExecuteCommand(ctx, driver.ExecuteCommandInput{Cluster: "prod", Task: tasks[0].ARN, Command: "ls"}) + exec, err := e2.ExecuteCommand(ctx, driver.ExecuteCommandInput{ + Cluster: "prod", Task: tasks[0].ARN, Command: "ls", Interactive: true, + }) require.NoError(t, err) assert.NotEmpty(t, exec.Session.SessionID) @@ -228,3 +230,117 @@ func TestWrapperCapacityProviderFlow(t *testing.T) { assert.Positive(t, rec.CallCountFor("ecs", op), op) } } + +// TestWrapperOptionalCapabilities drives every optional capability through the +// portable wrapper and checks each call is recorded. +func TestWrapperOptionalCapabilities(t *testing.T) { + e, _, rec := newWrapper(t) + ctx := context.Background() + + _, err := e.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + + _, err = e.RegisterTaskDefinition(ctx, driver.RegisterTaskDefinitionInput{ + Family: "fg", + ContainerDefinitions: []driver.ContainerDefinition{{Name: "c", Image: "img", Essential: true}}, + CPU: "256", + Memory: "512", + NetworkMode: "awsvpc", + RequiresCompatibilities: []string{"FARGATE"}, + }) + require.NoError(t, err) + + netCfg := &driver.NetworkConfiguration{AwsVpcConfiguration: &driver.AwsVpcConfiguration{Subnets: []string{"subnet-1"}}} + + _, err = e.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "ext", Cluster: "prod", DeploymentController: "EXTERNAL", DesiredCount: 2, + }) + require.NoError(t, err) + + ts, err := e.CreateTaskSet(ctx, driver.CreateTaskSetInput{ + Cluster: "prod", Service: "ext", TaskDefinition: "fg", LaunchType: "FARGATE", NetworkConfiguration: netCfg, + }) + require.NoError(t, err) + + half := driver.Scale{Unit: driver.ScaleUnitPercent, Value: 50} + _, err = e.UpdateTaskSet(ctx, driver.UpdateTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: ts.ID, Scale: half}) + require.NoError(t, err) + + sets, failures, err := e.DescribeTaskSets(ctx, "prod", "ext", []string{ts.ID}) + require.NoError(t, err) + require.Empty(t, failures) + require.Len(t, sets, 1) + + _, err = e.UpdateServicePrimaryTaskSet(ctx, "prod", "ext", ts.ID) + require.NoError(t, err) + + _, err = e.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: ts.ID, Force: true}) + require.NoError(t, err) + + _, err = e.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "web", Cluster: "prod", TaskDefinition: "fg", LaunchType: "FARGATE", NetworkConfiguration: netCfg, DesiredCount: 1, + ServiceConnect: &driver.ServiceConnectConfiguration{Namespace: "mesh"}, + }) + require.NoError(t, err) + + tasks, err := e.ListTasks(ctx, "prod", "", "", "web") + require.NoError(t, err) + require.Len(t, tasks, 1) + + _, _, err = e.UpdateTaskProtection(ctx, driver.UpdateTaskProtectionInput{Cluster: "prod", Tasks: []string{tasks[0].ARN}, ProtectionEnabled: true}) + require.NoError(t, err) + + got, _, err := e.GetTaskProtection(ctx, "prod", []string{tasks[0].ARN}) + require.NoError(t, err) + assert.True(t, got[0].ProtectionEnabled) + + deps, next, err := e.ListServiceDeployments(ctx, driver.ListServiceDeploymentsInput{Cluster: "prod", Service: "web"}) + require.NoError(t, err) + require.Len(t, deps, 1) + assert.Empty(t, next) + + described, _, err := e.DescribeServiceDeployments(ctx, []string{deps[0].ARN}) + require.NoError(t, err) + assert.Len(t, described, 1) + + revs, _, err := e.DescribeServiceRevisions(ctx, []string{deps[0].TargetServiceRevision.ARN}) + require.NoError(t, err) + assert.Len(t, revs, 1) + + _, err = e.StopServiceDeployment(ctx, deps[0].ARN, driver.StopTypeRollback) + require.Error(t, err, "a completed deployment cannot be stopped") + + arns, _, err := e.ListServicesByNamespace(ctx, "mesh", 0, "") + require.NoError(t, err) + assert.Len(t, arns, 1) + + for _, op := range []string{ + "CreateTaskSet", "UpdateTaskSet", "DescribeTaskSets", "UpdateServicePrimaryTaskSet", "DeleteTaskSet", + "UpdateTaskProtection", "GetTaskProtection", "ListServiceDeployments", "DescribeServiceDeployments", + "DescribeServiceRevisions", "StopServiceDeployment", "ListServicesByNamespace", + } { + assert.Equal(t, 1, rec.CallCountFor("ecs", op), op) + } +} + +// bareDriver satisfies driver.ECS without any optional capability. +type bareDriver struct{ driver.ECS } + +// TestWrapperCapabilityMissing pins that a driver without a capability gets a +// typed FailedPrecondition, not a panic. +func TestWrapperCapabilityMissing(t *testing.T) { + e := ecs.NewECS(bareDriver{}) + ctx := context.Background() + + _, _, err := e.GetTaskProtection(ctx, "prod", []string{"t"}) + require.Error(t, err) + + _, err = e.CreateTaskSet(ctx, driver.CreateTaskSetInput{}) + require.Error(t, err) + + _, _, err = e.ListServiceDeployments(ctx, driver.ListServiceDeploymentsInput{}) + require.Error(t, err) + + _, _, err = e.ListServicesByNamespace(ctx, "ns", 0, "") + require.Error(t, err) +} From 2d4eddfa9759c163a31731054219ff8e63ed52f8 Mon Sep 17 00:00:00 2001 From: Gajendra Malviya Date: Sat, 10 Oct 2026 21:55:01 +0530 Subject: [PATCH 2/2] fix(aws-ecs): address review feedback on the ECS parity PR - DeleteTaskSet drains the set with or without force (Terraform destroy works) - StopServiceDeployment treats an omitted stopType as ROLLBACK - EXTERNAL services report running/pending counts from their task sets and reject task-set-level UpdateService fields (task definition, platform version, network configuration, load balancers, capacity providers, service registries) - stop publishing SERVICE_DESIRED_COUNT_UPDATED from UpdateService - one shared deployment-complete check for rollout state, event and record - task protection: stopped tasks are TASK_NOT_VALID and read as unprotected; protect runs under placeMu so it cannot revive a stopping task - placeFargate no longer stores a half-built task (data race with readers) - describe-container-instances returns version; event attributes is [] - replace the generic capability helper so the generated ECS docs list driver.ECS (41 operations) again - move ListServicesByNamespace handler to its own file, rename test files by feature, regenerate docs --- docs/coverage/README.md | 2 +- docs/coverage/aws/README.md | 2 +- docs/coverage/aws/ecs.md | 40 +-- docs/coverage/coverage.json | 270 +++++++++--------- docs/coverage/nongoals/ecs.md | 4 +- docs/services.md | 2 +- ...est.go => cluster_task_validation_test.go} | 0 providers/aws/ecs/concurrency_race_test.go | 102 +++++++ providers/aws/ecs/container_instances.go | 4 +- providers/aws/ecs/events.go | 19 +- providers/aws/ecs/events_test.go | 45 ++- ...arity_helpers_test.go => fixtures_test.go} | 0 providers/aws/ecs/metrics.go | 5 +- providers/aws/ecs/service_deployments.go | 9 +- providers/aws/ecs/service_deployments_test.go | 27 ++ providers/aws/ecs/services.go | 138 +++++++-- providers/aws/ecs/services_test.go | 18 ++ providers/aws/ecs/target_registration.go | 2 +- providers/aws/ecs/task_protection.go | 20 +- providers/aws/ecs/task_protection_test.go | 149 ++++++++++ providers/aws/ecs/task_sets.go | 11 +- providers/aws/ecs/task_sets_test.go | 213 +++++++++++++- providers/aws/ecs/tasks.go | 12 +- server/aws/ecs/capacity_providers.go | 4 +- server/aws/ecs/clusters.go | 2 +- server/aws/ecs/container_instances.go | 4 +- server/aws/ecs/handler.go | 2 +- server/aws/ecs/namespace.go | 41 +++ server/aws/ecs/sdk_roundtrip_test.go | 17 ++ server/aws/ecs/service_deployments.go | 33 +-- .../aws/ecs/service_deployments_sdk_test.go | 13 + server/aws/ecs/services.go | 2 +- server/aws/ecs/task_protection.go | 2 +- server/aws/ecs/task_protection_sdk_test.go | 12 + server/aws/ecs/task_sets.go | 10 +- server/aws/ecs/task_sets_sdk_test.go | 62 +++- server/aws/ecs/tasks.go | 2 +- server/aws/ecs/types.go | 2 + ...ion_sdk_test.go => validation_sdk_test.go} | 0 services/ecs/capabilities.go | 61 +++- services/ecs/driver/capabilities.go | 4 +- 41 files changed, 1084 insertions(+), 283 deletions(-) rename providers/aws/ecs/{validation_defaults_test.go => cluster_task_validation_test.go} (100%) rename providers/aws/ecs/{parity_helpers_test.go => fixtures_test.go} (100%) create mode 100644 server/aws/ecs/namespace.go rename server/aws/ecs/{parity_validation_sdk_test.go => validation_sdk_test.go} (100%) diff --git a/docs/coverage/README.md b/docs/coverage/README.md index 57d2a78db..7e23f62a3 100644 --- a/docs/coverage/README.md +++ b/docs/coverage/README.md @@ -80,7 +80,7 @@ code does not implement. Machine-readable: [`coverage.json`](./coverage.json). | `digitaltwins` | - | [DigitalTwins](./azure/digitaltwins.md) | - | - | 8 | | `disks` | - | [Disks](./azure/disks.md) | - | - | 7 | | `dns` | [Route53](./aws/route53.md) | [DNS](./azure/dns.md) | [CloudDNS](./gcp/clouddns.md) | - | 16 | -| `ecs` | [ECS](./aws/ecs.md) | - | - | - | 5 | +| `ecs` | [ECS](./aws/ecs.md) | - | - | - | 41 | | `efs` | [EFS](./aws/efs.md) | - | - | - | 27 | | `eks` | [EKS](./aws/eks.md) | - | - | - | 40 | | `elasticsan` | - | [ElasticSan](./azure/elasticsan.md) | - | - | 8 | diff --git a/docs/coverage/aws/README.md b/docs/coverage/aws/README.md index 37e976060..8f21f315f 100644 --- a/docs/coverage/aws/README.md +++ b/docs/coverage/aws/README.md @@ -31,7 +31,7 @@ Services cloudemu emulates for AWS, by native name. Back to the [cross-provider | [DynamoDB](./dynamodb.md) | `database` | 24 | | [EC2](./ec2.md) | `compute` | 37 | | [ECR](./ecr.md) | `containerregistry` | 15 | -| [ECS](./ecs.md) | `ecs` | 5 | +| [ECS](./ecs.md) | `ecs` | 41 | | [EFS](./efs.md) | `efs` | 27 | | [EKS](./eks.md) | (provider-native) | 40 | | [ELB](./elb.md) | `loadbalancer` | 19 | diff --git a/docs/coverage/aws/ecs.md b/docs/coverage/aws/ecs.md index 600bd5ee1..4a32e2826 100644 --- a/docs/coverage/aws/ecs.md +++ b/docs/coverage/aws/ecs.md @@ -1,25 +1,9 @@ # ECS -AWS's `ecs` service · portable interface `driver.TaskSets` · [AWS index](./README.md) +AWS's `ecs` service · portable interface `driver.ECS` · [AWS index](./README.md) -## Operations (5) - -| Operation | Description | -| --- | --- | -| `CreateTaskSet` | | -| `DeleteTaskSet` | | -| `DescribeTaskSets` | | -| `UpdateServicePrimaryTaskSet` | | -| `UpdateTaskSet` | | - -## Optional capabilities - -Discovered by type assertion; only some providers implement these. - -### ECS - -ECS is the interface an ECS provider implementation must satisfy. +## Operations (41) | Operation | Description | | --- | --- | @@ -65,6 +49,10 @@ ECS is the interface an ECS provider implementation must satisfy. | `UpdateContainerInstancesState` | | | `UpdateService` | | +## Optional capabilities + +Discovered by type assertion; only some providers implement these. + ### ServiceDeployments ServiceDeployments exposes the deployment history of ECS-controller services. @@ -93,14 +81,28 @@ TaskProtection reads and changes the scale-in protection of service tasks. | `GetTaskProtection` | | | `UpdateTaskProtection` | | +### TaskSets + +TaskSets manages task sets, the task groups of EXTERNAL-controller services. + +| Operation | Description | +| --- | --- | +| `CreateTaskSet` | | +| `DeleteTaskSet` | | +| `DescribeTaskSets` | | +| `UpdateServicePrimaryTaskSet` | | +| `UpdateTaskSet` | | + ## Not in scope - No workload runs unless a container engine is attached, so the usage-driven metrics (`CPUUtilization`, `MemoryUtilization`, network and storage series) are never published. Only metrics derived from state the emulator holds are: `AWS/ECS` `CPUReservation`, `MemoryReservation` and `LiveTaskCount`, and the `ECS/ContainerInsights` task, service, instance, deployment and task-set counts (only for clusters with `containerInsights` enabled or enhanced, or enabled on the account). - Task lifecycle events cover PROVISIONING (awsvpc tasks), PENDING, ACTIVATING, RUNNING, DEACTIVATING, STOPPING, DEPROVISIONING (awsvpc tasks) and STOPPED. The transient `DELETED` state is not published. A task counts as awsvpc when it has an ENI attachment, which today only Fargate tasks do. - Container-instance state-change events omit `versionInfo` (agent and Docker versions), since the emulator runs no agent. - `ECS Deployment State Change` events are published for the in-progress and completed states. The deployment circuit breaker is stored and echoed but never trips, so `SERVICE_DEPLOYMENT_FAILED` and automatic rollback do not happen. `SERVICE_TASK_PLACEMENT_FAILURE` is published when a service create or update leaves tasks unplaced, not when a replacement task cannot be placed. +- `SERVICE_DESIRED_COUNT_UPDATED` is not published: it signals scheduler or Service Auto Scaling changes, which the emulator does not model. - Service deployments and revisions exist for services that use the ECS deployment controller; they complete immediately, so `StopServiceDeployment` only succeeds while the deployment is still in progress, which is the `--async-settle` window. A stop of a completed deployment is a `ConflictException`. The lifecycle stage is not reported, and blue/green, linear and canary strategies, lifecycle hooks, alarms and `ContinueServiceDeployment` are not modelled. The newest 100 deployments of a service are kept. - Task sets run their tasks directly and replace a stopped task to restore `computedDesiredCount`. They are not registered with load balancer target groups or service discovery, and a capacity provider strategy on a task set is stored and echoed but not used for placement (the same as for services). - `ListServicesByNamespace` matches a service by the namespace string (name or ARN) it was configured with. There is no Cloud Map in the emulator to resolve one to the other, so an unknown namespace lists nothing instead of returning `NamespaceNotFoundException`. -- Where AWS does not document an exact message or exception, the emulator picks the closest documented exception and its own wording: a task-set call on a service that does not use the EXTERNAL controller, deleting a task set that is not scaled to zero without `force`, an invalid `launchType`, `ExecuteCommand` on a task that is not RUNNING, and `StopTask` on an already-stopped task (it keeps the original reason). +- Where AWS does not document an exact message or exception, the emulator picks the closest documented exception and its own wording: a task-set call on a service that does not use the EXTERNAL controller, an invalid `launchType`, `ExecuteCommand` on a task that is not RUNNING, and `StopTask` on an already-stopped task (it keeps the original reason). +- `DeleteTaskSet` drains the set immediately whether or not `force` is set. - `CreateCluster` on an existing ACTIVE name returns that cluster unchanged and ignores the tags and settings in the new request. diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index e6ab5f6ee..aa6a5bbf8 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -6587,154 +6587,133 @@ }, { "service": "ecs", - "interface": "TaskSets", + "interface": "ECS", "operations": [ { - "name": "CreateTaskSet" + "name": "CreateCapacityProvider" }, { - "name": "DeleteTaskSet" + "name": "CreateCluster" + }, + { + "name": "CreateService" + }, + { + "name": "DeleteAccountSetting" + }, + { + "name": "DeleteAttributes" + }, + { + "name": "DeleteCapacityProvider" + }, + { + "name": "DeleteCluster" + }, + { + "name": "DeleteService" + }, + { + "name": "DeregisterContainerInstance" + }, + { + "name": "DeregisterTaskDefinition" + }, + { + "name": "DescribeCapacityProviders" + }, + { + "name": "DescribeClusters" + }, + { + "name": "DescribeContainerInstances" + }, + { + "name": "DescribeServices" + }, + { + "name": "DescribeTaskDefinition" + }, + { + "name": "DescribeTasks" + }, + { + "name": "ExecuteCommand" + }, + { + "name": "ListAccountSettings" + }, + { + "name": "ListAttributes" + }, + { + "name": "ListClusters" + }, + { + "name": "ListContainerInstances" + }, + { + "name": "ListServices" + }, + { + "name": "ListTagsForResource" + }, + { + "name": "ListTaskDefinitionFamilies" + }, + { + "name": "ListTaskDefinitions" + }, + { + "name": "ListTasks" + }, + { + "name": "PutAccountSetting" + }, + { + "name": "PutAccountSettingDefault" + }, + { + "name": "PutAttributes" }, { - "name": "DescribeTaskSets" + "name": "PutClusterCapacityProviders" }, { - "name": "UpdateServicePrimaryTaskSet" + "name": "RegisterContainerInstance" + }, + { + "name": "RegisterTaskDefinition" + }, + { + "name": "RunTask" + }, + { + "name": "StopTask" + }, + { + "name": "TagResource" + }, + { + "name": "UntagResource" }, { - "name": "UpdateTaskSet" + "name": "UpdateCapacityProvider" + }, + { + "name": "UpdateCluster" + }, + { + "name": "UpdateClusterSettings" + }, + { + "name": "UpdateContainerInstancesState" + }, + { + "name": "UpdateService" } ], "optionalCapabilities": [ - { - "name": "ECS", - "doc": "ECS is the interface an ECS provider implementation must satisfy.", - "operations": [ - { - "name": "CreateCapacityProvider" - }, - { - "name": "CreateCluster" - }, - { - "name": "CreateService" - }, - { - "name": "DeleteAccountSetting" - }, - { - "name": "DeleteAttributes" - }, - { - "name": "DeleteCapacityProvider" - }, - { - "name": "DeleteCluster" - }, - { - "name": "DeleteService" - }, - { - "name": "DeregisterContainerInstance" - }, - { - "name": "DeregisterTaskDefinition" - }, - { - "name": "DescribeCapacityProviders" - }, - { - "name": "DescribeClusters" - }, - { - "name": "DescribeContainerInstances" - }, - { - "name": "DescribeServices" - }, - { - "name": "DescribeTaskDefinition" - }, - { - "name": "DescribeTasks" - }, - { - "name": "ExecuteCommand" - }, - { - "name": "ListAccountSettings" - }, - { - "name": "ListAttributes" - }, - { - "name": "ListClusters" - }, - { - "name": "ListContainerInstances" - }, - { - "name": "ListServices" - }, - { - "name": "ListTagsForResource" - }, - { - "name": "ListTaskDefinitionFamilies" - }, - { - "name": "ListTaskDefinitions" - }, - { - "name": "ListTasks" - }, - { - "name": "PutAccountSetting" - }, - { - "name": "PutAccountSettingDefault" - }, - { - "name": "PutAttributes" - }, - { - "name": "PutClusterCapacityProviders" - }, - { - "name": "RegisterContainerInstance" - }, - { - "name": "RegisterTaskDefinition" - }, - { - "name": "RunTask" - }, - { - "name": "StopTask" - }, - { - "name": "TagResource" - }, - { - "name": "UntagResource" - }, - { - "name": "UpdateCapacityProvider" - }, - { - "name": "UpdateCluster" - }, - { - "name": "UpdateClusterSettings" - }, - { - "name": "UpdateContainerInstancesState" - }, - { - "name": "UpdateService" - } - ] - }, { "name": "ServiceDeployments", "doc": "ServiceDeployments exposes the deployment history of ECS-controller services.", @@ -6773,6 +6752,27 @@ "name": "UpdateTaskProtection" } ] + }, + { + "name": "TaskSets", + "doc": "TaskSets manages task sets, the task groups of EXTERNAL-controller services.", + "operations": [ + { + "name": "CreateTaskSet" + }, + { + "name": "DeleteTaskSet" + }, + { + "name": "DescribeTaskSets" + }, + { + "name": "UpdateServicePrimaryTaskSet" + }, + { + "name": "UpdateTaskSet" + } + ] } ], "providers": { diff --git a/docs/coverage/nongoals/ecs.md b/docs/coverage/nongoals/ecs.md index 4acf38ed6..053394c8a 100644 --- a/docs/coverage/nongoals/ecs.md +++ b/docs/coverage/nongoals/ecs.md @@ -2,8 +2,10 @@ - Task lifecycle events cover PROVISIONING (awsvpc tasks), PENDING, ACTIVATING, RUNNING, DEACTIVATING, STOPPING, DEPROVISIONING (awsvpc tasks) and STOPPED. The transient `DELETED` state is not published. A task counts as awsvpc when it has an ENI attachment, which today only Fargate tasks do. - Container-instance state-change events omit `versionInfo` (agent and Docker versions), since the emulator runs no agent. - `ECS Deployment State Change` events are published for the in-progress and completed states. The deployment circuit breaker is stored and echoed but never trips, so `SERVICE_DEPLOYMENT_FAILED` and automatic rollback do not happen. `SERVICE_TASK_PLACEMENT_FAILURE` is published when a service create or update leaves tasks unplaced, not when a replacement task cannot be placed. +- `SERVICE_DESIRED_COUNT_UPDATED` is not published: it signals scheduler or Service Auto Scaling changes, which the emulator does not model. - Service deployments and revisions exist for services that use the ECS deployment controller; they complete immediately, so `StopServiceDeployment` only succeeds while the deployment is still in progress, which is the `--async-settle` window. A stop of a completed deployment is a `ConflictException`. The lifecycle stage is not reported, and blue/green, linear and canary strategies, lifecycle hooks, alarms and `ContinueServiceDeployment` are not modelled. The newest 100 deployments of a service are kept. - Task sets run their tasks directly and replace a stopped task to restore `computedDesiredCount`. They are not registered with load balancer target groups or service discovery, and a capacity provider strategy on a task set is stored and echoed but not used for placement (the same as for services). - `ListServicesByNamespace` matches a service by the namespace string (name or ARN) it was configured with. There is no Cloud Map in the emulator to resolve one to the other, so an unknown namespace lists nothing instead of returning `NamespaceNotFoundException`. -- Where AWS does not document an exact message or exception, the emulator picks the closest documented exception and its own wording: a task-set call on a service that does not use the EXTERNAL controller, deleting a task set that is not scaled to zero without `force`, an invalid `launchType`, `ExecuteCommand` on a task that is not RUNNING, and `StopTask` on an already-stopped task (it keeps the original reason). +- Where AWS does not document an exact message or exception, the emulator picks the closest documented exception and its own wording: a task-set call on a service that does not use the EXTERNAL controller, an invalid `launchType`, `ExecuteCommand` on a task that is not RUNNING, and `StopTask` on an already-stopped task (it keeps the original reason). +- `DeleteTaskSet` drains the set immediately whether or not `force` is set. - `CreateCluster` on an existing ACTIVE name returns that cluster unchanged and ignores the tags and settings in the new request. diff --git a/docs/services.md b/docs/services.md index dea78c36a..8889865ed 100644 --- a/docs/services.md +++ b/docs/services.md @@ -2391,7 +2391,7 @@ no usage-driven series. What is deliberately not emulated is listed in Task sets of services that use the `EXTERNAL` deployment controller. Each set runs its own tasks (`computedDesiredCount` = the service desired count x scale, rounded up), starts `ACTIVE`, and `UpdateServicePrimaryTaskSet` makes one `PRIMARY` (the previous primary becomes -`ACTIVE`). A set that is not scaled to zero needs `force` to be deleted. `DescribeServices` +`ACTIVE`). `DeleteTaskSet` drains the set immediately; `force` is accepted but has no visible effect. `DescribeServices` reports a service's task sets. | Operation | Signature | diff --git a/providers/aws/ecs/validation_defaults_test.go b/providers/aws/ecs/cluster_task_validation_test.go similarity index 100% rename from providers/aws/ecs/validation_defaults_test.go rename to providers/aws/ecs/cluster_task_validation_test.go diff --git a/providers/aws/ecs/concurrency_race_test.go b/providers/aws/ecs/concurrency_race_test.go index 631da95ad..6ff4c0fa2 100644 --- a/providers/aws/ecs/concurrency_race_test.go +++ b/providers/aws/ecs/concurrency_race_test.go @@ -122,3 +122,105 @@ func TestConcurrentServiceAccessRace(t *testing.T) { wg.Wait() } + +// fargateRaceFixture registers a Fargate awsvpc task definition in a fresh +// "prod" cluster and returns its network configuration. +func fargateRaceFixture(t *testing.T, m *Mock) *driver.NetworkConfiguration { + t.Helper() + + ctx := context.Background() + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + + _, err = m.RegisterTaskDefinition(ctx, driver.RegisterTaskDefinitionInput{ + Family: "web", NetworkMode: networkModeAwsvpc, RequiresCompatibilities: []string{launchFargate}, + CPU: "256", Memory: "512", + ContainerDefinitions: []driver.ContainerDefinition{{Name: "c", Image: "img", Essential: true}}, + }) + require.NoError(t, err) + + return &driver.NetworkConfiguration{AwsVpcConfiguration: &driver.AwsVpcConfiguration{Subnets: []string{"subnet-1"}}} +} + +// readTasksUntil starts readers that list and describe every task of "prod" +// until stop closes, and returns a func that waits for them. +func readTasksUntil(m *Mock, stop <-chan struct{}) func() { + var wg sync.WaitGroup + + for range 4 { + wg.Add(1) + + go func() { + defer wg.Done() + + for { + select { + case <-stop: + return + default: + } + + tasks, _ := m.ListTasks(context.Background(), "prod", "", "", "") + + arns := make([]string, 0, len(tasks)) + for i := range tasks { + arns = append(arns, tasks[i].ARN) + } + + if len(arns) > 0 && len(arns) <= 100 { + _, _, _ = m.DescribeTasks(context.Background(), "prod", arns) + } + } + }() + } + + return wg.Wait +} + +// TestConcurrentFargateRunTaskVsReadRace guards launchTask publishing a Fargate +// task to the store before stampLaunch finished writing it: a concurrent +// List/DescribeTasks clone then raced those writes under `go test -race`. +func TestConcurrentFargateRunTaskVsReadRace(t *testing.T) { + m := newTestMock() + ctx := context.Background() + netCfg := fargateRaceFixture(t, m) + + stop := make(chan struct{}) + wait := readTasksUntil(m, stop) + + for range 50 { + _, _, err := m.RunTask(ctx, driver.RunTaskInput{ + Cluster: "prod", TaskDefinition: "web", LaunchType: launchFargate, NetworkConfiguration: netCfg, Count: 1, + }) + require.NoError(t, err) + } + + close(stop) + wait() +} + +// TestConcurrentFargateForceDeployVsReadRace is the same race reached through +// UpdateService ForceNewDeployment launching replacement Fargate tasks. +func TestConcurrentFargateForceDeployVsReadRace(t *testing.T) { + m := newTestMock() + ctx := context.Background() + netCfg := fargateRaceFixture(t, m) + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "svc", Cluster: "prod", TaskDefinition: "web", LaunchType: launchFargate, + NetworkConfiguration: netCfg, DesiredCount: 2, + }) + require.NoError(t, err) + + stop := make(chan struct{}) + wait := readTasksUntil(m, stop) + + for range 50 { + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Cluster: "prod", Service: "svc", ForceNewDeployment: true}) + require.NoError(t, err) + } + + close(stop) + wait() +} diff --git a/providers/aws/ecs/container_instances.go b/providers/aws/ecs/container_instances.go index 0ada2f3d7..2477f35fc 100644 --- a/providers/aws/ecs/container_instances.go +++ b/providers/aws/ecs/container_instances.go @@ -291,11 +291,11 @@ func capacityFromResources(resources []driver.Resource) (cpu, memory int) { for i := range resources { switch resources[i].Name { - case "CPU": + case resourceCPU: if v := resourceInt(&resources[i]); v > 0 { cpu = v } - case "MEMORY": + case resourceMemory: if v := resourceInt(&resources[i]); v > 0 { memory = v } diff --git a/providers/aws/ecs/events.go b/providers/aws/ecs/events.go index ea89599ee..7490c99d3 100644 --- a/providers/aws/ecs/events.go +++ b/providers/aws/ecs/events.go @@ -21,7 +21,6 @@ const ( serviceEventTypeError = "ERROR" serviceSteadyStateName = "SERVICE_STEADY_STATE" - serviceDesiredCountUpdated = "SERVICE_DESIRED_COUNT_UPDATED" serviceDeploymentInProgress = "SERVICE_DEPLOYMENT_IN_PROGRESS" serviceDeploymentCompleted = "SERVICE_DEPLOYMENT_COMPLETED" serviceTaskPlacementFailure = "SERVICE_TASK_PLACEMENT_FAILURE" @@ -369,7 +368,7 @@ func (m *Mock) emitDeploymentEvents(ctx context.Context, svc *driver.Service) { m.emitDeploymentChange(ctx, svc, id, serviceDeploymentInProgress, "ECS deployment "+id+" in progress.") - if svc.RunningCount >= svc.DesiredCount { + if deploymentComplete(svc.RunningCount, svc.DesiredCount) { m.emitDeploymentChange(ctx, svc, id, serviceDeploymentCompleted, "ECS deployment "+id+" completed.") } } @@ -407,12 +406,18 @@ func (m *Mock) emitInstanceChangeOfTask(ctx context.Context, t *driver.Task) { } } +// Names of the INTEGER resources a container instance registers. +const ( + resourceCPU = "CPU" + resourceMemory = "MEMORY" +) + // emitContainerInstanceDetail publishes an "ECS Container Instance State Change" // event for the instance as given (its Version already counts this change). func (m *Mock) emitContainerInstanceDetail(ctx context.Context, ci *driver.ContainerInstance) { cluster := instanceClusterName(ci.ARN) - var attrs []containerInstanceAttr + attrs := []containerInstanceAttr{} for _, a := range m.attributes.SortedValues() { if a.TargetID == ci.ARN { @@ -425,12 +430,12 @@ func (m *Mock) emitContainerInstanceDetail(ctx context.Context, ci *driver.Conta ClusterArn: m.arnIn(arnRegion(ci.ARN, m.opts.Region), "cluster/"+cluster), ContainerInstanceArn: ci.ARN, EC2InstanceID: ci.EC2InstanceID, RegisteredResources: []containerInstanceResource{ - {Name: "CPU", Type: containerInstanceResourceType, IntegerValue: ci.RegisteredCPU}, - {Name: "MEMORY", Type: containerInstanceResourceType, IntegerValue: ci.RegisteredMemory}, + {Name: resourceCPU, Type: containerInstanceResourceType, IntegerValue: ci.RegisteredCPU}, + {Name: resourceMemory, Type: containerInstanceResourceType, IntegerValue: ci.RegisteredMemory}, }, RemainingResources: []containerInstanceResource{ - {Name: "CPU", Type: containerInstanceResourceType, IntegerValue: ci.RemainingCPU}, - {Name: "MEMORY", Type: containerInstanceResourceType, IntegerValue: ci.RemainingMemory}, + {Name: resourceCPU, Type: containerInstanceResourceType, IntegerValue: ci.RemainingCPU}, + {Name: resourceMemory, Type: containerInstanceResourceType, IntegerValue: ci.RemainingMemory}, }, Status: ci.Status, Version: ci.Version, UpdatedAt: m.now(), }, ci.ARN) diff --git a/providers/aws/ecs/events_test.go b/providers/aws/ecs/events_test.go index 1b2f2efa9..b9c82ef3d 100644 --- a/providers/aws/ecs/events_test.go +++ b/providers/aws/ecs/events_test.go @@ -2,6 +2,7 @@ package ecs import ( "context" + "encoding/json" "sync/atomic" "testing" "time" @@ -243,6 +244,31 @@ func TestContainerInstanceStateChange(t *testing.T) { assert.Equal(t, statusInactive, got[4].Status) } +func TestContainerInstanceStateChange_EmptyAttributesIsNotNull(t *testing.T) { + m := newTestMock() + rec := &recordingPublisher{} + m.SetEventPublisher(rec) + ctx := context.Background() + + _, err := m.CreateCluster(ctx, driver.CreateClusterInput{Name: "prod"}) + require.NoError(t, err) + + _, err = m.RegisterContainerInstance(ctx, driver.RegisterContainerInstanceInput{Cluster: "prod"}) + require.NoError(t, err) + + evs := eventsOf(rec, eventContainerInstanceStateChange) + require.Len(t, evs, 1) + + d, ok := evs[0].Detail.(containerInstanceDetail) + require.True(t, ok) + assert.NotNil(t, d.Attributes, "no attributes must marshal as [] not null") + assert.Empty(t, d.Attributes) + + raw, err := json.Marshal(d) + require.NoError(t, err) + assert.Contains(t, string(raw), `"attributes":[]`) +} + func ciResourceInt(rs []containerInstanceResource, name string) int { for _, r := range rs { if r.Name == name { @@ -253,6 +279,23 @@ func ciResourceInt(rs []containerInstanceResource, name string) int { return -1 } +func TestExternalDesiredCountUpdate_NoDesiredCountEvent(t *testing.T) { + m := newTestMock() + rec := &recordingPublisher{} + m.SetEventPublisher(rec) + ctx := context.Background() + + externalFixture(t, m, 1) + + three := 3 + _, err := m.UpdateService(ctx, driver.UpdateServiceInput{Service: "ext", Cluster: "prod", DesiredCount: &three}) + require.NoError(t, err) + + for _, e := range eventsOf(rec, eventServiceAction) { + assert.NotEqual(t, "SERVICE_DESIRED_COUNT_UPDATED", e.Detail.(serviceActionDetail).EventName) + } +} + func TestServiceActionEvents(t *testing.T) { m := newTestMock() rec := &recordingPublisher{} @@ -275,7 +318,7 @@ func TestServiceActionEvents(t *testing.T) { names = append(names, e.Detail.(serviceActionDetail).EventName) } - assert.Contains(t, names, "SERVICE_DESIRED_COUNT_UPDATED") + assert.NotContains(t, names, "SERVICE_DESIRED_COUNT_UPDATED", "a user-driven count change is not published") // EC2 service on a cluster too small for its task: placement failure. _, err = m.CreateCluster(ctx, driver.CreateClusterInput{Name: "small"}) diff --git a/providers/aws/ecs/parity_helpers_test.go b/providers/aws/ecs/fixtures_test.go similarity index 100% rename from providers/aws/ecs/parity_helpers_test.go rename to providers/aws/ecs/fixtures_test.go diff --git a/providers/aws/ecs/metrics.go b/providers/aws/ecs/metrics.go index b603a9f26..ffcb04646 100644 --- a/providers/aws/ecs/metrics.go +++ b/providers/aws/ecs/metrics.go @@ -23,6 +23,7 @@ const ( metricUnitCount = "Count" metricUnitPercent = "Percent" percentOf = 100.0 + dimClusterName = "ClusterName" settingContainerInsights = "containerInsights" insightsEnabled = "enabled" @@ -100,7 +101,7 @@ func (m *Mock) publishClusterMetrics(cluster string) { // container instances and, with Container Insights, the instance, service and // task counts. func (m *Mock) addClusterMetrics(b *metricBatch, cluster string, insights bool) { - dims := map[string]string{"ClusterName": cluster} + dims := map[string]string{dimClusterName: cluster} if cpuPct, memPct, ok := m.reservationPercent(cluster); ok { b.add(metricsNamespaceECS, "CPUReservation", metricUnitPercent, cpuPct, dims) @@ -123,7 +124,7 @@ func (m *Mock) addClusterMetrics(b *metricBatch, cluster string, insights bool) // counts. func (m *Mock) addServiceMetrics(b *metricBatch, cluster string, svc *driver.Service, insights bool) { running, pending := m.liveServiceTaskCounts(cluster, serviceGroup(svc.Name)) - dims := map[string]string{"ClusterName": cluster, "ServiceName": svc.Name} + dims := map[string]string{dimClusterName: cluster, "ServiceName": svc.Name} if running > 0 { b.add(metricsNamespaceECS, "LiveTaskCount", metricUnitCount, float64(running), dims) diff --git a/providers/aws/ecs/service_deployments.go b/providers/aws/ecs/service_deployments.go index 86d1c5088..65abb4387 100644 --- a/providers/aws/ecs/service_deployments.go +++ b/providers/aws/ecs/service_deployments.go @@ -104,7 +104,7 @@ func (m *Mock) addDeploymentLocked(svc *driver.Service) { dep.SourceServiceRevisions = []driver.ServiceRevisionSummary{{ARN: prior[0].TargetServiceRevision.ARN}} } - if svc.RunningCount < svc.DesiredCount { + if !deploymentComplete(svc.RunningCount, svc.DesiredCount) { dep.Status, dep.FinishedAt = driver.DeploymentStatusInProgress, "" } @@ -393,8 +393,13 @@ func (m *Mock) DescribeServiceRevisions(_ context.Context, arns []string) ([]dri // ROLLBACK_SUCCESSFUL; ABORT ends STOPPED and leaves the service as it is. With // --async-settle the transient ROLLBACK_IN_PROGRESS / STOP_REQUESTED status is // visible first and a repeated stop continues as-is; a completed deployment is a -// ConflictException. +// ConflictException. stopType is optional on the wire: an omitted stopType +// means ROLLBACK. func (m *Mock) StopServiceDeployment(ctx context.Context, arn, stopType string) (string, error) { + if stopType == "" { + stopType = driver.StopTypeRollback + } + if stopType != driver.StopTypeAbort && stopType != driver.StopTypeRollback { return "", apiErrf(errors.InvalidArgument, excInvalidParameter, "stopType must be %s or %s.", driver.StopTypeAbort, driver.StopTypeRollback) diff --git a/providers/aws/ecs/service_deployments_test.go b/providers/aws/ecs/service_deployments_test.go index 14424157f..ee2df9bb5 100644 --- a/providers/aws/ecs/service_deployments_test.go +++ b/providers/aws/ecs/service_deployments_test.go @@ -279,6 +279,27 @@ func TestStopServiceDeployment_Rollback_AsyncWindow(t *testing.T) { assert.Len(t, after, 2, "a rollback does not record a new deployment") } +func TestStopServiceDeployment_OmittedStopTypeIsRollback(t *testing.T) { + m, clock, deps := asyncDeploymentFixture(t) + ctx := context.Background() + + arn, err := m.StopServiceDeployment(ctx, deps[0].ARN, "") + require.NoError(t, err) + assert.Equal(t, deps[0].ARN, arn) + + clock.Advance(time.Minute) + + got, _, err := m.DescribeServiceDeployments(ctx, []string{deps[0].ARN}) + require.NoError(t, err) + assert.Equal(t, driver.DeploymentStatusRollbackSuccessful, got[0].Status) + require.NotNil(t, got[0].Rollback) + assert.Equal(t, deps[1].TargetServiceRevision.ARN, got[0].Rollback.ServiceRevisionARN) + + svc, _, err := m.DescribeServices(ctx, "prod", []string{"web"}) + require.NoError(t, err) + assert.Contains(t, svc[0].TaskDefinition, "task-definition/fg:1", "the service is back on the source revision") +} + func TestStopServiceDeployment_Abort_AsyncWindow(t *testing.T) { m, clock, deps := asyncDeploymentFixture(t) ctx := context.Background() @@ -313,6 +334,12 @@ func TestStopServiceDeployment_ConflictAndNotFound(t *testing.T) { _, err = m.StopServiceDeployment(ctx, "arn:aws:ecs:us-east-1:000000000000:service-deployment/prod/web/nope", driver.StopTypeAbort) assert.Equal(t, excServiceDeploymentNotFound, ecsException(t, err)) + _, err = m.StopServiceDeployment(ctx, "arn:aws:ecs:us-east-1:000000000000:service-deployment/prod/web/nope", "") + assert.Equal(t, excServiceDeploymentNotFound, ecsException(t, err), "an omitted stopType still resolves the ARN first") + + _, err = m.StopServiceDeployment(ctx, deps[0].ARN, "") + assert.Equal(t, excConflict, ecsException(t, err), "an omitted stopType on a completed deployment") + _, err = m.StopServiceDeployment(ctx, deps[0].ARN, "HALT") assert.Equal(t, excInvalidParameter, ecsException(t, err)) } diff --git a/providers/aws/ecs/services.go b/providers/aws/ecs/services.go index ed458fe53..9e69d74b1 100644 --- a/providers/aws/ecs/services.go +++ b/providers/aws/ecs/services.go @@ -3,6 +3,8 @@ package ecs import ( "context" "fmt" + "reflect" + "slices" "strings" "github.com/stackshy/cloudemu/v2/errors" @@ -645,12 +647,18 @@ func (m *Mock) newDeployment(id, status string, svc *driver.Service, running, pe } } +// deploymentComplete reports whether a deployment has converged: the service +// runs exactly its desired count. Scale-in-protected tasks can keep it above. +func deploymentComplete(running, desired int) bool { + return running == desired +} + // rolloutState reports COMPLETED once the running count equals the desired // count, else IN_PROGRESS (running fewer tasks, or more because scale-in // protection kept surplus ones). Circuit-breaker FAILED transitions are accepted but // not simulated. func rolloutState(running, desired int) string { - if running == desired { + if deploymentComplete(running, desired) { return rolloutCompleted } @@ -662,10 +670,14 @@ func (m *Mock) serviceEvent(message string) driver.ServiceEvent { return driver.ServiceEvent{ID: m.hexID(), CreatedAt: m.now(), Message: message} } -// UpdateService updates a service. It stores/echoes every supplied field and, -// when the task definition or desired count changes (or forceNewDeployment is -// set), promotes a new PRIMARY deployment, drains the previous one, relaunches -// the tasks against the new target/definition, and appends an event. +// UpdateService updates a service. It applies every supplied field and, when +// the task definition or desired count changes (or forceNewDeployment is set), +// promotes a new PRIMARY deployment, drains the previous one, relaunches the +// tasks against the new target/definition, and appends an event. On an +// EXTERNAL-controller service, task-set-only fields (task definition, platform +// version, network configuration, load balancers, capacity providers, service +// registries) that differ from the stored values are rejected instead, and the +// rest is applied without a deployment (see validateExternalServiceUpdate). // //nolint:gocritic // in matches the driver.ECS interface signature; copied once on entry. func (m *Mock) UpdateService(ctx context.Context, in driver.UpdateServiceInput) (*driver.Service, error) { @@ -683,14 +695,27 @@ func (m *Mock) UpdateService(ctx context.Context, in driver.UpdateServiceInput) in.NetworkConfiguration = withDefaultAssignPublicIP(in.NetworkConfiguration) if svc.DeploymentController == deployControllerExternal { + if err := m.validateExternalServiceUpdate(svc, &in); err != nil { + return nil, err + } + return m.updateExternalService(ctx, svc, &in) } + return m.updateManagedService(ctx, cluster, svc, &in) +} + +// updateManagedService applies an UpdateService to a service whose deployments +// the scheduler manages (ECS or CODE_DEPLOY controller): it merges the supplied +// fields, redeploys when needed, and records the deployment and its events. +func (m *Mock) updateManagedService( + ctx context.Context, cluster string, svc *driver.Service, in *driver.UpdateServiceInput, +) (*driver.Service, error) { updated := cloneService(svc) - applyServiceScalars(&updated, &in) - applyServiceRefs(&updated, &in) + applyServiceScalars(&updated, in) + applyServiceRefs(&updated, in) - tdChanged, err := m.applyTaskDefChange(&updated, svc, &in) + tdChanged, err := m.applyTaskDefChange(&updated, svc, in) if err != nil { return nil, err } @@ -701,18 +726,18 @@ func (m *Mock) UpdateService(ctx context.Context, in driver.UpdateServiceInput) var events pendingTaskEvents if redeployed { - m.redeployService(ctx, &updated, &in, &events) + m.redeployService(ctx, &updated, in, &events) } m.services.Set(serviceKey(cluster, updated.Name), &updated) - deployed := in.ForceNewDeployment || tdChanged || serviceConfigChanged(&in) + deployed := in.ForceNewDeployment || tdChanged || serviceConfigChanged(in) if deployed { m.recordServiceDeployment(&updated) } m.publish(ctx, &events) - m.emitServiceUpdateEvents(ctx, &updated, countChanged, deployed, redeployed) + m.emitServiceUpdateEvents(ctx, &updated, deployed, redeployed) m.publishClusterMetrics(cluster) out := cloneService(&updated) @@ -732,13 +757,71 @@ func validateServiceUpdate(svc *driver.Service, in *driver.UpdateServiceInput) e return validateAssignPublicIP(in.NetworkConfiguration) } -// emitServiceUpdateEvents publishes the events a committed UpdateService caused: -// a desired-count change, a started deployment and the steady state it reached. -func (m *Mock) emitServiceUpdateEvents(ctx context.Context, svc *driver.Service, countChanged, deployed, redeployed bool) { - if countChanged { - m.emitServiceAction(ctx, svc.ARN, svc.ClusterARN, serviceDesiredCountUpdated, serviceEventTypeInfo, "") +// validateExternalServiceUpdate rejects an UpdateService on an EXTERNAL-controller +// service that would change a field only task sets can change (task definition, +// platform version, network configuration, load balancers, capacity providers, +// service registries). A field is rejected only when it is supplied and differs +// from the stored value, so a client re-sending unchanged values (Terraform) still +// succeeds. The whole call is rejected, so nothing else is applied. +func (m *Mock) validateExternalServiceUpdate(svc *driver.Service, in *driver.UpdateServiceInput) error { + changed := "" + + if in.TaskDefinition != "" { + td, ok := m.resolveTaskDef(in.TaskDefinition) + if !ok { + return apiErrf(errors.NotFound, excClient, "task definition %q not found", in.TaskDefinition) + } + + if td.ARN != svc.TaskDefinition { + changed = "taskDefinition" + } + } + + if changed == "" { + changed = changedTaskSetField(svc, in) + } + + if changed == "" { + return nil + } + + return apiErrf(errors.InvalidArgument, excInvalidParameter, + "Unable to update %s on services with an EXTERNAL deployment controller. Create a new task set instead.", changed) +} + +// changedTaskSetField returns the name of the first supplied task-set-only field +// (other than the task definition) whose value differs from the service's, or "" +// when none does. +func changedTaskSetField(svc *driver.Service, in *driver.UpdateServiceInput) string { + switch { + case in.PlatformVersion != "" && in.PlatformVersion != svc.PlatformVersion: + return "platformVersion" + case in.NetworkConfiguration != nil && !reflect.DeepEqual(in.NetworkConfiguration, svc.NetworkConfiguration): + return "networkConfiguration" + } + + return changedTaskSetList(svc, in) +} + +// changedTaskSetList is changedTaskSetField for the list-typed fields. A nil list +// is "not supplied"; an empty non-nil list is a removal, so it differs from a +// non-empty stored list. +func changedTaskSetList(svc *driver.Service, in *driver.UpdateServiceInput) string { + switch { + case in.LoadBalancers != nil && !slices.Equal(in.LoadBalancers, svc.LoadBalancers): + return "loadBalancers" + case in.CapacityProviderStrategy != nil && !slices.Equal(in.CapacityProviderStrategy, svc.CapacityProviderStrategy): + return "capacityProviderStrategy" + case in.ServiceRegistries != nil && !slices.Equal(in.ServiceRegistries, svc.ServiceRegistries): + return "serviceRegistries" } + return "" +} + +// emitServiceUpdateEvents publishes the events a committed UpdateService caused: +// a started deployment and the steady state it reached. +func (m *Mock) emitServiceUpdateEvents(ctx context.Context, svc *driver.Service, deployed, redeployed bool) { if deployed { m.emitDeploymentEvents(ctx, svc) } @@ -785,6 +868,7 @@ func (m *Mock) updateExternalService(ctx context.Context, svc *driver.Service, i } out := cloneService(&updated) + applyTaskSetCounts(&out, m.taskSetViews(&updated)) return &out, nil }() @@ -794,10 +878,6 @@ func (m *Mock) updateExternalService(ctx context.Context, svc *driver.Service, i m.publish(ctx, &events) - if in.DesiredCount != nil && *in.DesiredCount != svc.DesiredCount { - m.emitServiceAction(ctx, out.ARN, out.ClusterARN, serviceDesiredCountUpdated, serviceEventTypeInfo, "") - } - m.publishClusterMetrics(cluster) return out, nil @@ -971,6 +1051,7 @@ func (m *Mock) DescribeServices(ctx context.Context, cluster string, ids []strin out := cloneService(s) out.Tags = m.liveTags(s.ARN, s.Tags) out.TaskSets = m.describedTaskSets(want, s) + applyTaskSetCounts(&out, out.TaskSets) found = append(found, out) continue } @@ -984,6 +1065,23 @@ func (m *Mock) DescribeServices(ctx context.Context, cluster string, ids []strin return found, failures, nil } +// applyTaskSetCounts sets an EXTERNAL service's running/pending counts to the +// sum over its task sets. The stored record never carries them because the +// tasks belong to the task sets, so they are computed whenever the service is +// read. +func applyTaskSetCounts(svc *driver.Service, sets []driver.TaskSet) { + if svc.DeploymentController != deployControllerExternal || svc.Status != statusActive { + return + } + + svc.RunningCount, svc.PendingCount = 0, 0 + + for i := range sets { + svc.RunningCount += sets[i].RunningCount + svc.PendingCount += sets[i].PendingCount + } +} + // describedTaskSets returns the task sets DescribeServices reports for svc: the // refreshed sets of an ACTIVE EXTERNAL-controller service, none otherwise. func (m *Mock) describedTaskSets(cluster string, svc *driver.Service) []driver.TaskSet { diff --git a/providers/aws/ecs/services_test.go b/providers/aws/ecs/services_test.go index 8a9665459..6505b1cb9 100644 --- a/providers/aws/ecs/services_test.go +++ b/providers/aws/ecs/services_test.go @@ -309,3 +309,21 @@ func TestStopTask_StandaloneTaskNoServiceGroupNoop(t *testing.T) { _, err = m.StopTask(ctx, "prod", tasks[0].ARN, "standalone stop again") require.NoError(t, err) } + +func TestDeploymentComplete(t *testing.T) { + tests := []struct { + name string + running, desire int + want bool + }{ + {"below desired", 1, 2, false}, + {"equal to desired", 2, 2, true}, + {"above desired", 3, 2, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, deploymentComplete(tc.running, tc.desire)) + }) + } +} diff --git a/providers/aws/ecs/target_registration.go b/providers/aws/ecs/target_registration.go index 8c4b027c3..72d2cda6e 100644 --- a/providers/aws/ecs/target_registration.go +++ b/providers/aws/ecs/target_registration.go @@ -112,7 +112,7 @@ func eniPrivateIP(attachments []driver.Attachment) string { } for _, kv := range attachments[i].Details { - if kv.Name == "privateIPv4Address" { + if kv.Name == detailPrivateIPv4Address { return kv.Value } } diff --git a/providers/aws/ecs/task_protection.go b/providers/aws/ecs/task_protection.go index fd3168dc4..460aa470a 100644 --- a/providers/aws/ecs/task_protection.go +++ b/providers/aws/ecs/task_protection.go @@ -26,9 +26,10 @@ const ( var _ driver.TaskProtection = (*Mock)(nil) // taskProtected reports whether t's scale-in protection is active at now: -// enabled and not past its expiry. +// enabled and not past its expiry. A stopped task is never protected, whatever +// protection it carried when it stopped. func taskProtected(t *driver.Task, now time.Time) bool { - if !t.ProtectionEnabled { + if !t.ProtectionEnabled || t.LastStatus == statusStopped { return false } @@ -126,15 +127,28 @@ func (m *Mock) UpdateTaskProtection( } // protectOne applies a protection change to one task under protectMu and -// returns its new state, or the failure that kept it from applying. +// returns its new state, or the failure that kept it from applying. It also +// holds placeMu (the lock StopTask writes STOPPED under) so the read, check and +// write are one step: a stop cannot land between them and be overwritten with +// the stale RUNNING copy. Lock order is protectMu, then placeMu. func (m *Mock) protectOne( cluster, id string, enable bool, expires string, now time.Time, ) (*driver.ProtectedTask, *driver.Failure) { + m.placeMu.Lock() + defer m.placeMu.Unlock() + t, failure := m.protectableTask(cluster, id) if failure != nil { return nil, failure } + if enable && t.LastStatus == statusStopped { + return nil, &driver.Failure{ + ARN: t.ARN, Reason: failureTaskNotValid, + Detail: "The task is stopped; only running service tasks can be protected.", + } + } + if enable && m.protectionBlocked(t, now) { return nil, &driver.Failure{ ARN: id, Reason: failureDeploymentBlocked, diff --git a/providers/aws/ecs/task_protection_test.go b/providers/aws/ecs/task_protection_test.go index 916b37e34..2a61ce4aa 100644 --- a/providers/aws/ecs/task_protection_test.go +++ b/providers/aws/ecs/task_protection_test.go @@ -223,6 +223,65 @@ func TestScaleIn_SkipsProtectedTasks(t *testing.T) { assert.Equal(t, rolloutCompleted, svc.Deployments[0].RolloutState) } +func TestForceDeploy_ProtectedSurplusStaysInProgress(t *testing.T) { + m := newTestMock() + rec := &recordingPublisher{} + m.SetEventPublisher(rec) + ctx := context.Background() + arns := protectionFixture(t, m, 3) + + protect(t, m, nil, arns[0], arns[1]) + + one := 1 + _, err := m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", DesiredCount: &one}) + require.NoError(t, err) + + completed := func() int { + n := 0 + + for _, e := range eventsOf(rec, eventDeploymentStateChange) { + if e.Detail.(deploymentStateChangeDetail).EventName == serviceDeploymentCompleted { + n++ + } + } + + for _, e := range eventsOf(rec, eventServiceAction) { + if e.Detail.(serviceActionDetail).EventName == serviceDeploymentCompleted { + n++ + } + } + + return n + } + newest := func() driver.ServiceDeployment { + deps, _, listErr := m.ListServiceDeployments(ctx, driver.ListServiceDeploymentsInput{Cluster: "prod", Service: "web"}) + require.NoError(t, listErr) + require.NotEmpty(t, deps) + + return deps[0] + } + + before := completed() + + svc, err := m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", ForceNewDeployment: true}) + require.NoError(t, err) + assert.Equal(t, rolloutInProgress, svc.Deployments[0].RolloutState) + assert.Equal(t, before, completed(), "a deployment above desired is not completed") + assert.Equal(t, driver.DeploymentStatusInProgress, newest().Status) + assert.Empty(t, newest().FinishedAt) + + _, _, err = m.UpdateTaskProtection(ctx, driver.UpdateTaskProtectionInput{ + Cluster: "prod", Tasks: []string{arns[0], arns[1]}, ProtectionEnabled: false, + }) + require.NoError(t, err) + + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "web", Cluster: "prod", ForceNewDeployment: true}) + require.NoError(t, err) + assert.Equal(t, before+2, completed(), "the settled deployment publishes both completed events") + assert.Equal(t, driver.DeploymentStatusSuccessful, newest().Status) + assert.NotEmpty(t, newest().FinishedAt) +} + func TestUpdateTaskProtection_DeploymentBlocked(t *testing.T) { m := newTestMock() arns := protectionFixture(t, m, 2) @@ -276,3 +335,93 @@ func TestTaskProtection_ParallelWithStop(t *testing.T) { wg.Wait() } + +func TestUpdateTaskProtection_StoppedTask(t *testing.T) { + m := newTestMock() + ctx := context.Background() + arns := protectionFixture(t, m, 1) + + _, err := m.StopTask(ctx, "prod", arns[0], "done") + require.NoError(t, err) + + out, failures := protect(t, m, nil, arns...) + assert.Empty(t, out) + require.Len(t, failures, 1) + assert.Equal(t, arns[0], failures[0].ARN) + assert.Equal(t, failureTaskNotValid, failures[0].Reason) + assert.Contains(t, failures[0].Detail, "stopped") + + stored, ok := m.tasks.Get(arns[0]) + require.True(t, ok) + assert.Equal(t, statusStopped, stored.LastStatus) + assert.False(t, stored.ProtectionEnabled) + + out, failures, err = m.UpdateTaskProtection(ctx, driver.UpdateTaskProtectionInput{ + Cluster: "prod", Tasks: arns, ProtectionEnabled: false, + }) + require.NoError(t, err) + require.Empty(t, failures) + require.Len(t, out, 1) + assert.False(t, out[0].ProtectionEnabled) +} + +func TestGetTaskProtection_StoppedAfterProtected(t *testing.T) { + m := newTestMock() + ctx := context.Background() + arns := protectionFixture(t, m, 1) + + out, failures := protect(t, m, nil, arns...) + require.Empty(t, failures) + require.True(t, out[0].ProtectionEnabled) + + _, err := m.StopTask(ctx, "prod", arns[0], "done") + require.NoError(t, err) + + got, failures, err := m.GetTaskProtection(ctx, "prod", arns) + require.NoError(t, err) + require.Empty(t, failures) + require.Len(t, got, 1) + assert.False(t, got[0].ProtectionEnabled) + assert.Empty(t, got[0].ExpirationDate) +} + +// TestTaskProtection_EnableRacingStopKeepsStopped guards the lost update where +// an enable read a RUNNING task, StopTask stored STOPPED, and the enable then +// wrote its RUNNING copy back over it. +func TestTaskProtection_EnableRacingStopKeepsStopped(t *testing.T) { + const iterations = 400 + + ctx := context.Background() + + for range iterations { + m := newTestMock() + arns := protectionFixture(t, m, 1) + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + _, _, err := m.UpdateTaskProtection(ctx, driver.UpdateTaskProtectionInput{ + Cluster: "prod", Tasks: arns, ProtectionEnabled: true, + }) + assert.NoError(t, err) + }() + + go func() { + defer wg.Done() + + _, err := m.StopTask(ctx, "prod", arns[0], "race") + assert.NoError(t, err) + }() + + wg.Wait() + + got, _, err := m.DescribeTasks(ctx, "prod", arns) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, statusStopped, got[0].LastStatus, "a stopped task must not come back RUNNING") + } +} diff --git a/providers/aws/ecs/task_sets.go b/providers/aws/ecs/task_sets.go index f9e6edc3b..352b21c23 100644 --- a/providers/aws/ecs/task_sets.go +++ b/providers/aws/ecs/task_sets.go @@ -394,8 +394,10 @@ func (m *Mock) mutateTaskSet( return &out, nil } -// DeleteTaskSet deletes a task set and stops its tasks. Unless force is set the -// set must already be scaled down to zero. The returned copy is DRAINING. +// DeleteTaskSet deletes a task set and stops its tasks. The set drains +// immediately whether or not force is set (scale-down is instant, so force has +// no visible effect and is only accepted for compatibility). The returned copy +// is DRAINING. func (m *Mock) DeleteTaskSet(ctx context.Context, in driver.DeleteTaskSetInput) (*driver.TaskSet, error) { want := resolveClusterName(in.Cluster) @@ -415,11 +417,6 @@ func (m *Mock) DeleteTaskSet(ctx context.Context, in driver.DeleteTaskSetInput) return nil, apiErrf(errors.NotFound, excTaskSetNotFound, "The specified task set %q was not found.", in.TaskSet) } - if !in.Force && stored.Scale.Value > 0 { - return nil, apiErrf(errors.FailedPrecondition, excInvalidParameter, - "The task set must be scaled down to 0 before it is deleted. Scale it down or set force.") - } - m.stopTaskSetTasks(ctx, svc, stored, &events) m.taskSets.Delete(taskSetKey(want, svc.Name, stored.ID)) m.tags.Delete(stored.ARN) diff --git a/providers/aws/ecs/task_sets_test.go b/providers/aws/ecs/task_sets_test.go index cce68d4c6..3db4486b5 100644 --- a/providers/aws/ecs/task_sets_test.go +++ b/providers/aws/ecs/task_sets_test.go @@ -179,6 +179,48 @@ func TestTaskSet_UpdateScaleAndStability(t *testing.T) { assert.Equal(t, 2, got[0].RunningCount) } +func TestExternalService_CountsFollowTaskSets(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 2) + ctx := context.Background() + + describe := func() driver.Service { + t.Helper() + + got, _, err := m.DescribeServices(ctx, "prod", []string{"ext"}) + require.NoError(t, err) + require.Len(t, got, 1) + + return got[0] + } + + newTaskSet(t, m, netCfg, nil) + + svc := describe() + assert.Equal(t, 2, svc.RunningCount) + assert.Equal(t, 0, svc.PendingCount) + + newTaskSet(t, m, netCfg, &driver.Scale{Unit: driver.ScaleUnitPercent, Value: 50}) + assert.Equal(t, 3, describe().RunningCount, "the service reports the sum over its task sets") + + tasks, err := m.ListTasks(ctx, "prod", "", "RUNNING", "ext") + require.NoError(t, err) + require.Len(t, tasks, 3) + + _, err = m.StopTask(ctx, "prod", tasks[0].ARN, "test") + require.NoError(t, err) + assert.Equal(t, 3, describe().RunningCount, "a stopped task is relaunched and the count recovers") + + four := 4 + up, err := m.UpdateService(ctx, driver.UpdateServiceInput{Service: "ext", Cluster: "prod", DesiredCount: &four}) + require.NoError(t, err) + + after := describe() + assert.Equal(t, 6, after.RunningCount) + assert.Equal(t, after.RunningCount, up.RunningCount, "UpdateService agrees with DescribeServices") + assert.Equal(t, after.PendingCount, up.PendingCount) +} + func TestTaskSet_StabilizingWhenTasksStopped(t *testing.T) { m := newTestMock() netCfg := externalFixture(t, m, 2) @@ -238,20 +280,20 @@ func TestUpdateServicePrimaryTaskSet(t *testing.T) { require.Len(t, svc[0].TaskSets, 2) } -func TestDeleteTaskSet_ScaleGuardAndForce(t *testing.T) { +func TestDeleteTaskSet_DrainsWithOrWithoutForce(t *testing.T) { m := newTestMock() netCfg := externalFixture(t, m, 2) ctx := context.Background() ts := newTaskSet(t, m, netCfg, nil) - _, err := m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: ts.ID}) - require.Error(t, err) - assert.Equal(t, excInvalidParameter, ecsException(t, err)) - - deleted, err := m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: ts.ID, Force: true}) + // A scaled set deletes without force (Terraform sends Force=false). + deleted, err := m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: ts.ID}) require.NoError(t, err) assert.Equal(t, "DRAINING", deleted.Status) + assert.Zero(t, deleted.RunningCount) + assert.Zero(t, deleted.PendingCount) + assert.Zero(t, deleted.ComputedDesiredCount) running, err := m.ListTasks(ctx, "prod", "", "RUNNING", "ext") require.NoError(t, err) @@ -260,7 +302,13 @@ func TestDeleteTaskSet_ScaleGuardAndForce(t *testing.T) { _, err = m.UpdateServicePrimaryTaskSet(ctx, "prod", "ext", ts.ID) assert.Equal(t, excTaskSetNotFound, ecsException(t, err)) - // A set scaled to zero deletes without force. + // A forced delete behaves the same. + forced := newTaskSet(t, m, netCfg, nil) + deleted, err = m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: forced.ID, Force: true}) + require.NoError(t, err) + assert.Equal(t, "DRAINING", deleted.Status) + + // A set scaled to zero deletes too. zero := newTaskSet(t, m, netCfg, &driver.Scale{Unit: driver.ScaleUnitPercent, Value: 0}) _, err = m.DeleteTaskSet(ctx, driver.DeleteTaskSetInput{Cluster: "prod", Service: "ext", TaskSet: zero.ID}) require.NoError(t, err) @@ -470,3 +518,154 @@ func TestMalformedAndForeignIdentifiers_TaskSets(t *testing.T) { }) assert.Equal(t, excTaskSetNotFound, ecsException(t, err)) } + +// externalUpdateFixture creates a second EXTERNAL service "ext2" carrying a task +// definition, platform version, network configuration, load balancers and +// service registries, so identical echoes can be compared. +func externalUpdateFixture(t *testing.T, m *Mock, netCfg *driver.NetworkConfiguration) driver.CreateServiceInput { + t.Helper() + + in := driver.CreateServiceInput{ + ServiceName: "ext2", Cluster: "prod", DeploymentController: deployControllerExternal, DesiredCount: 2, + TaskDefinition: "fg", LaunchType: launchFargate, PlatformVersion: "1.4.0", NetworkConfiguration: netCfg, + LoadBalancers: []driver.LoadBalancer{{TargetGroupARN: "tg-1", ContainerName: "c", ContainerPort: 80}}, + ServiceRegistries: []driver.ServiceRegistry{{RegistryARN: "reg-1"}}, + } + _, err := m.CreateService(context.Background(), in) + require.NoError(t, err) + + return in +} + +func TestUpdateService_ExternalRejectsTaskDefinitionChange(t *testing.T) { + m := newTestMock() + externalFixture(t, m, 2) + ctx := context.Background() + + _, err := m.UpdateService(ctx, driver.UpdateServiceInput{Service: "ext", Cluster: "prod", TaskDefinition: "fg"}) + require.Error(t, err) + assert.Equal(t, excInvalidParameter, ecsException(t, err)) + + got, _, err := m.DescribeServices(ctx, "prod", []string{"ext"}) + require.NoError(t, err) + assert.Empty(t, got[0].TaskDefinition, "the stored service keeps its (empty) task definition") + + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{Service: "ext", Cluster: "prod", TaskDefinition: "missing"}) + require.Error(t, err) + assert.NotEqual(t, excInvalidParameter, ecsException(t, err), "an unresolvable reference is the not-found error") +} + +func TestUpdateService_ExternalRejectsTaskSetFields(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 2) + ctx := context.Background() + base := externalUpdateFixture(t, m, netCfg) + + newNet := &driver.NetworkConfiguration{AwsVpcConfiguration: &driver.AwsVpcConfiguration{Subnets: []string{"subnet-2"}}} + otherLB := []driver.LoadBalancer{{TargetGroupARN: "tg-2", ContainerName: "c", ContainerPort: 80}} + seven := 7 + + tests := []struct { + name string + in driver.UpdateServiceInput + }{ + {"platformVersion", driver.UpdateServiceInput{PlatformVersion: "1.3.0"}}, + {"networkConfiguration", driver.UpdateServiceInput{NetworkConfiguration: newNet}}, + {"loadBalancers", driver.UpdateServiceInput{LoadBalancers: otherLB}}, + {"emptyLoadBalancers", driver.UpdateServiceInput{LoadBalancers: []driver.LoadBalancer{}}}, + {"capacityProviderStrategy", driver.UpdateServiceInput{ + CapacityProviderStrategy: []driver.CapacityProviderStrategyItem{{CapacityProvider: "FARGATE", Weight: 1}}}}, + {"serviceRegistries", driver.UpdateServiceInput{ServiceRegistries: []driver.ServiceRegistry{{RegistryARN: "reg-2"}}}}, + {"fieldWithDesiredCount", driver.UpdateServiceInput{PlatformVersion: "1.3.0", DesiredCount: &seven}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + before, _, err := m.DescribeServices(ctx, "prod", []string{base.ServiceName}) + require.NoError(t, err) + + tc.in.Service, tc.in.Cluster = base.ServiceName, "prod" + + _, err = m.UpdateService(ctx, tc.in) + require.Error(t, err) + assert.Equal(t, excInvalidParameter, ecsException(t, err)) + + after, _, err := m.DescribeServices(ctx, "prod", []string{base.ServiceName}) + require.NoError(t, err) + assert.Equal(t, before, after, "a rejected update changes nothing, desiredCount included") + }) + } +} + +func TestUpdateService_ExternalAcceptsIdenticalEchoes(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 2) + ctx := context.Background() + base := externalUpdateFixture(t, m, netCfg) + + before, _, err := m.DescribeServices(ctx, "prod", []string{"ext2"}) + require.NoError(t, err) + + echo := &driver.NetworkConfiguration{AwsVpcConfiguration: &driver.AwsVpcConfiguration{ + Subnets: []string{"subnet-1"}, AssignPublicIP: assignPublicIPDisabled}} + td := before[0].TaskDefinition + + for _, ref := range []string{td, "fg", "fg:1"} { + _, err = m.UpdateService(ctx, driver.UpdateServiceInput{ + Service: "ext2", Cluster: "prod", TaskDefinition: ref, PlatformVersion: base.PlatformVersion, + NetworkConfiguration: echo, LoadBalancers: base.LoadBalancers, ServiceRegistries: base.ServiceRegistries, + }) + require.NoError(t, err, ref) + } + + after, _, err := m.DescribeServices(ctx, "prod", []string{"ext2"}) + require.NoError(t, err) + assert.Equal(t, before[0].TaskDefinition, after[0].TaskDefinition) + assert.Equal(t, before[0].NetworkConfiguration, after[0].NetworkConfiguration) + assert.Equal(t, before[0].LoadBalancers, after[0].LoadBalancers) +} + +func TestUpdateService_ExternalAllowsScalarUpdates(t *testing.T) { + m := newTestMock() + netCfg := externalFixture(t, m, 2) + ctx := context.Background() + newTaskSet(t, m, netCfg, nil) + + four, grace := 4, 30 + + _, err := m.UpdateService(ctx, driver.UpdateServiceInput{ + Service: "ext", Cluster: "prod", DesiredCount: &four, HealthCheckGracePeriodSeconds: &grace, PropagateTags: propagateService, + }) + require.NoError(t, err) + + got, _, err := m.DescribeServices(ctx, "prod", []string{"ext"}) + require.NoError(t, err) + assert.Equal(t, 4, got[0].DesiredCount) + assert.Equal(t, 4, got[0].RunningCount, "the task set rescaled") + assert.Equal(t, propagateService, got[0].PropagateTags) + require.NotNil(t, got[0].HealthCheckGracePeriodSeconds) + assert.Equal(t, grace, *got[0].HealthCheckGracePeriodSeconds) +} + +func TestUpdateService_ECSControllerStillAcceptsTaskSetFields(t *testing.T) { + m := newTestMock() + netCfg := fargateFixture(t, m) + ctx := context.Background() + + _, err := m.CreateService(ctx, driver.CreateServiceInput{ + ServiceName: "rolling", Cluster: "prod", TaskDefinition: "fg", LaunchType: launchFargate, NetworkConfiguration: netCfg, + }) + require.NoError(t, err) + + newNet := &driver.NetworkConfiguration{AwsVpcConfiguration: &driver.AwsVpcConfiguration{Subnets: []string{"subnet-2"}}} + lbs := []driver.LoadBalancer{{TargetGroupARN: "tg-2", ContainerName: "c", ContainerPort: 80}} + + got, err := m.UpdateService(ctx, driver.UpdateServiceInput{ + Service: "rolling", Cluster: "prod", TaskDefinition: "fg", PlatformVersion: "1.3.0", NetworkConfiguration: newNet, + LoadBalancers: lbs, ServiceRegistries: []driver.ServiceRegistry{{RegistryARN: "reg-2"}}, + }) + require.NoError(t, err) + assert.Equal(t, "1.3.0", got.PlatformVersion) + assert.Equal(t, lbs, got.LoadBalancers) + assert.Equal(t, []string{"subnet-2"}, got.NetworkConfiguration.AwsVpcConfiguration.Subnets) +} diff --git a/providers/aws/ecs/tasks.go b/providers/aws/ecs/tasks.go index eb27613bf..33e5682a1 100644 --- a/providers/aws/ecs/tasks.go +++ b/providers/aws/ecs/tasks.go @@ -170,6 +170,10 @@ func validateLaunchType(launchType string) error { } } +// detailPrivateIPv4Address is the ElasticNetworkInterface attachment detail +// that carries a task's private IP. +const detailPrivateIPv4Address = "privateIPv4Address" + // Public-IP assignment values of an awsvpc configuration. const ( assignPublicIPEnabled = "ENABLED" @@ -497,13 +501,13 @@ func markContainers(task *driver.Task, status string) { // placeFargate marks a Fargate task RUNNING, echoes/normalizes its platform // version (LATEST or empty resolves to 1.4.0), and synthesizes an elastic // network interface attachment. Fargate has no capacity pool (treated as -// unlimited), so placement never fails. +// unlimited), so placement never fails. It does not store the task: the caller +// still owns it and publishes it once, fully built. func (m *Mock) placeFargate(task *driver.Task, netCfg *driver.NetworkConfiguration, platformVersion string) { task.LastStatus = statusRunning task.PlatformVersion = fargatePlatformVersion(platformVersion) task.Attachments = []driver.Attachment{m.syntheticENI(netCfg)} linkContainerInterfaces(task) - m.tasks.Set(task.ARN, task) } // linkContainerInterfaces gives every container of an awsvpc task the network @@ -518,7 +522,7 @@ func linkContainerInterfaces(task *driver.Task) { privateIP := "" for _, d := range eni.Details { - if d.Name == "privateIPv4Address" { + if d.Name == detailPrivateIPv4Address { privateIP = d.Value } } @@ -564,7 +568,7 @@ func (m *Mock) syntheticENI(nc *driver.NetworkConfiguration) driver.Attachment { Status: "ATTACHED", Details: []driver.KeyValue{ {Name: "networkInterfaceId", Value: "eni-" + id[:17]}, - {Name: "privateIPv4Address", Value: "10.0.0." + strconv.Itoa(int(id[0])%254+1)}, + {Name: detailPrivateIPv4Address, Value: "10.0.0." + strconv.Itoa(int(id[0])%254+1)}, {Name: "subnetId", Value: subnet}, {Name: "macAddress", Value: "0a:58:0a:00:00:01"}, }, diff --git a/server/aws/ecs/capacity_providers.go b/server/aws/ecs/capacity_providers.go index 691c9000f..282631d3d 100644 --- a/server/aws/ecs/capacity_providers.go +++ b/server/aws/ecs/capacity_providers.go @@ -17,6 +17,8 @@ const describeCapacityProvidersDefaultPage = 10 const ( keyCapacityProvider = "capacityProvider" keyFailures = "failures" + keyTaskSet = "taskSet" + jsonNull = "null" ) type wireManagedScaling struct { @@ -196,7 +198,7 @@ func (h *Handler) deleteCapacityProvider(w http.ResponseWriter, r *http.Request) // nonNullRaw drops an absent or JSON-null raw block so the driver sees "not set". func nonNullRaw(raw json.RawMessage) json.RawMessage { - if len(raw) == 0 || string(raw) == "null" { + if len(raw) == 0 || string(raw) == jsonNull { return nil } diff --git a/server/aws/ecs/clusters.go b/server/aws/ecs/clusters.go index 7ffb2afe5..401ae72fb 100644 --- a/server/aws/ecs/clusters.go +++ b/server/aws/ecs/clusters.go @@ -124,7 +124,7 @@ func (h *Handler) describeClusters(w http.ResponseWriter, r *http.Request) { out = append(out, wc) } - wire.WriteJSON(w, map[string]any{"clusters": out, "failures": fromFailures(failures)}) + wire.WriteJSON(w, map[string]any{"clusters": out, keyFailures: fromFailures(failures)}) } // includes reports whether field (case-insensitive) is present in the request's diff --git a/server/aws/ecs/container_instances.go b/server/aws/ecs/container_instances.go index f9cf92695..69571cb59 100644 --- a/server/aws/ecs/container_instances.go +++ b/server/aws/ecs/container_instances.go @@ -100,7 +100,7 @@ func (h *Handler) updateContainerInstancesState(w http.ResponseWriter, r *http.R out = append(out, instanceToWire(&instances[i])) } - wire.WriteJSON(w, map[string]any{"containerInstances": out, "failures": fromFailures(failures)}) + wire.WriteJSON(w, map[string]any{"containerInstances": out, keyFailures: fromFailures(failures)}) } func (h *Handler) listContainerInstances(w http.ResponseWriter, r *http.Request) { @@ -167,5 +167,5 @@ func (h *Handler) describeContainerInstances(w http.ResponseWriter, r *http.Requ out = append(out, wci) } - wire.WriteJSON(w, map[string]any{"containerInstances": out, "failures": fromFailures(failures)}) + wire.WriteJSON(w, map[string]any{"containerInstances": out, keyFailures: fromFailures(failures)}) } diff --git a/server/aws/ecs/handler.go b/server/aws/ecs/handler.go index 5d5d0e314..d00324ac8 100644 --- a/server/aws/ecs/handler.go +++ b/server/aws/ecs/handler.go @@ -45,7 +45,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { routers := []func(http.ResponseWriter, *http.Request, string) bool{ h.routeClusters, h.routeTaskDefs, h.routeTasks, h.routeServices, h.routeContainerInstances, h.routeTags, h.routeAccount, h.routeAttributes, h.routeCapacityProviders, - h.routeTaskProtection, h.routeTaskSets, h.routeServiceDeployments, + h.routeTaskProtection, h.routeTaskSets, h.routeServiceDeployments, h.routeNamespace, } for _, route := range routers { if route(w, r, op) { diff --git a/server/aws/ecs/namespace.go b/server/aws/ecs/namespace.go new file mode 100644 index 000000000..7f33aef6b --- /dev/null +++ b/server/aws/ecs/namespace.go @@ -0,0 +1,41 @@ +package ecs + +import ( + "net/http" + + "github.com/stackshy/cloudemu/v2/server/wire" + "github.com/stackshy/cloudemu/v2/services/ecs/driver" +) + +func (h *Handler) routeNamespace(w http.ResponseWriter, r *http.Request, op string) bool { + if op != "ListServicesByNamespace" { + return false + } + + if sn, ok := capabilityOf[driver.ServiceNamespaces](h.ecs, w, op); ok { + listServicesByNamespace(w, r, sn) + } + + return true +} + +func listServicesByNamespace(w http.ResponseWriter, r *http.Request, sn driver.ServiceNamespaces) { + var req struct { + Namespace string `json:"namespace"` + MaxResults int `json:"maxResults"` + NextToken string `json:"nextToken"` + } + + if !wire.DecodeJSON(w, r, &req) { + return + } + + arns, next, err := sn.ListServicesByNamespace(r.Context(), req.Namespace, req.MaxResults, req.NextToken) + if err != nil { + writeErr(w, err) + + return + } + + wire.WriteJSON(w, listResponse("serviceArns", arns, next)) +} diff --git a/server/aws/ecs/sdk_roundtrip_test.go b/server/aws/ecs/sdk_roundtrip_test.go index 83eb9daa0..2c5b5c2e3 100644 --- a/server/aws/ecs/sdk_roundtrip_test.go +++ b/server/aws/ecs/sdk_roundtrip_test.go @@ -526,6 +526,11 @@ func TestSDKEC2Placement(t *testing.T) { if got := remainingResource(ci.ContainerInstances[0].RemainingResources, "CPU"); got != 1024-256 { t.Fatalf("remaining CPU = %d, want %d", got, 1024-256) } + + // The placement is a change to the instance: seeded at version 1, now 2. + if ci.ContainerInstances[0].Version != 2 { + t.Fatalf("container instance version after placement = %d, want 2", ci.ContainerInstances[0].Version) + } } // remainingResource returns the integerValue of the named INTEGER resource. @@ -1015,6 +1020,10 @@ func TestSDKContainerInstanceRoundtrip(t *testing.T) { ciARN := aws.ToString(reg.ContainerInstance.ContainerInstanceArn) + if reg.ContainerInstance.Version != 1 { + t.Fatalf("Register version = %d, want 1", reg.ContainerInstance.Version) + } + listCI, err := client.ListContainerInstances(ctx, &awsecs.ListContainerInstancesInput{Cluster: aws.String("prod")}) if err != nil { t.Fatalf("ListContainerInstances: %v", err) @@ -1036,6 +1045,10 @@ func TestSDKContainerInstanceRoundtrip(t *testing.T) { t.Fatalf("DescribeContainerInstances = %d, want 1", len(descCI.ContainerInstances)) } + if descCI.ContainerInstances[0].Version != 1 { + t.Fatalf("Describe version = %d, want 1", descCI.ContainerInstances[0].Version) + } + upd, err := client.UpdateContainerInstancesState(ctx, &awsecs.UpdateContainerInstancesStateInput{ Cluster: aws.String("prod"), ContainerInstances: []string{ciARN, "i-missing"}, @@ -1049,6 +1062,10 @@ func TestSDKContainerInstanceRoundtrip(t *testing.T) { t.Fatalf("UpdateContainerInstancesState instances = %+v", upd.ContainerInstances) } + if upd.ContainerInstances[0].Version != 2 { + t.Fatalf("Update version = %d, want 2", upd.ContainerInstances[0].Version) + } + if len(upd.Failures) != 1 { t.Fatalf("UpdateContainerInstancesState failures = %+v, want 1", upd.Failures) } diff --git a/server/aws/ecs/service_deployments.go b/server/aws/ecs/service_deployments.go index 1be592b01..832cd3de5 100644 --- a/server/aws/ecs/service_deployments.go +++ b/server/aws/ecs/service_deployments.go @@ -134,14 +134,6 @@ func (h *Handler) routeServiceDeployments(w http.ResponseWriter, r *http.Request return true } - if op == "ListServicesByNamespace" { - if sn, ok := capabilityOf[driver.ServiceNamespaces](h.ecs, w, op); ok { - listServicesByNamespace(w, r, sn) - } - - return true - } - return false } @@ -241,7 +233,7 @@ func describeByARNs[T any]( out = append(out, toWire(&items[i])) } - wire.WriteJSON(w, map[string]any{itemsKey: out, "failures": fromFailures(failures)}) + wire.WriteJSON(w, map[string]any{itemsKey: out, keyFailures: fromFailures(failures)}) } func stopServiceDeployment(w http.ResponseWriter, r *http.Request, sd driver.ServiceDeployments) { @@ -264,31 +256,10 @@ func stopServiceDeployment(w http.ResponseWriter, r *http.Request, sd driver.Ser wire.WriteJSON(w, map[string]any{"serviceDeploymentArn": arn}) } -func listServicesByNamespace(w http.ResponseWriter, r *http.Request, sn driver.ServiceNamespaces) { - var req struct { - Namespace string `json:"namespace"` - MaxResults int `json:"maxResults"` - NextToken string `json:"nextToken"` - } - - if !wire.DecodeJSON(w, r, &req) { - return - } - - arns, next, err := sn.ListServicesByNamespace(r.Context(), req.Namespace, req.MaxResults, req.NextToken) - if err != nil { - writeErr(w, err) - - return - } - - wire.WriteJSON(w, listResponse("serviceArns", arns, next)) -} - // toServiceConnect converts the wire serviceConnectConfiguration to the driver // shape, keeping the raw JSON and extracting the namespace. func toServiceConnect(raw json.RawMessage) *driver.ServiceConnectConfiguration { - if len(raw) == 0 || string(raw) == "null" { + if len(raw) == 0 || string(raw) == jsonNull { return nil } diff --git a/server/aws/ecs/service_deployments_sdk_test.go b/server/aws/ecs/service_deployments_sdk_test.go index d5c05c99f..b118b4e32 100644 --- a/server/aws/ecs/service_deployments_sdk_test.go +++ b/server/aws/ecs/service_deployments_sdk_test.go @@ -92,6 +92,19 @@ func TestSDK_ServiceDeployments(t *testing.T) { if !errorsAs(err, ¬Found) { t.Fatalf("want *ServiceDeploymentNotFoundException, got %T: %v", err, err) } + + // stopType is optional on the wire (an omitted value means ROLLBACK). + _, err = client.StopServiceDeployment(ctx, &awsecs.StopServiceDeploymentInput{ + ServiceDeploymentArn: aws.String("arn:aws:ecs:us-east-1:000000000000:service-deployment/prod/web/none"), + }) + if !errorsAs(err, ¬Found) { + t.Fatalf("no stopType, unknown ARN: want *ServiceDeploymentNotFoundException, got %T: %v", err, err) + } + + _, err = client.StopServiceDeployment(ctx, &awsecs.StopServiceDeploymentInput{ServiceDeploymentArn: aws.String(arn)}) + if !errorsAs(err, &conflict) { + t.Fatalf("no stopType, completed deployment: want *ConflictException, got %T: %v", err, err) + } } func TestSDK_ServiceConnectNamespace(t *testing.T) { diff --git a/server/aws/ecs/services.go b/server/aws/ecs/services.go index ca121db41..0143ba39a 100644 --- a/server/aws/ecs/services.go +++ b/server/aws/ecs/services.go @@ -221,7 +221,7 @@ func (h *Handler) describeServices(w http.ResponseWriter, r *http.Request) { out = append(out, ws) } - wire.WriteJSON(w, map[string]any{"services": out, "failures": fromFailures(failures)}) + wire.WriteJSON(w, map[string]any{"services": out, keyFailures: fromFailures(failures)}) } func (h *Handler) deleteService(w http.ResponseWriter, r *http.Request) { diff --git a/server/aws/ecs/task_protection.go b/server/aws/ecs/task_protection.go index 3cfc5546e..322c31fd9 100644 --- a/server/aws/ecs/task_protection.go +++ b/server/aws/ecs/task_protection.go @@ -100,5 +100,5 @@ func writeProtectedTasks(w http.ResponseWriter, tasks []driver.ProtectedTask, fa }) } - wire.WriteJSON(w, map[string]any{"protectedTasks": out, "failures": fromFailures(failures)}) + wire.WriteJSON(w, map[string]any{"protectedTasks": out, keyFailures: fromFailures(failures)}) } diff --git a/server/aws/ecs/task_protection_sdk_test.go b/server/aws/ecs/task_protection_sdk_test.go index a2095b387..26a6a4c46 100644 --- a/server/aws/ecs/task_protection_sdk_test.go +++ b/server/aws/ecs/task_protection_sdk_test.go @@ -38,6 +38,18 @@ func TestSDK_TaskProtection(t *testing.T) { t.Fatalf("GetTaskProtection: %v %+v", err, get) } + if _, err = client.StopTask(ctx, &awsecs.StopTaskInput{Cluster: aws.String("prod"), Task: aws.String(list.TaskArns[0])}); err != nil { + t.Fatalf("StopTask: %v", err) + } + + stopped, err := client.UpdateTaskProtection(ctx, &awsecs.UpdateTaskProtectionInput{ + Cluster: aws.String("prod"), Tasks: list.TaskArns, ProtectionEnabled: true, + }) + if err != nil || len(stopped.ProtectedTasks) != 0 || len(stopped.Failures) != 1 || + aws.ToString(stopped.Failures[0].Reason) != "TASK_NOT_VALID" { + t.Fatalf("protecting a stopped task must be a TASK_NOT_VALID failure: %v %+v", err, stopped) + } + missing, err := client.GetTaskProtection(ctx, &awsecs.GetTaskProtectionInput{Cluster: aws.String("prod"), Tasks: []string{"nope"}}) if err != nil || len(missing.Failures) != 1 || aws.ToString(missing.Failures[0].Reason) != "MISSING" { t.Fatalf("missing task must be a MISSING failure: %v %+v", err, missing) diff --git a/server/aws/ecs/task_sets.go b/server/aws/ecs/task_sets.go index 1c405207b..3623ed015 100644 --- a/server/aws/ecs/task_sets.go +++ b/server/aws/ecs/task_sets.go @@ -137,7 +137,7 @@ func createTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { return } - wire.WriteJSON(w, map[string]any{"taskSet": taskSetToWire(out)}) + wire.WriteJSON(w, map[string]any{keyTaskSet: taskSetToWire(out)}) } func updateTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { @@ -162,7 +162,7 @@ func updateTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { return } - wire.WriteJSON(w, map[string]any{"taskSet": taskSetToWire(out)}) + wire.WriteJSON(w, map[string]any{keyTaskSet: taskSetToWire(out)}) } func deleteTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { @@ -186,7 +186,7 @@ func deleteTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { return } - wire.WriteJSON(w, map[string]any{"taskSet": taskSetToWire(out)}) + wire.WriteJSON(w, map[string]any{keyTaskSet: taskSetToWire(out)}) } func describeTaskSets(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { @@ -213,7 +213,7 @@ func describeTaskSets(w http.ResponseWriter, r *http.Request, ts driver.TaskSets out = []wireTaskSet{} } - wire.WriteJSON(w, map[string]any{"taskSets": out, "failures": fromFailures(failures)}) + wire.WriteJSON(w, map[string]any{"taskSets": out, keyFailures: fromFailures(failures)}) } func updatePrimaryTaskSet(w http.ResponseWriter, r *http.Request, ts driver.TaskSets) { @@ -234,5 +234,5 @@ func updatePrimaryTaskSet(w http.ResponseWriter, r *http.Request, ts driver.Task return } - wire.WriteJSON(w, map[string]any{"taskSet": taskSetToWire(out)}) + wire.WriteJSON(w, map[string]any{keyTaskSet: taskSetToWire(out)}) } diff --git a/server/aws/ecs/task_sets_sdk_test.go b/server/aws/ecs/task_sets_sdk_test.go index 68723f742..3e0e73c1e 100644 --- a/server/aws/ecs/task_sets_sdk_test.go +++ b/server/aws/ecs/task_sets_sdk_test.go @@ -2,6 +2,7 @@ package ecs_test import ( "context" + "errors" "testing" "github.com/aws/aws-sdk-go-v2/aws" @@ -37,6 +38,11 @@ func TestSDK_TaskSets_FullLifecycle(t *testing.T) { t.Fatalf("unexpected task set: %+v", set) } + svcs, err := client.DescribeServices(ctx, &awsecs.DescribeServicesInput{Cluster: aws.String("prod"), Services: []string{"ext"}}) + if err != nil || len(svcs.Services) != 1 || svcs.Services[0].RunningCount != 2 || svcs.Services[0].PendingCount != 0 { + t.Fatalf("EXTERNAL service counts must follow its task sets: %v %+v", err, svcs) + } + plain, err := client.DescribeTaskSets(ctx, &awsecs.DescribeTaskSetsInput{Cluster: aws.String("prod"), Service: aws.String("ext")}) if err != nil || len(plain.TaskSets) != 1 || len(plain.TaskSets[0].Tags) != 0 { t.Fatalf("DescribeTaskSets must omit tags without include=TAGS: %v %+v", err, plain) @@ -70,18 +76,24 @@ func TestSDK_TaskSets_FullLifecycle(t *testing.T) { t.Fatalf("DescribeServices taskSets: %v", err) } - _, err = client.DeleteTaskSet(ctx, &awsecs.DeleteTaskSetInput{Cluster: aws.String("prod"), Service: aws.String("ext"), TaskSet: set.Id}) + del, err := client.DeleteTaskSet(ctx, &awsecs.DeleteTaskSetInput{Cluster: aws.String("prod"), Service: aws.String("ext"), TaskSet: set.Id}) + if err != nil || aws.ToString(del.TaskSet.Status) != "DRAINING" { + t.Fatalf("DeleteTaskSet without force: %v %+v", err, del) + } - var invalid *ecstypes.InvalidParameterException - if !errorsAs(err, &invalid) { - t.Fatalf("DeleteTaskSet without force on a scaled set: want *InvalidParameterException, got %T: %v", err, err) + second, err := client.CreateTaskSet(ctx, &awsecs.CreateTaskSetInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), TaskDefinition: aws.String("fg"), + LaunchType: ecstypes.LaunchTypeFargate, NetworkConfiguration: netCfg, + }) + if err != nil { + t.Fatalf("CreateTaskSet second: %v", err) } - del, err := client.DeleteTaskSet(ctx, &awsecs.DeleteTaskSetInput{ - Cluster: aws.String("prod"), Service: aws.String("ext"), TaskSet: set.Id, Force: aws.Bool(true), + forced, err := client.DeleteTaskSet(ctx, &awsecs.DeleteTaskSetInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), TaskSet: second.TaskSet.Id, Force: aws.Bool(true), }) - if err != nil || aws.ToString(del.TaskSet.Status) != "DRAINING" { - t.Fatalf("DeleteTaskSet force: %v %+v", err, del) + if err != nil || aws.ToString(forced.TaskSet.Status) != "DRAINING" { + t.Fatalf("DeleteTaskSet force: %v %+v", err, forced) } _, err = client.UpdateTaskSet(ctx, &awsecs.UpdateTaskSetInput{ @@ -138,3 +150,37 @@ func TestSDK_TaskSets_RequireExternalController(t *testing.T) { t.Fatalf("want *ServiceNotFoundException, got %T: %v", err, err) } } + +func TestSDK_UpdateService_ExternalRejectsTaskDefinition(t *testing.T) { + client := newECSClient(t) + ctx := context.Background() + + fargateSetup(t, client) + + if _, err := client.CreateService(ctx, &awsecs.CreateServiceInput{ + Cluster: aws.String("prod"), ServiceName: aws.String("ext"), DesiredCount: aws.Int32(1), + DeploymentController: &ecstypes.DeploymentController{Type: ecstypes.DeploymentControllerTypeExternal}, + }); err != nil { + t.Fatalf("CreateService EXTERNAL: %v", err) + } + + _, err := client.UpdateService(ctx, &awsecs.UpdateServiceInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), TaskDefinition: aws.String("fg"), DesiredCount: aws.Int32(3), + }) + + var invalid *ecstypes.InvalidParameterException + if !errors.As(err, &invalid) { + t.Fatalf("UpdateService taskDefinition on EXTERNAL = %v, want InvalidParameterException", err) + } + + got, err := client.DescribeServices(ctx, &awsecs.DescribeServicesInput{Cluster: aws.String("prod"), Services: []string{"ext"}}) + if err != nil || got.Services[0].DesiredCount != 1 || aws.ToString(got.Services[0].TaskDefinition) != "" { + t.Fatalf("a rejected update must change nothing: %v %+v", err, got) + } + + if _, err := client.UpdateService(ctx, &awsecs.UpdateServiceInput{ + Cluster: aws.String("prod"), Service: aws.String("ext"), DesiredCount: aws.Int32(3), + }); err != nil { + t.Fatalf("UpdateService desiredCount on EXTERNAL: %v", err) + } +} diff --git a/server/aws/ecs/tasks.go b/server/aws/ecs/tasks.go index aad10d70b..a38381386 100644 --- a/server/aws/ecs/tasks.go +++ b/server/aws/ecs/tasks.go @@ -196,5 +196,5 @@ func (*Handler) writeTasks(w http.ResponseWriter, tasks []driver.Task, failures out = append(out, taskToWire(&tasks[i])) } - wire.WriteJSON(w, map[string]any{"tasks": out, "failures": fromFailures(failures)}) + wire.WriteJSON(w, map[string]any{"tasks": out, keyFailures: fromFailures(failures)}) } diff --git a/server/aws/ecs/types.go b/server/aws/ecs/types.go index b883f6603..ecd8a40cf 100644 --- a/server/aws/ecs/types.go +++ b/server/aws/ecs/types.go @@ -422,6 +422,7 @@ type wireContainerInstance struct { RegisteredResources []wireResource `json:"registeredResources,omitempty"` RemainingResources []wireResource `json:"remainingResources,omitempty"` Tags []wireTag `json:"tags,omitempty"` + Version int64 `json:"version"` } // --- request -> driver converters --- @@ -1547,5 +1548,6 @@ func instanceToWire(ci *driver.ContainerInstance) wireContainerInstance { RegisteredResources: resourcesFromCapacity(ci.RegisteredCPU, ci.RegisteredMemory), RemainingResources: resourcesFromCapacity(ci.RemainingCPU, ci.RemainingMemory), Tags: fromTags(ci.Tags), + Version: int64(ci.Version), } } diff --git a/server/aws/ecs/parity_validation_sdk_test.go b/server/aws/ecs/validation_sdk_test.go similarity index 100% rename from server/aws/ecs/parity_validation_sdk_test.go rename to server/aws/ecs/validation_sdk_test.go diff --git a/services/ecs/capabilities.go b/services/ecs/capabilities.go index e250d6216..b62f153ac 100644 --- a/services/ecs/capabilities.go +++ b/services/ecs/capabilities.go @@ -53,16 +53,12 @@ func (e *ECS) UpdateTaskProtection(ctx context.Context, in driver.UpdateTaskProt }) } -// callCapability runs one single-result capability operation through do(). -func callCapability[C, R any](ctx context.Context, e *ECS, op string, input any, call func(C) (R, error)) (R, error) { +// doOne runs one single-result capability operation through do(). Its closure takes no +// parameter so that the coverage generator does not count driver interfaces here. +func doOne[R any](ctx context.Context, e *ECS, op string, input any, call func() (R, error)) (R, error) { var zero R - c, err := capabilityOf[C](e, op) - if err != nil { - return zero, err - } - - out, err := e.do(ctx, op, input, func() (any, error) { return call(c) }) + out, err := e.do(ctx, op, input, func() (any, error) { return call() }) if err != nil { return zero, err } @@ -74,21 +70,36 @@ func callCapability[C, R any](ctx context.Context, e *ECS, op string, input any, // //nolint:gocritic // in is passed by value to mirror the driver interface; the copy is cheap for a mock. func (e *ECS) CreateTaskSet(ctx context.Context, in driver.CreateTaskSetInput) (*driver.TaskSet, error) { - return callCapability(ctx, e, "CreateTaskSet", in, func(c driver.TaskSets) (*driver.TaskSet, error) { + c, err := capabilityOf[driver.TaskSets](e, "CreateTaskSet") + if err != nil { + return nil, err + } + + return doOne(ctx, e, "CreateTaskSet", in, func() (*driver.TaskSet, error) { return c.CreateTaskSet(ctx, in) }) } // UpdateTaskSet changes a task set's scale. func (e *ECS) UpdateTaskSet(ctx context.Context, in driver.UpdateTaskSetInput) (*driver.TaskSet, error) { - return callCapability(ctx, e, "UpdateTaskSet", in, func(c driver.TaskSets) (*driver.TaskSet, error) { + c, err := capabilityOf[driver.TaskSets](e, "UpdateTaskSet") + if err != nil { + return nil, err + } + + return doOne(ctx, e, "UpdateTaskSet", in, func() (*driver.TaskSet, error) { return c.UpdateTaskSet(ctx, in) }) } // DeleteTaskSet deletes a task set. func (e *ECS) DeleteTaskSet(ctx context.Context, in driver.DeleteTaskSetInput) (*driver.TaskSet, error) { - return callCapability(ctx, e, "DeleteTaskSet", in, func(c driver.TaskSets) (*driver.TaskSet, error) { + c, err := capabilityOf[driver.TaskSets](e, "DeleteTaskSet") + if err != nil { + return nil, err + } + + return doOne(ctx, e, "DeleteTaskSet", in, func() (*driver.TaskSet, error) { return c.DeleteTaskSet(ctx, in) }) } @@ -109,7 +120,12 @@ func (e *ECS) DescribeTaskSets(ctx context.Context, cluster, service string, ids // UpdateServicePrimaryTaskSet promotes a task set to PRIMARY. func (e *ECS) UpdateServicePrimaryTaskSet(ctx context.Context, cluster, service, primary string) (*driver.TaskSet, error) { - return callCapability(ctx, e, "UpdateServicePrimaryTaskSet", primary, func(c driver.TaskSets) (*driver.TaskSet, error) { + c, err := capabilityOf[driver.TaskSets](e, "UpdateServicePrimaryTaskSet") + if err != nil { + return nil, err + } + + return doOne(ctx, e, "UpdateServicePrimaryTaskSet", primary, func() (*driver.TaskSet, error) { return c.UpdateServicePrimaryTaskSet(ctx, cluster, service, primary) }) } @@ -126,7 +142,12 @@ type deploymentPage struct { func (e *ECS) ListServiceDeployments(ctx context.Context, in driver.ListServiceDeploymentsInput) ( items []driver.ServiceDeployment, next string, err error, ) { - page, err := callCapability(ctx, e, "ListServiceDeployments", in, func(c driver.ServiceDeployments) (deploymentPage, error) { + c, err := capabilityOf[driver.ServiceDeployments](e, "ListServiceDeployments") + if err != nil { + return nil, "", err + } + + page, err := doOne(ctx, e, "ListServiceDeployments", in, func() (deploymentPage, error) { got, token, cerr := c.ListServiceDeployments(ctx, in) return deploymentPage{items: got, next: token}, cerr @@ -168,7 +189,12 @@ func (e *ECS) DescribeServiceRevisions(ctx context.Context, arns []string) ( // StopServiceDeployment stops a deployment that has not completed. func (e *ECS) StopServiceDeployment(ctx context.Context, arn, stopType string) (string, error) { - return callCapability(ctx, e, "StopServiceDeployment", arn, func(c driver.ServiceDeployments) (string, error) { + c, err := capabilityOf[driver.ServiceDeployments](e, "StopServiceDeployment") + if err != nil { + return "", err + } + + return doOne(ctx, e, "StopServiceDeployment", arn, func() (string, error) { return c.StopServiceDeployment(ctx, arn, stopType) }) } @@ -177,7 +203,12 @@ func (e *ECS) StopServiceDeployment(ctx context.Context, arn, stopType string) ( func (e *ECS) ListServicesByNamespace(ctx context.Context, namespace string, maxResults int, nextToken string) ( arns []string, next string, err error, ) { - page, err := callCapability(ctx, e, "ListServicesByNamespace", namespace, func(c driver.ServiceNamespaces) (arnPage, error) { + c, err := capabilityOf[driver.ServiceNamespaces](e, "ListServicesByNamespace") + if err != nil { + return nil, "", err + } + + page, err := doOne(ctx, e, "ListServicesByNamespace", namespace, func() (arnPage, error) { got, token, cerr := c.ListServicesByNamespace(ctx, namespace, maxResults, nextToken) return arnPage{items: got, next: token}, cerr diff --git a/services/ecs/driver/capabilities.go b/services/ecs/driver/capabilities.go index a40a68f3e..7d5794a98 100644 --- a/services/ecs/driver/capabilities.go +++ b/services/ecs/driver/capabilities.go @@ -97,8 +97,8 @@ type UpdateTaskSetInput struct { Scale Scale } -// DeleteTaskSetInput identifies a task set to delete. Without Force the set must -// already be scaled down to zero. +// DeleteTaskSetInput identifies a task set to delete. Force is accepted for API +// compatibility and has no visible effect: the set drains immediately either way. type DeleteTaskSetInput struct { Cluster string Service string