diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8dbc01bc28..8afca91036 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -190,6 +190,58 @@ jobs: artifact-name: worker-executor-tests-${{ matrix.group.name }}-report github-token: ${{ secrets.GITHUB_TOKEN }} + managed-xfs-tests: + name: managed-xfs-tests + runs-on: blacksmith-8vcpu-ubuntu-2404 + needs: + - build-and-store + - merge-test-components + timeout-minutes: 45 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 1 + - uses: ./.github/actions/setup-rust + with: + use-cache: 'false' + install-cargo-binstall: 'true' + - uses: ./.github/actions/restore-binaries + with: + run-id: ${{ github.run_id }} + fail-on-cache-miss: 'true' + - uses: ./.github/actions/restore-test-components + with: + run-id: ${{ github.run_id }} + - name: Set up Lima + id: lima + env: + GH_TOKEN: ${{ github.token }} + LIMA_VERSION: v2.2.0 + run: | + sudo apt-get update -qq + sudo apt-get install -qqy --no-install-recommends ovmf qemu-system-x86 qemu-utils + test -c /dev/kvm + sudo chown "$(whoami)" /dev/kvm + archive="lima-${LIMA_VERSION#v}-Linux-x86_64.tar.gz" + curl -fOSL "https://github.com/lima-vm/lima/releases/download/${LIMA_VERSION}/${archive}" + gh attestation verify --owner lima-vm "${archive}" + sudo tar -C /usr/local -xzf "${archive}" + rm "${archive}" + echo "version=${LIMA_VERSION}" >> "${GITHUB_OUTPUT}" + - name: Cache Lima images + uses: actions/cache@v5 + with: + path: ~/.cache/lima + key: lima-${{ runner.os }}-${{ steps.lima.outputs.version }} + - name: Run managed XFS suite + env: + GOLEM_MANAGED_XFS_REUSE_TEST_BINARIES: 1 + GOLEM_MANAGED_XFS_TARGET_DIR: ${{ github.workspace }}/target + GOLEM_MANAGED_XFS_CARGO_TEST_R: ${{ github.workspace }}/target/cargo-test-r + run: | + cp "$(command -v cargo-test-r)" "${GOLEM_MANAGED_XFS_CARGO_TEST_R}" + integration-tests/scripts/managed-filesystem/run-lima.sh + build-cli-test-bins: env: CARGO_BUILD_JOBS: 20 @@ -1020,6 +1072,7 @@ jobs: - unit-tests-and-checks - golem-schema-guest - worker-tests + - managed-xfs-tests - it - it-cli if: "startsWith(github.ref, 'refs/tags/v')" diff --git a/Cargo.lock b/Cargo.lock index 339fc33d13..be525f42a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4292,6 +4292,7 @@ dependencies = [ "bit-vec 0.6.3", "blake3", "bytes", + "cap-fs-ext", "cap-std", "cap-time-ext", "chrono", @@ -4326,6 +4327,8 @@ dependencies = [ "include_dir", "itertools 0.14.0", "lazy_static", + "libc", + "linux-raw-sys 0.12.1", "log", "mac_address", "md5", @@ -4343,6 +4346,7 @@ dependencies = [ "redis", "regex", "ringbuf", + "rustix 1.1.4", "rustls 0.23.37", "scc", "serde", diff --git a/Cargo.toml b/Cargo.toml index 95676aa6ec..8b0bf5274a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,7 @@ bitflags = "2.6.0" blake3 = { version = "1.8.2", features = ["rayon"] } bytes = "1.10.1" camino = "1.1.10" +cap-fs-ext = "3.4.5" # keep in sync with wasmtime cap-std = "3.4.5" # keep in sync with wasmtime cap-time-ext = "3.4.5" cargo_metadata = "0.21.0" @@ -153,6 +154,8 @@ kube-derive = "0.98.0" lazy_static = "1.5.0" leb128 = "0.2.5" lenient_bool = "0.1.1" +libc = "0.2.186" +linux-raw-sys = { version = "0.12.1", features = ["general", "ioctl"] } log = "0.4.26" mac_address = "1.1.8" mappable-rc = "0.1.1" @@ -220,6 +223,7 @@ rsa = "0.9.7" rust_decimal = "1.39.0" rustc-hash = "2.1.1" +rustix = { version = "1.1.4", features = ["fs"] } rustls = { version = "0.23.23", features = ["ring"] } rustls-pemfile = "2.2.0" sanitize-filename = "0.6.0" diff --git a/cli/golem-cli/command-output-schema/command-output.schema.json b/cli/golem-cli/command-output-schema/command-output.schema.json index 9375eface4..6a5b312b98 100644 --- a/cli/golem-cli/command-output-schema/command-output.schema.json +++ b/cli/golem-cli/command-output-schema/command-output.schema.json @@ -4347,7 +4347,6 @@ { "$ref": "#/definitions/PublicOplogEntrySuccessfulUpdate" }, { "$ref": "#/definitions/PublicOplogEntryFailedUpdate" }, { "$ref": "#/definitions/PublicOplogEntryGrowMemory" }, - { "$ref": "#/definitions/PublicOplogEntryFilesystemStorageUsageUpdate" }, { "$ref": "#/definitions/PublicOplogEntryCreateResource" }, { "$ref": "#/definitions/PublicOplogEntryDropResource" }, { "$ref": "#/definitions/PublicOplogEntryLog" }, @@ -4553,12 +4552,6 @@ "properties": { "type": { "const": "GrowMemory" }, "timestamp": { "type": "string" }, "delta": { "type": "integer", "minimum": 0 } }, "additionalProperties": false }, - "PublicOplogEntryFilesystemStorageUsageUpdate": { - "type": "object", - "required": ["type", "timestamp", "delta"], - "properties": { "type": { "const": "FilesystemStorageUsageUpdate" }, "timestamp": { "type": "string" }, "delta": { "type": "integer" } }, - "additionalProperties": false - }, "PublicOplogEntryCreateResource": { "type": "object", "required": ["type", "timestamp", "id", "name", "owner"], diff --git a/cli/golem-cli/src/model/agent/oplog.rs b/cli/golem-cli/src/model/agent/oplog.rs index 509a061e15..84fdeb59e7 100644 --- a/cli/golem-cli/src/model/agent/oplog.rs +++ b/cli/golem-cli/src/model/agent/oplog.rs @@ -354,17 +354,6 @@ impl TextOutput for PublicOplogEntry { format_id(&format_binary_size(¶ms.delta)), )); } - PublicOplogEntry::FilesystemStorageUsageUpdate(params) => { - logln(format_message_highlight("STORAGE USAGE UPDATE")); - logln(format!( - "{pad}at: {}", - format_id(¶ms.timestamp) - )); - logln(format!( - "{pad}delta: {}", - format_id(¶ms.delta.to_string()), - )); - } PublicOplogEntry::CreateResource(params) => { logln(format_message_highlight("CREATE RESOURCE")); logln(format!( diff --git a/cli/golem-cli/src/model/cli_output/tests.rs b/cli/golem-cli/src/model/cli_output/tests.rs index 31277e5b9f..b9e5952cc2 100644 --- a/cli/golem-cli/src/model/cli_output/tests.rs +++ b/cli/golem-cli/src/model/cli_output/tests.rs @@ -1972,10 +1972,6 @@ fn sample_public_oplog_entries() -> Vec, pub oplog_idx: OplogIndex, pub active_plugins: HashSet, @@ -746,7 +745,6 @@ impl Default for AgentStatusRecord { component_revision: ComponentRevision::INITIAL, component_size: 0, total_linear_memory_size: 0, - current_filesystem_storage_usage: 0, owned_resources: HashMap::new(), oplog_idx: OplogIndex::default(), active_plugins: HashSet::new(), diff --git a/golem-common/src/model/oplog/matcher.rs b/golem-common/src/model/oplog/matcher.rs index 1d9e4b9f18..96e97f2f9e 100644 --- a/golem-common/src/model/oplog/matcher.rs +++ b/golem-common/src/model/oplog/matcher.rs @@ -322,10 +322,6 @@ impl PublicOplogEntry { Self::string_match("growmemory", &[], query_path, query) || Self::string_match("grow-memory", &[], query_path, query) } - PublicOplogEntry::FilesystemStorageUsageUpdate(_params) => { - Self::string_match("filesystemstorageusageupdate", &[], query_path, query) - || Self::string_match("filesystem-storage-usage-update", &[], query_path, query) - } PublicOplogEntry::CreateResource(_params) => { Self::string_match("createresource", &[], query_path, query) || Self::string_match("create-resource", &[], query_path, query) diff --git a/golem-common/src/model/oplog/protobuf.rs b/golem-common/src/model/oplog/protobuf.rs index 575eaccbc8..2695e4dfce 100644 --- a/golem-common/src/model/oplog/protobuf.rs +++ b/golem-common/src/model/oplog/protobuf.rs @@ -46,13 +46,13 @@ use crate::model::oplog::public_oplog_entry::{ CancelledParams, CardEventQueuedParams, CardInstallFailedParams, CardInstalledParams, CardRevokedParams, CommittedRemoteTransactionParams, CompletionDiscardedParams, CreateParams, CreateResourceParams, DeactivatePluginParams, DropResourceParams, EndAtomicRegionParams, - EndParams, ErrorParams, ExitedParams, FailedUpdateParams, FilesystemStorageUsageUpdateParams, - FinishSpanParams, GrowMemoryParams, HostStreamFrameParams, InterruptedParams, JumpParams, - LogParams, NoOpParams, OplogProcessorCheckpointParams, PendingAgentInvocationParams, - PendingUpdateParams, PreCommitRemoteTransactionParams, PreRollbackRemoteTransactionParams, - RemoveRetryPolicyParams, RestartParams, RevertParams, RolledBackRemoteTransactionParams, - SetRetryPolicyParams, SetSpanAttributeParams, SnapshotParams, StartParams, StartSpanParams, - SuccessfulUpdateParams, SuspendParams, + EndParams, ErrorParams, ExitedParams, FailedUpdateParams, FinishSpanParams, GrowMemoryParams, + HostStreamFrameParams, InterruptedParams, JumpParams, LogParams, NoOpParams, + OplogProcessorCheckpointParams, PendingAgentInvocationParams, PendingUpdateParams, + PreCommitRemoteTransactionParams, PreRollbackRemoteTransactionParams, RemoveRetryPolicyParams, + RestartParams, RevertParams, RolledBackRemoteTransactionParams, SetRetryPolicyParams, + SetSpanAttributeParams, SnapshotParams, StartParams, StartSpanParams, SuccessfulUpdateParams, + SuspendParams, }; use crate::model::oplog::{ AgentTerminatedByQuotaError, DurableFunctionType, EphemeralCannotSuspendError, @@ -288,10 +288,6 @@ impl TryFrom for AgentError { Error::ExceededTableLimit(_) => Ok(Self::ExceededTableLimit), Error::ExceededHttpCallLimit(_) => Ok(Self::ExceededHttpCallLimit), Error::ExceededRpcCallLimit(_) => Ok(Self::ExceededRpcCallLimit), - Error::NodeOutOfFilesystemStorage(_) => Ok(Self::NodeOutOfFilesystemStorage), - Error::AgentExceededFilesystemStorageLimit(_) => { - Ok(Self::AgentExceededFilesystemStorageLimit) - } Error::AgentTerminatedByQuota(inner) => { Ok(Self::AgentTerminatedByQuota(AgentTerminatedByQuotaError { environment_id: inner @@ -364,14 +360,6 @@ impl From for golem_api_grpc::proto::golem::worker::AgentError { AgentError::ExceededRpcCallLimit => { Error::ExceededRpcCallLimit(grpc_worker::ExceededRpcCallLimit {}) } - AgentError::NodeOutOfFilesystemStorage => { - Error::NodeOutOfFilesystemStorage(grpc_worker::NodeOutOfFilesystemStorage {}) - } - AgentError::AgentExceededFilesystemStorageLimit => { - Error::AgentExceededFilesystemStorageLimit( - grpc_worker::AgentExceededFilesystemStorageLimit {}, - ) - } AgentError::AgentTerminatedByQuota(details) => { Error::AgentTerminatedByQuota(grpc_worker::AgentTerminatedByQuota { environment_id: Some(details.environment_id.into()), @@ -722,17 +710,6 @@ impl TryFrom for PublicOplogEn delta: grow_memory.delta, })) } - oplog_entry::Entry::FilesystemStorageUsageUpdate(filesystem_storage_usage_update) => { - Ok(PublicOplogEntry::FilesystemStorageUsageUpdate( - FilesystemStorageUsageUpdateParams { - timestamp: filesystem_storage_usage_update - .timestamp - .ok_or("Missing timestamp field")? - .into(), - delta: filesystem_storage_usage_update.delta, - }, - )) - } oplog_entry::Entry::CreateResource(create_resource) => { Ok(PublicOplogEntry::CreateResource(CreateResourceParams { timestamp: create_resource @@ -1021,8 +998,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::OplogEn agent_id: Some(create.agent_id.into()), agent_mode: golem_api_grpc::proto::golem::component::AgentMode::from( create.agent_mode, - ) - as i32, + ) as i32, component_revision: create.component_revision.into(), env: create.env.into_iter().collect(), config: create @@ -1045,23 +1021,19 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::OplogEn }, )), }, - PublicOplogEntry::Start(start) => { - golem_api_grpc::proto::golem::worker::OplogEntry { - entry: Some(oplog_entry::Entry::Start( - golem_api_grpc::proto::golem::worker::StartParameters { - timestamp: Some(start.timestamp.into()), - parent_start_index: start.parent_start_index.map(|id| id.as_u64()), - function_name: start.function_name.clone(), - invocation_id: start.invocation_id.map(Into::into), - observational_owner: start - .observational_owner - .map(|id| id.as_u64()), - request: start.request.map(Into::into), - durable_function_type: Some(start.durable_function_type.into()), - }, - )), - } - } + PublicOplogEntry::Start(start) => golem_api_grpc::proto::golem::worker::OplogEntry { + entry: Some(oplog_entry::Entry::Start( + golem_api_grpc::proto::golem::worker::StartParameters { + timestamp: Some(start.timestamp.into()), + parent_start_index: start.parent_start_index.map(|id| id.as_u64()), + function_name: start.function_name.clone(), + invocation_id: start.invocation_id.map(Into::into), + observational_owner: start.observational_owner.map(|id| id.as_u64()), + request: start.request.map(Into::into), + durable_function_type: Some(start.durable_function_type.into()), + }, + )), + }, PublicOplogEntry::End(end) => golem_api_grpc::proto::golem::worker::OplogEntry { entry: Some(oplog_entry::Entry::End( golem_api_grpc::proto::golem::worker::EndParameters { @@ -1246,16 +1218,6 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::OplogEn )), } } - PublicOplogEntry::FilesystemStorageUsageUpdate(filesystem_storage_usage_update) => { - golem_api_grpc::proto::golem::worker::OplogEntry { - entry: Some(oplog_entry::Entry::FilesystemStorageUsageUpdate( - golem_api_grpc::proto::golem::worker::FilesystemStorageUsageUpdateParameters { - timestamp: Some(filesystem_storage_usage_update.timestamp.into()), - delta: filesystem_storage_usage_update.delta, - }, - )), - } - } PublicOplogEntry::CreateResource(create_resource) => { golem_api_grpc::proto::golem::worker::OplogEntry { entry: Some(oplog_entry::Entry::CreateResource( @@ -2594,12 +2556,6 @@ impl TryFrom for OplogEntry { timestamp: p.timestamp, delta: p.delta, }), - PublicOplogEntry::FilesystemStorageUsageUpdate(p) => { - Ok(OplogEntry::FilesystemStorageUsageUpdate { - timestamp: p.timestamp, - delta: p.delta, - }) - } PublicOplogEntry::CreateResource(p) => Ok(OplogEntry::CreateResource { timestamp: p.timestamp, id: p.id, @@ -3161,8 +3117,7 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry RawCardInstalledParameters, RawCardRevokedParameters, RawCompletionDiscardedParameters, RawCreateParameters, RawCreateResourceParameters, RawDeactivatePluginParameters, RawDropResourceParameters, RawEndAtomicRegionParameters, RawEndParameters, RawEnvVar, - RawErrorParameters, RawFailedUpdateParameters, - RawFilesystemStorageUsageUpdateParameters, RawFinishSpanParameters, + RawErrorParameters, RawFailedUpdateParameters, RawFinishSpanParameters, RawGrowMemoryParameters, RawHostStreamFrameParameters, RawJumpParameters, RawLogParameters, RawOplogProcessorCheckpointParameters, RawOplogRegion, RawPendingAgentInvocationParameters, RawPendingUpdateParameters, @@ -3360,11 +3315,6 @@ impl TryFrom for golem_api_grpc::proto::golem::worker::RawOplogEntry OplogEntry::GrowMemory { delta, .. } => { Entry::GrowMemory(RawGrowMemoryParameters { delta }) } - OplogEntry::FilesystemStorageUsageUpdate { delta, .. } => { - Entry::FilesystemStorageUsageUpdate(RawFilesystemStorageUsageUpdateParameters { - delta, - }) - } OplogEntry::CreateResource { id, resource_type_id, @@ -3810,12 +3760,6 @@ impl TryFrom for OplogEntry timestamp, delta: p.delta, }), - Entry::FilesystemStorageUsageUpdate(p) => { - Ok(OplogEntry::FilesystemStorageUsageUpdate { - timestamp, - delta: p.delta, - }) - } Entry::CreateResource(p) => { let rt = p.resource_type_id.ok_or("Missing resource_type_id")?; Ok(OplogEntry::CreateResource { diff --git a/golem-common/src/model/oplog/raw_types.rs b/golem-common/src/model/oplog/raw_types.rs index 1b96482ed3..59bfca70de 100644 --- a/golem-common/src/model/oplog/raw_types.rs +++ b/golem-common/src/model/oplog/raw_types.rs @@ -328,10 +328,6 @@ pub enum AgentError { ExceededHttpCallLimit, // The worker exceeded the per-invocation RPC call limit from the plan ExceededRpcCallLimit, - // The executor-wide storage semaphore pool is exhausted (retriable) - NodeOutOfFilesystemStorage, - // The worker tried to use more storage than allowed by its plan (permanent) - AgentExceededFilesystemStorageLimit, // The agent was terminated by a quota with the terminae enforcement action (permanent) AgentTerminatedByQuota(AgentTerminatedByQuotaError), // Ephemeral agents cannot suspend and the requested sleep exceeded the configured maximum @@ -360,8 +356,6 @@ impl AgentError { Self::ExceededTableLimit => "Exceeded plan function table limit", Self::ExceededHttpCallLimit => "Exceeded per-invocation HTTP call limit", Self::ExceededRpcCallLimit => "Exceeded per-invocation RPC call limit", - Self::NodeOutOfFilesystemStorage => "Out of storage space", - Self::AgentExceededFilesystemStorageLimit => "Exceeded plan storage limit", Self::AgentTerminatedByQuota(_) => "Terminated by quota", Self::EphemeralSleepTooLong(_) => "Ephemeral sleep too long", Self::EphemeralFuelExhausted(_) => "Ephemeral fuel exhausted", diff --git a/golem-common/wit/deps/golem-1.x/golem-oplog.wit b/golem-common/wit/deps/golem-1.x/golem-oplog.wit index 28f7d9d514..97ef23469d 100644 --- a/golem-common/wit/deps/golem-1.x/golem-oplog.wit +++ b/golem-common/wit/deps/golem-1.x/golem-oplog.wit @@ -406,11 +406,6 @@ interface oplog { delta: u64 } - record filesystem-storage-usage-update-parameters { - timestamp: datetime, - delta: s64 - } - type agent-resource-id = u64; record create-resource-parameters { @@ -566,8 +561,6 @@ interface oplog { exceeded-table-limit, exceeded-http-call-limit, exceeded-rpc-call-limit, - node-out-of-filesystem-storage, - agent-exceeded-filesystem-storage-limit, agent-terminated-by-quota(agent-terminated-by-quota-error), ephemeral-sleep-too-long(ephemeral-sleep-too-long), ephemeral-fuel-exhausted(ephemeral-fuel-exhausted), @@ -786,8 +779,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(raw-create-resource-parameters), /// Dropped a resource instance @@ -895,8 +886,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(create-resource-parameters), /// Dropped a resource instance diff --git a/golem-debugging-service/src/debug_context.rs b/golem-debugging-service/src/debug_context.rs index 011cd8e601..0927e08498 100644 --- a/golem-debugging-service/src/debug_context.rs +++ b/golem-debugging-service/src/debug_context.rs @@ -575,6 +575,8 @@ impl WorkerCtx for DebugContext { component_service: Arc, _extra_deps: Self::ExtraDeps, config: Arc, + filesystem_root: std::path::PathBuf, + filesystem_runtime: golem_worker_executor::services::agent_filesystem::AgentFilesystemRuntime, worker_config: AgentConfig, execution_status: Arc>, file_loader: Arc, @@ -619,6 +621,8 @@ impl WorkerCtx for DebugContext { component_service, account_resource_limits, config, + filesystem_root, + filesystem_runtime, worker_config, execution_status, file_loader, diff --git a/golem-debugging-service/src/oplog/debug_oplog.rs b/golem-debugging-service/src/oplog/debug_oplog.rs index 55bfa45c74..6a8387671a 100644 --- a/golem-debugging-service/src/oplog/debug_oplog.rs +++ b/golem-debugging-service/src/oplog/debug_oplog.rs @@ -125,16 +125,16 @@ impl Oplog for DebugOplog { .oplog_state .debug_session .get(&self.oplog_state.debug_session_id) - .await - .expect("Internal Error. Current Oplog Index failed. Debug session not found"); + .await; - // If a debug session not found but hasn't been set up with a target index, - // it implies, we only connected to the worker and haven't started debugging yet. - if let Some(index) = debug_session_data.target_oplog_index { - index - } else { - self.inner.current_oplog_index().await + if let Some(debug_session_data) = debug_session_data + && let Some(index) = debug_session_data.target_oplog_index + { + return index; } + + // Worker construction precedes session registration when first connecting. + self.inner.current_oplog_index().await } async fn last_added_non_hint_entry(&self) -> Option { @@ -147,18 +147,19 @@ impl Oplog for DebugOplog { // Reads never move the debug session's replay position: replay's single-entry reads are // speculative (progress is only committed via `on_replay_progress`), and other components - // (for example P3 request-body reconstruction) perform unrelated point lookups. + // (for example P3 request-body reconstruction) perform unrelated point lookups. Worker + // construction may read before session registration, when the raw oplog is the only view. async fn read(&self, oplog_index: OplogIndex) -> OplogEntry { - let debug_session_data = self + let playback_overrides = self .oplog_state .debug_session .get(&self.oplog_state.debug_session_id) .await - .expect("Internal Error. Read failed. Debug session not found"); - let playback_overrides = debug_session_data.playback_overrides.clone(); + .map(|data| data.playback_overrides.overrides) + .unwrap_or_default(); Self::get_oplog_entry_applying_overrides( - playback_overrides.overrides, + playback_overrides, oplog_index, self.inner.clone(), ) @@ -187,20 +188,16 @@ impl Oplog for DebugOplog { // Like `read`, this never moves the debug session's replay position; it only applies the // playback overrides on top of the underlying entries. - let debug_session_data = self + let playback_overrides = self .oplog_state .debug_session .get(&self.oplog_state.debug_session_id) .await - .expect("Internal Error. Read failed. Debug session not found"); - let playback_overrides = debug_session_data.playback_overrides; + .map(|data| data.playback_overrides.overrides) + .unwrap_or_default(); for (idx, entry) in self.inner.read_many(oplog_index, count).await { - let entry = playback_overrides - .overrides - .get(&idx) - .cloned() - .unwrap_or(entry); + let entry = playback_overrides.get(&idx).cloned().unwrap_or(entry); result.insert(idx, entry); } result diff --git a/golem-service-base/src/error/worker_executor.rs b/golem-service-base/src/error/worker_executor.rs index 9a722b5928..de30c6db20 100644 --- a/golem-service-base/src/error/worker_executor.rs +++ b/golem-service-base/src/error/worker_executor.rs @@ -981,8 +981,6 @@ pub enum GolemSpecificWasmTrap { WorkerExceededTableLimit, WorkerExceededHttpCallLimit, WorkerExceededRpcCallLimit, - NodeOutOfFilesystemStorage, - WorkerAgentExceededFilesystemStorageLimit, WorkerMonthlyHttpCallBudgetExhausted, WorkerMonthlyRpcCallBudgetExhausted, AgentTerminatedByQuota { @@ -1016,12 +1014,6 @@ impl Display for GolemSpecificWasmTrap { Self::WorkerExceededRpcCallLimit => { write!(f, "Worker exceeded per-invocation RPC call limit") } - Self::NodeOutOfFilesystemStorage => { - write!(f, "Worker cannot acquire more storage space") - } - Self::WorkerAgentExceededFilesystemStorageLimit => { - write!(f, "Worker exceeded plan storage limits") - } Self::WorkerMonthlyHttpCallBudgetExhausted => { write!(f, "Worker exhausted monthly HTTP call budget") } diff --git a/golem-skills/skills/common/golem-debug-agent-history/SKILL.md b/golem-skills/skills/common/golem-debug-agent-history/SKILL.md index 395ad0629d..8b35df6391 100644 --- a/golem-skills/skills/common/golem-debug-agent-history/SKILL.md +++ b/golem-skills/skills/common/golem-debug-agent-history/SKILL.md @@ -63,7 +63,6 @@ Each text entry is printed with its index (e.g. `#00042:`) followed by a labeled | `INTERRUPTED` / `EXITED` | Agent interrupted or exited | | `NOP` | No-operation marker | | `JUMP` | Oplog jump — shows from/to indices | -| `STORAGE USAGE UPDATE` | Filesystem storage usage change | ### Examples diff --git a/golem-test-framework/src/dsl/debug_render.rs b/golem-test-framework/src/dsl/debug_render.rs index be6deed760..cfdeb116b3 100644 --- a/golem-test-framework/src/dsl/debug_render.rs +++ b/golem-test-framework/src/dsl/debug_render.rs @@ -252,11 +252,6 @@ pub fn debug_render_oplog_entry(entry: &PublicOplogEntry) -> String { let _ = writeln!(result, "{pad}at: {}", params.timestamp); let _ = writeln!(result, "{pad}increase: {}", params.delta,); } - PublicOplogEntry::FilesystemStorageUsageUpdate(params) => { - let _ = writeln!(result, "STORAGE USAGE UPDATE"); - let _ = writeln!(result, "{pad}at: {}", params.timestamp); - let _ = writeln!(result, "{pad}delta: {}", params.delta); - } PublicOplogEntry::CreateResource(params) => { let _ = writeln!(result, "CREATE RESOURCE"); let _ = writeln!(result, "{pad}at: {}", params.timestamp); diff --git a/golem-worker-executor-test-utils/src/lib.rs b/golem-worker-executor-test-utils/src/lib.rs index 34b4d66273..1f144602bc 100644 --- a/golem-worker-executor-test-utils/src/lib.rs +++ b/golem-worker-executor-test-utils/src/lib.rs @@ -106,11 +106,11 @@ use golem_worker_executor::services::environment_state::EnvironmentStateService; use golem_worker_executor::services::file_loader::FileLoader; use golem_worker_executor::services::golem_config::{ AgentTypesServiceConfig, AgentTypesServiceLocalConfig, EngineConfig, - EnvironmentStateServiceConfig, FilesystemStorageConfig, GolemConfig, GrpcApiConfig, - HttpClientConfig, IndexedStorageConfig, IndexedStorageKVStoreRedisConfig, - IndexedStorageKVStoreSqliteConfig, KeyValueStorageConfig, KeyValueStorageInnerConfig, - KeyValueStorageNamespaceRoutedConfig, MemoryConfig, OplogConfig, ResourceLimitsConfig, - ResourceLimitsDisabledConfig, SchedulerStorageConfig, SnapshotPolicy, + EnvironmentStateServiceConfig, GolemConfig, GrpcApiConfig, HttpClientConfig, + IndexedStorageConfig, IndexedStorageKVStoreRedisConfig, IndexedStorageKVStoreSqliteConfig, + KeyValueStorageConfig, KeyValueStorageInnerConfig, KeyValueStorageNamespaceRoutedConfig, + MemoryConfig, OplogConfig, ResourceLimitsConfig, ResourceLimitsDisabledConfig, + SchedulerStorageConfig, SnapshotPolicy, }; use golem_worker_executor::services::key_value::{DefaultKeyValueService, KeyValueService}; use golem_worker_executor::services::oplog::{CommitLevel, Oplog, OplogService, OrderedOplogStart}; @@ -627,6 +627,15 @@ impl TestWorkerExecutor { } } + pub async fn stop_worker_if_idle(&self, owned_agent_id: &OwnedAgentId) -> anyhow::Result { + let worker = self + .additional_test_deps + .try_get_worker(owned_agent_id) + .await + .ok_or_else(|| anyhow!("worker {owned_agent_id} is not currently in ActiveWorkers"))?; + Ok(worker.stop_if_idle().await) + } + /// Returns the current eviction classification for the worker shell /// registered in `ActiveWorkers`, or `None` if the worker is missing or /// non-evictable. Used by tests to wait until the worker is `LoadedIdle` @@ -884,7 +893,7 @@ pub async fn start( deps: &WorkerExecutorTestDependencies, context: &TestContext, ) -> anyhow::Result { - start_customized(deps, context, None, None, None, None, None, None).await + start_customized(deps, context, None, None, None, None, None).await } pub async fn start_with_snapshot_policy( @@ -892,17 +901,7 @@ pub async fn start_with_snapshot_policy( context: &TestContext, snapshot_policy: SnapshotPolicy, ) -> anyhow::Result { - start_customized( - deps, - context, - None, - None, - None, - Some(snapshot_policy), - None, - None, - ) - .await + start_customized(deps, context, None, None, Some(snapshot_policy), None, None).await } pub async fn start_with_http_client_config( @@ -910,17 +909,7 @@ pub async fn start_with_http_client_config( context: &TestContext, http_client: HttpClientConfig, ) -> anyhow::Result { - start_customized( - deps, - context, - None, - None, - None, - None, - Some(http_client), - None, - ) - .await + start_customized(deps, context, None, None, None, Some(http_client), None).await } pub async fn start_with_oplog_config( @@ -928,17 +917,7 @@ pub async fn start_with_oplog_config( context: &TestContext, oplog_config_override: Option, ) -> anyhow::Result { - start_customized( - deps, - context, - None, - None, - None, - None, - None, - oplog_config_override, - ) - .await + start_customized(deps, context, None, None, None, None, oplog_config_override).await } pub async fn start_with_redis_storage( @@ -1175,7 +1154,6 @@ pub async fn start_customized( deps: &WorkerExecutorTestDependencies, context: &TestContext, system_memory_override: Option, - system_storage_override: Option, retry_override: Option, snapshot_policy_override: Option, http_client_override: Option, @@ -1187,10 +1165,6 @@ pub async fn start_customized( system_memory_override, ..Default::default() }; - config.filesystem_storage = FilesystemStorageConfig { - total_worker_filesystem_storage_bytes: system_storage_override, - ..Default::default() - }; if let Some(retry) = retry_override { config.retry = retry; } @@ -1537,6 +1511,8 @@ impl WorkerCtx for TestWorkerCtx { component_service: Arc, extra_deps: Self::ExtraDeps, config: Arc, + filesystem_root: PathBuf, + filesystem_runtime: golem_worker_executor::services::agent_filesystem::AgentFilesystemRuntime, worker_config: AgentConfig, execution_status: Arc>, file_loader: Arc, @@ -1591,6 +1567,8 @@ impl WorkerCtx for TestWorkerCtx { component_service, account_resource_limits, config, + filesystem_root, + filesystem_runtime, worker_config, execution_status, file_loader, @@ -1887,7 +1865,7 @@ impl Bootstrap for TestServerBootstrap { &self, golem_config: &GolemConfig, shutdown_token: tokio_util::sync::CancellationToken, - ) -> Arc> { + ) -> anyhow::Result>> { // The in-process test harness shares its process (and RSS) with the test // framework and other services, so a process-RSS probe cannot isolate // this executor's footprint. Disable measured admission for ordinary @@ -1897,22 +1875,22 @@ impl Bootstrap for TestServerBootstrap { // granted accounting (exact and process-isolated) against the pinned // limit. The usable_ratio (worker_memory_ratio) still applies. match golem_config.memory.system_memory_override { - Some(limit) => Arc::new(ActiveWorkers::new_with_probe( + Some(limit) => Ok(Arc::new(ActiveWorkers::new_with_probe( Box::new(FixedProbe::new(limit, 0)), &golem_config.memory, &golem_config.filesystem_storage, &golem_config.agent_status_flush, shutdown_token, - )), + )?)), None => { let mut memory_config = golem_config.memory.clone(); memory_config.enable_measured_admission = false; - Arc::new(ActiveWorkers::new( + Ok(Arc::new(ActiveWorkers::new( &memory_config, &golem_config.filesystem_storage, &golem_config.agent_status_flush, shutdown_token, - )) + )?)) } } } @@ -2205,14 +2183,20 @@ fn make_production_context_config( config } +type ProductionContextConfigOverride = Arc; + async fn run_production_context_bootstrap( deps: &WorkerExecutorTestDependencies, context: &TestContext, resource_limits: Arc, + configure: Option, timeout_msg: &'static str, ) -> anyhow::Result { let prometheus = golem_worker_executor::metrics::register_all(); - let config = make_production_context_config(deps, context); + let mut config = make_production_context_config(deps, context); + if let Some(configure) = configure { + configure(&mut config); + } let handle = tokio::runtime::Handle::current(); let mut join_set = tokio::task::JoinSet::new(); @@ -2295,6 +2279,7 @@ pub async fn start_with_resource_limits( deps, context, resource_limits, + None, "Timeout waiting for custom-resource-limits server to start", ) .await @@ -2314,6 +2299,7 @@ pub async fn start_with_table_limit( deps, context, Arc::new(FixedTableLimitResourceLimits { max_table_elements }), + None, "Timeout waiting for table-limit server to start", ) .await @@ -2361,6 +2347,7 @@ pub async fn start_with_concurrent_agent_limit( Arc::new(FixedConcurrentAgentLimitResourceLimits { max_concurrent_agents_per_executor: max_concurrent_agents, }), + None, "Timeout waiting for concurrent-agent-limit server to start", ) .await @@ -2373,6 +2360,43 @@ struct FixedFilesystemStorageQuotaResourceLimits { max_disk_space_bytes: u64, } +#[derive(Clone)] +pub struct MutableFilesystemStorageQuota { + entry: Arc, +} + +impl MutableFilesystemStorageQuota { + pub async fn set_limit(&self, allocated_bytes: u64) -> anyhow::Result<()> { + self.entry + .apply_agent_filesystem_limit(allocated_bytes) + .await + .map_err(|(agent_id, error)| { + anyhow::anyhow!("failed to apply filesystem limit to agent {agent_id}: {error}") + }) + } + + pub fn flush_durable_storage_byte_seconds(&self) -> i64 { + self.entry.flush_durable_storage_byte_seconds_for_test() + } +} + +struct MutableFilesystemStorageQuotaResourceLimits { + entry: Arc, +} + +#[async_trait] +impl ResourceLimits for MutableFilesystemStorageQuotaResourceLimits { + async fn initialize_account( + &self, + _account_id: golem_common::model::account::AccountId, + ) -> Result< + Arc, + golem_service_base::error::worker_executor::WorkerExecutorError, + > { + Ok(Arc::clone(&self.entry)) + } +} + #[async_trait] impl ResourceLimits for FixedFilesystemStorageQuotaResourceLimits { async fn initialize_account( @@ -2394,10 +2418,7 @@ impl ResourceLimits for FixedFilesystemStorageQuotaResourceLimits { /// Starts a worker executor with a per-agent plan-level storage limit. /// -/// Uses the production [`Context`] so that `check_filesystem_quota` enforces -/// `max_disk_space_bytes` against each agent's `current_filesystem_storage_usage`. -/// Exceeding it returns `WorkerAgentExceededFilesystemStorageLimit` (permanent, not retried). -/// The executor-wide semaphore pool is left unlimited (10 GB default). +/// Managed-XFS variants install this limit as an authoritative project quota. pub async fn start_with_agent_storage_quota( deps: &WorkerExecutorTestDependencies, context: &TestContext, @@ -2409,11 +2430,94 @@ pub async fn start_with_agent_storage_quota( Arc::new(FixedFilesystemStorageQuotaResourceLimits { max_disk_space_bytes, }), + None, "Timeout waiting for agent-storage-quota server to start", ) .await } +#[cfg(target_os = "linux")] +pub async fn start_with_agent_storage_quota_on_managed_xfs( + deps: &WorkerExecutorTestDependencies, + context: &TestContext, + max_disk_space_bytes: u64, + managed_xfs_root: PathBuf, +) -> anyhow::Result { + run_production_context_bootstrap( + deps, + context, + Arc::new(FixedFilesystemStorageQuotaResourceLimits { + max_disk_space_bytes, + }), + Some(Arc::new(move |config| { + config.filesystem_storage.managed_xfs_root_dir = Some(managed_xfs_root.clone()); + config.oplog.default_snapshotting = SnapshotPolicy::Disabled; + config.oplog.oplog_processor_snapshotting = SnapshotPolicy::Disabled; + })), + "Timeout waiting for managed agent-storage-quota server to start", + ) + .await +} + +#[cfg(target_os = "linux")] +pub async fn start_with_agent_storage_and_object_quota_on_managed_xfs( + deps: &WorkerExecutorTestDependencies, + context: &TestContext, + max_disk_space_bytes: u64, + filesystem_objects: u64, + managed_xfs_root: PathBuf, +) -> anyhow::Result { + run_production_context_bootstrap( + deps, + context, + Arc::new(FixedFilesystemStorageQuotaResourceLimits { + max_disk_space_bytes, + }), + Some(Arc::new(move |config| { + config.filesystem_storage.managed_xfs_root_dir = Some(managed_xfs_root.clone()); + let policy = &mut config.filesystem_storage.filesystem_object_limit_policy; + policy.objects_per_gib = 1; + policy.minimum_objects = filesystem_objects; + policy.maximum_objects = filesystem_objects; + })), + "Timeout waiting for managed agent storage/object quota server to start", + ) + .await +} + +#[cfg(target_os = "linux")] +pub async fn start_with_mutable_agent_storage_quota_on_managed_xfs( + deps: &WorkerExecutorTestDependencies, + context: &TestContext, + max_disk_space_bytes: u64, + managed_xfs_root: PathBuf, +) -> anyhow::Result<(TestWorkerExecutor, MutableFilesystemStorageQuota)> { + let entry = Arc::new(AtomicResourceEntry::new( + u64::MAX, + usize::MAX, + usize::MAX, + max_disk_space_bytes, + u64::MAX, + )); + let executor = run_production_context_bootstrap( + deps, + context, + Arc::new(MutableFilesystemStorageQuotaResourceLimits { + entry: Arc::clone(&entry), + }), + Some(Arc::new(move |config| { + config.filesystem_storage.managed_xfs_root_dir = Some(managed_xfs_root.clone()); + let policy = &mut config.filesystem_storage.filesystem_object_limit_policy; + policy.objects_per_gib = 262_144; + policy.minimum_objects = 1; + policy.maximum_objects = 1024; + })), + "Timeout waiting for mutable managed agent-storage-quota server to start", + ) + .await?; + Ok((executor, MutableFilesystemStorageQuota { entry })) +} + /// A `ResourceLimits` implementation that enforces fixed per-invocation HTTP and RPC /// call limits while keeping fuel, memory, and table elements unlimited. /// Used by per-invocation call-limit tests. @@ -2459,6 +2563,7 @@ pub async fn start_with_invocation_limits( per_invocation_http_call_limit, per_invocation_rpc_call_limit, }), + None, "Timeout waiting for invocation-limit server to start", ) .await @@ -2514,30 +2619,8 @@ pub async fn start_with_monthly_call_limits( monthly_http_calls, monthly_rpc_calls, }), - "Timeout waiting for monthly-call-limit server to start", - ) - .await -} - -/// Starts a worker executor with a constrained executor-wide storage pool. -/// -/// The pool is shared across all agents on the node. Uses `TestWorkerCtx` -/// (no per-agent plan limit). Exhausting the pool returns `NodeOutOfFilesystemStorage` -/// (retriable). Use this to test node-level storage pressure and eviction. -pub async fn start_with_executor_storage_pool( - deps: &WorkerExecutorTestDependencies, - context: &TestContext, - pool_bytes: u64, -) -> anyhow::Result { - start_customized( - deps, - context, - None, - Some(pool_bytes), - None, - None, - None, None, + "Timeout waiting for monthly-call-limit server to start", ) .await } diff --git a/golem-worker-executor/Cargo.toml b/golem-worker-executor/Cargo.toml index 42e71c2eb4..379e8675d6 100644 --- a/golem-worker-executor/Cargo.toml +++ b/golem-worker-executor/Cargo.toml @@ -44,6 +44,7 @@ bigdecimal = { workspace = true } bit-vec = { workspace = true } blake3 = { workspace = true } bytes = { workspace = true } +cap-fs-ext = { workspace = true } cap-std = { workspace = true } cap-time-ext = { workspace = true } # keep in sync with wasmtime chrono = { workspace = true } @@ -111,6 +112,11 @@ zstd = { workspace = true } [target.'cfg(windows)'.dependencies] windows-sys = { workspace = true } +[target.'cfg(target_os = "linux")'.dependencies] +libc = { workspace = true } +linux-raw-sys = { workspace = true } +rustix = { workspace = true } + [[bench]] name = "invocation_lowering" harness = false diff --git a/golem-worker-executor/config/worker-executor.sample.env b/golem-worker-executor/config/worker-executor.sample.env index 85aee455f5..e809cb5f10 100644 --- a/golem-worker-executor/config/worker-executor.sample.env +++ b/golem-worker-executor/config/worker-executor.sample.env @@ -33,9 +33,22 @@ GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_CAPACITY=1000 GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_EVICTION_INTERVAL="1m" GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_TTL__NANOS=0 GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_TTL__SECS=300 -GOLEM__FILESYSTEM_STORAGE__ACQUIRE_RETRY_DELAY="500ms" #GOLEM__FILESYSTEM_STORAGE__DETERMINISTIC_ROOT_DIR= -#GOLEM__FILESYSTEM_STORAGE__TOTAL_WORKER_FILESYSTEM_STORAGE_BYTES= +#GOLEM__FILESYSTEM_STORAGE__MANAGED_XFS_ROOT_DIR= +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MAX_ATTEMPTS=4 +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MAX_DELAY="250ms" +#GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MAX_JITTER_FACTOR= +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MIN_DELAY="25ms" +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MULTIPLIER=4.0 +GOLEM__FILESYSTEM_STORAGE__FILESYSTEM_OBJECT_LIMIT_POLICY__MAXIMUM_OBJECTS=131072 +GOLEM__FILESYSTEM_STORAGE__FILESYSTEM_OBJECT_LIMIT_POLICY__MINIMUM_OBJECTS=8192 +GOLEM__FILESYSTEM_STORAGE__FILESYSTEM_OBJECT_LIMIT_POLICY__OBJECTS_PER_GIB=32768 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__MINIMUM_AVAILABLE_BYTES=67108864 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__MINIMUM_AVAILABLE_FILESYSTEM_OBJECTS=8192 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__RECLAMATION_OBSERVATION_ATTEMPTS=4 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__RECLAMATION_OBSERVATION_DELAY="25ms" +GOLEM__FILESYSTEM_STORAGE__PRESSURE__TARGET_AVAILABLE_BYTES=134217728 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__TARGET_AVAILABLE_FILESYSTEM_OBJECTS=16384 GOLEM__GRPC__MAX_MESSAGE_SIZE=33554432 GOLEM__GRPC__PORT=9093 GOLEM__GRPC__TLS__TYPE="Disabled" @@ -283,9 +296,22 @@ GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_CAPACITY=1000 GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_EVICTION_INTERVAL="1m" GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_TTL__NANOS=0 GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_TTL__SECS=300 -GOLEM__FILESYSTEM_STORAGE__ACQUIRE_RETRY_DELAY="500ms" #GOLEM__FILESYSTEM_STORAGE__DETERMINISTIC_ROOT_DIR= -#GOLEM__FILESYSTEM_STORAGE__TOTAL_WORKER_FILESYSTEM_STORAGE_BYTES= +#GOLEM__FILESYSTEM_STORAGE__MANAGED_XFS_ROOT_DIR= +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MAX_ATTEMPTS=4 +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MAX_DELAY="250ms" +#GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MAX_JITTER_FACTOR= +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MIN_DELAY="25ms" +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MULTIPLIER=4.0 +GOLEM__FILESYSTEM_STORAGE__FILESYSTEM_OBJECT_LIMIT_POLICY__MAXIMUM_OBJECTS=131072 +GOLEM__FILESYSTEM_STORAGE__FILESYSTEM_OBJECT_LIMIT_POLICY__MINIMUM_OBJECTS=8192 +GOLEM__FILESYSTEM_STORAGE__FILESYSTEM_OBJECT_LIMIT_POLICY__OBJECTS_PER_GIB=32768 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__MINIMUM_AVAILABLE_BYTES=67108864 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__MINIMUM_AVAILABLE_FILESYSTEM_OBJECTS=8192 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__RECLAMATION_OBSERVATION_ATTEMPTS=4 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__RECLAMATION_OBSERVATION_DELAY="25ms" +GOLEM__FILESYSTEM_STORAGE__PRESSURE__TARGET_AVAILABLE_BYTES=134217728 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__TARGET_AVAILABLE_FILESYSTEM_OBJECTS=16384 GOLEM__GRPC__MAX_MESSAGE_SIZE=33554432 GOLEM__GRPC__PORT=9093 GOLEM__GRPC__TLS__TYPE="Disabled" @@ -512,9 +538,22 @@ GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_CAPACITY=1000 GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_EVICTION_INTERVAL="1m" GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_TTL__NANOS=0 GOLEM__ENVIRONMENT_STATE_SERVICE__CACHE_TTL__SECS=300 -GOLEM__FILESYSTEM_STORAGE__ACQUIRE_RETRY_DELAY="500ms" #GOLEM__FILESYSTEM_STORAGE__DETERMINISTIC_ROOT_DIR= -#GOLEM__FILESYSTEM_STORAGE__TOTAL_WORKER_FILESYSTEM_STORAGE_BYTES= +#GOLEM__FILESYSTEM_STORAGE__MANAGED_XFS_ROOT_DIR= +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MAX_ATTEMPTS=4 +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MAX_DELAY="250ms" +#GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MAX_JITTER_FACTOR= +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MIN_DELAY="25ms" +GOLEM__FILESYSTEM_STORAGE__CLEANUP_RETRY__MULTIPLIER=4.0 +GOLEM__FILESYSTEM_STORAGE__FILESYSTEM_OBJECT_LIMIT_POLICY__MAXIMUM_OBJECTS=131072 +GOLEM__FILESYSTEM_STORAGE__FILESYSTEM_OBJECT_LIMIT_POLICY__MINIMUM_OBJECTS=8192 +GOLEM__FILESYSTEM_STORAGE__FILESYSTEM_OBJECT_LIMIT_POLICY__OBJECTS_PER_GIB=32768 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__MINIMUM_AVAILABLE_BYTES=67108864 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__MINIMUM_AVAILABLE_FILESYSTEM_OBJECTS=8192 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__RECLAMATION_OBSERVATION_ATTEMPTS=4 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__RECLAMATION_OBSERVATION_DELAY="25ms" +GOLEM__FILESYSTEM_STORAGE__PRESSURE__TARGET_AVAILABLE_BYTES=134217728 +GOLEM__FILESYSTEM_STORAGE__PRESSURE__TARGET_AVAILABLE_FILESYSTEM_OBJECTS=16384 GOLEM__GRPC__MAX_MESSAGE_SIZE=33554432 GOLEM__GRPC__PORT=9093 GOLEM__GRPC__TLS__TYPE="Disabled" diff --git a/golem-worker-executor/config/worker-executor.toml b/golem-worker-executor/config/worker-executor.toml index 2d58be938b..943a4b03f4 100644 --- a/golem-worker-executor/config/worker-executor.toml +++ b/golem-worker-executor/config/worker-executor.toml @@ -63,8 +63,24 @@ cache_eviction_interval = "1m" nanos = 0 secs = 300 -[filesystem_storage] -acquire_retry_delay = "500ms" +[filesystem_storage.cleanup_retry] +max_attempts = 4 +max_delay = "250ms" +min_delay = "25ms" +multiplier = 4.0 + +[filesystem_storage.filesystem_object_limit_policy] +maximum_objects = 131072 +minimum_objects = 8192 +objects_per_gib = 32768 + +[filesystem_storage.pressure] +minimum_available_bytes = 67108864 +minimum_available_filesystem_objects = 8192 +reclamation_observation_attempts = 4 +reclamation_observation_delay = "25ms" +target_available_bytes = 134217728 +target_available_filesystem_objects = 16384 [grpc] max_message_size = 33554432 @@ -440,8 +456,24 @@ without_time = false # nanos = 0 # secs = 300 # -# [filesystem_storage] -# acquire_retry_delay = "500ms" +# [filesystem_storage.cleanup_retry] +# max_attempts = 4 +# max_delay = "250ms" +# min_delay = "25ms" +# multiplier = 4.0 +# +# [filesystem_storage.filesystem_object_limit_policy] +# maximum_objects = 131072 +# minimum_objects = 8192 +# objects_per_gib = 32768 +# +# [filesystem_storage.pressure] +# minimum_available_bytes = 67108864 +# minimum_available_filesystem_objects = 8192 +# reclamation_observation_attempts = 4 +# reclamation_observation_delay = "25ms" +# target_available_bytes = 134217728 +# target_available_filesystem_objects = 16384 # # [grpc] # max_message_size = 33554432 @@ -790,8 +822,24 @@ without_time = false # nanos = 0 # secs = 300 # -# [filesystem_storage] -# acquire_retry_delay = "500ms" +# [filesystem_storage.cleanup_retry] +# max_attempts = 4 +# max_delay = "250ms" +# min_delay = "25ms" +# multiplier = 4.0 +# +# [filesystem_storage.filesystem_object_limit_policy] +# maximum_objects = 131072 +# minimum_objects = 8192 +# objects_per_gib = 32768 +# +# [filesystem_storage.pressure] +# minimum_available_bytes = 67108864 +# minimum_available_filesystem_objects = 8192 +# reclamation_observation_attempts = 4 +# reclamation_observation_delay = "25ms" +# target_available_bytes = 134217728 +# target_available_filesystem_objects = 16384 # # [grpc] # max_message_size = 33554432 diff --git a/golem-worker-executor/src/durable_host/concurrent/call.rs b/golem-worker-executor/src/durable_host/concurrent/call.rs index dd34686a63..49e320573c 100644 --- a/golem-worker-executor/src/durable_host/concurrent/call.rs +++ b/golem-worker-executor/src/durable_host/concurrent/call.rs @@ -2992,7 +2992,7 @@ struct AccessRevisionUpdateInputs { owned_agent_id: golem_common::model::OwnedAgentId, agent_id: Option, initial_agent_config: Vec, - worker_dir: PathBuf, + filesystem_runtime: crate::services::agent_filesystem::AgentFilesystemRuntime, current_revision: ComponentRevision, } @@ -3005,7 +3005,7 @@ type AccessRevisionUpdateAgentState = ( struct AccessRevisionUpdate { metadata: Component, agent_state: Option, - files: HashMap, + filesystem_update: crate::services::agent_filesystem::AgentFilesystemUpdateEffectLease, } async fn finalize_pending_automatic_update_access( @@ -3103,7 +3103,7 @@ where owned_agent_id: ctx.owned_agent_id.clone(), agent_id: ctx.state.agent_id.clone(), initial_agent_config: ctx.state.initial_agent_config.clone(), - worker_dir: ctx.worker_dir.path().to_path_buf(), + filesystem_runtime: ctx.filesystem_runtime(), current_revision: ctx.state.component_metadata.revision, } }); @@ -3122,8 +3122,8 @@ where } async fn prepare_revision_update_access( - store: &Accessor, - get_ctx: fn(&mut T) -> &mut DurableWorkerCtx, + _store: &Accessor, + _get_ctx: fn(&mut T) -> &mut DurableWorkerCtx, inputs: &AccessRevisionUpdateInputs, new_revision: ComponentRevision, ) -> Result @@ -3187,70 +3187,23 @@ where None }; - let mut files = take_initial_files_access(store, get_ctx)?; - let update_result = super::super::update_filesystem( - &mut files, - &inputs.file_loader, - inputs.owned_agent_id.environment_id, - &inputs.worker_dir, - provision_config - .as_ref() - .map(|c| c.files.as_slice()) - .unwrap_or_default(), - ) - .await; - - if let Err(error) = update_result { - restore_initial_files_access(store, get_ctx, files)?; - return Err(error); - } + let filesystem_update = inputs + .filesystem_runtime + .update_initial_files( + &inputs.file_loader, + inputs.owned_agent_id.environment_id, + provision_config + .as_ref() + .map(|c| c.files.as_slice()) + .unwrap_or_default(), + ) + .await + .map_err(|error| WorkerExecutorError::runtime(error.to_string()))?; Ok(AccessRevisionUpdate { metadata, agent_state, - files, - }) -} - -fn take_initial_files_access( - store: &Accessor, - get_ctx: fn(&mut T) -> &mut DurableWorkerCtx, -) -> Result, WorkerExecutorError> -where - T: 'static, - D: HasData + ?Sized, - Ctx: WorkerCtx, -{ - store.with(|mut access| { - let ctx = get_ctx(access.data_mut()); - let mut files = ctx.state.files.try_write().map_err(|_| { - WorkerExecutorError::runtime( - "p3 accessor durable call path cannot acquire initial-files lock", - ) - })?; - Ok(std::mem::take(&mut *files)) - }) -} - -fn restore_initial_files_access( - store: &Accessor, - get_ctx: fn(&mut T) -> &mut DurableWorkerCtx, - restored: HashMap, -) -> Result<(), WorkerExecutorError> -where - T: 'static, - D: HasData + ?Sized, - Ctx: WorkerCtx, -{ - store.with(|mut access| { - let ctx = get_ctx(access.data_mut()); - let mut files = ctx.state.files.try_write().map_err(|_| { - WorkerExecutorError::runtime( - "p3 accessor durable call path cannot restore initial-files state", - ) - })?; - *files = restored; - Ok(()) + filesystem_update, }) } @@ -3258,27 +3211,18 @@ fn apply_revision_update_access( ctx: &mut DurableWorkerCtx, update: AccessRevisionUpdate, ) { - let read_only_paths = super::super::compute_read_only_paths(&update.files); - { - let mut files = ctx - .state - .files - .try_write() - .expect("initial-files state was taken by this update path"); - *files = update.files; - } - { - let mut read_only = ctx.state.read_only_paths.write().unwrap(); - *read_only = read_only_paths; - } - - if let Some((agent_config, effective_surface, initial_wallet_cards)) = update.agent_state { + let AccessRevisionUpdate { + metadata, + agent_state, + filesystem_update: _filesystem_update, + } = update; + if let Some((agent_config, effective_surface, initial_wallet_cards)) = agent_state { ctx.state.agent_config = agent_config; ctx.state.cached_agent_config_retry_policies = None; ctx.state.agent_effective_surface = effective_surface; ctx.state.agent_wallet_cards = initial_wallet_cards; } - ctx.state.component_metadata = update.metadata; + ctx.state.component_metadata = metadata; } async fn record_worker_update_failed_access( diff --git a/golem-worker-executor/src/durable_host/concurrent/mod.rs b/golem-worker-executor/src/durable_host/concurrent/mod.rs index 9e7234c82b..af4c83b241 100644 --- a/golem-worker-executor/src/durable_host/concurrent/mod.rs +++ b/golem-worker-executor/src/durable_host/concurrent/mod.rs @@ -28,7 +28,6 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::future::Future; use std::marker::PhantomData; -use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -62,7 +61,7 @@ use crate::durable_host::durability::{ }; use crate::durable_host::replay_state::{OplogEntryLookupResult, ReplayState}; use crate::durable_host::{ - AtomicRegionLease, DurableScopeKind, DurableWorkerCtx, IFSWorkerFile, PublicDurableWorkerState, + AtomicRegionLease, DurableScopeKind, DurableWorkerCtx, PublicDurableWorkerState, }; use crate::services::HasWorker; use crate::services::component::ComponentService; diff --git a/golem-worker-executor/src/durable_host/filesystem/types.rs b/golem-worker-executor/src/durable_host/filesystem/types.rs index 4c579a7968..afdb9c0ea2 100644 --- a/golem-worker-executor/src/durable_host/filesystem/types.rs +++ b/golem-worker-executor/src/durable_host/filesystem/types.rs @@ -13,11 +13,16 @@ // limitations under the License. use std::hash::Hasher; +use std::sync::Arc; use std::time::SystemTime; +use std::time::{Duration, Instant}; +use bytes::Bytes; +use cap_std::fs::FileExt; use fs_set_times::{SystemTimeSpec, set_symlink_times}; use metrohash::MetroHash128; use wasmtime::component::Resource; +use wasmtime_wasi::FilePerms; use wasmtime_wasi::filesystem::WasiFilesystemView as _; use wasmtime_wasi::p2::FsError; use wasmtime_wasi::p2::ReaddirIterator; @@ -31,6 +36,32 @@ use wasmtime_wasi::runtime::spawn_blocking; use crate::durable_host::concurrent::{CallHandle, NotCancellable}; use crate::durable_host::{DurabilityHost, DurableWorkerCtx, FilesystemOutputStreamState}; +use crate::services::agent_filesystem::{ + AgentFilesystemRuntime, ClassifiedFileOutputStream, FILESYSTEM_MUTATION_MAX_ATTEMPTS, + FILESYSTEM_MUTATION_RETRY_TIMEOUT, FilesystemStreamMode, MutationDecision, MutationEffect, + MutationFailure, MutationOperation, MutationPostcondition as P2MutationPostcondition, + NativeMutationGuestError, NativeOpenOptions, NativeOpenResult, + PathObjectType as P2PathObjectType, RequestedTime, create_directory as native_create_directory, + create_directory_postcondition as p2_create_directory_postcondition, + descriptor_state as p2_descriptor_state, descriptor_times as p2_descriptor_times, + hard_link as native_hard_link, link_postcondition as p2_link_postcondition, + native_write_failure_effect, open as native_open, open_postcondition as p2_open_postcondition, + path_state as p2_path_state, path_state_with_follow as p2_path_state_with_follow, + path_times as p2_path_times, remove_directory as native_remove_directory, + remove_postcondition as p2_remove_postcondition, rename as native_rename, + rename_postcondition as p2_rename_postcondition, resize_file as native_resize_file, + resize_postcondition as p2_resize_postcondition, run_blocking_filesystem_mutation, + set_descriptor_times as native_set_descriptor_times, set_path_times as native_set_path_times, + symlink as native_symlink, symlink_postcondition as p2_symlink_postcondition, + symlink_state as p2_symlink_state, sync_descriptor as native_sync_descriptor, + times_postcondition, unlink_file as native_unlink_file, validate_descriptor_times, + validate_directory_mutation, validate_open, validate_resize, validate_two_directory_mutation, +}; +#[cfg(test)] +use crate::services::agent_filesystem::{ + ObjectIdentity as P2ObjectIdentity, PathState as P2PathState, same_object as p2_same_object, + same_optional_object as p2_same_optional_object, +}; use crate::workerctx::WorkerCtx; use golem_common::model::oplog::host_functions::{ FilesystemTypesDescriptorStat, FilesystemTypesDescriptorStatAt, @@ -42,6 +73,402 @@ use golem_common::model::oplog::{ DurableFunctionType, HostRequestFileSystemPath, HostResponseFileSystemStat, }; +enum P2MutationAction { + Retry, + Success, + Error(FsError), + Trap, +} + +struct P2MutationAdapter { + runtime: AgentFilesystemRuntime, + operation: MutationOperation, + started: Instant, + attempts: usize, +} + +impl P2MutationAdapter { + fn new(runtime: AgentFilesystemRuntime) -> Self { + Self::for_operation(runtime, MutationOperation::Metadata) + } + + fn for_operation(runtime: AgentFilesystemRuntime, operation: MutationOperation) -> Self { + Self { + runtime, + operation, + started: Instant::now(), + attempts: 0, + } + } + + fn begin_attempt(&mut self) { + self.attempts += 1; + } + + #[cfg(test)] + async fn failure( + &self, + error: FsError, + postcondition: P2MutationPostcondition, + retry_safe: bool, + ) -> P2MutationAction { + let Some(guest) = error.downcast_ref().copied() else { + let _ = self + .runtime + .classify_mutation_failure::( + MutationFailure::Infrastructure(std::io::Error::other(error.to_string())), + MutationEffect::Unknown, + ) + .await; + return P2MutationAction::Trap; + }; + let effect = match (postcondition, guest) { + (P2MutationPostcondition::Satisfied, _) => { + MutationEffect::DesiredPostconditionSatisfied + } + (P2MutationPostcondition::NoEffect, _) => MutationEffect::ProvenNoEffect, + (P2MutationPostcondition::Unknown, _) => MutationEffect::Unknown, + }; + let failure = match guest { + ErrorCode::Quota => MutationFailure::StorageExhaustion { + guest, + quota_hint: true, + }, + ErrorCode::InsufficientSpace => MutationFailure::StorageExhaustion { + guest, + quota_hint: false, + }, + ErrorCode::Busy + | ErrorCode::Interrupted + | ErrorCode::InProgress + | ErrorCode::Already => MutationFailure::TransientGuest(guest), + ErrorCode::Access | ErrorCode::NotPermitted => MutationFailure::AccessGuest(guest), + ErrorCode::Io => MutationFailure::UnclassifiedGuest(guest), + _ => MutationFailure::Guest(guest), + }; + match self + .runtime + .classify_mutation_failure_for(self.operation, failure, effect) + .await + { + MutationDecision::PreserveGuest(error) => P2MutationAction::Error(error.into()), + MutationDecision::Quota => P2MutationAction::Error(ErrorCode::Quota.into()), + MutationDecision::InsufficientSpace => { + P2MutationAction::Error(ErrorCode::InsufficientSpace.into()) + } + MutationDecision::PhysicalPressure + if retry_safe + && postcondition == P2MutationPostcondition::NoEffect + && self.attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => + { + if self + .runtime + .recover_physical_pressure( + self.operation, + self.started + FILESYSTEM_MUTATION_RETRY_TIMEOUT, + ) + .await + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT + { + P2MutationAction::Retry + } else { + P2MutationAction::Error(ErrorCode::InsufficientSpace.into()) + } + } + MutationDecision::PhysicalPressure => { + P2MutationAction::Error(ErrorCode::InsufficientSpace.into()) + } + MutationDecision::BoundedRetry + if retry_safe + && postcondition == P2MutationPostcondition::NoEffect + && self.attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => + { + P2MutationAction::Retry + } + MutationDecision::BoundedRetry => P2MutationAction::Error(guest.into()), + MutationDecision::PreserveRaw => P2MutationAction::Error(guest.into()), + MutationDecision::Success => P2MutationAction::Success, + MutationDecision::Invalidate => P2MutationAction::Trap, + } + } + + async fn io_failure( + &self, + error: std::io::Error, + postcondition: P2MutationPostcondition, + retry_safe: bool, + ) -> P2MutationAction { + let raw_os_error = error.raw_os_error(); + let error_kind = error.kind(); + let error_message = error.to_string(); + let effect = match postcondition { + P2MutationPostcondition::Satisfied => MutationEffect::DesiredPostconditionSatisfied, + P2MutationPostcondition::NoEffect => MutationEffect::ProvenNoEffect, + P2MutationPostcondition::Unknown => MutationEffect::Unknown, + }; + match self + .runtime + .classify_mutation_failure_for::( + self.operation, + MutationFailure::Io(error), + effect, + ) + .await + { + MutationDecision::PreserveGuest(error) => P2MutationAction::Error(error.into()), + MutationDecision::Quota => P2MutationAction::Error(ErrorCode::Quota.into()), + MutationDecision::InsufficientSpace => { + P2MutationAction::Error(ErrorCode::InsufficientSpace.into()) + } + MutationDecision::PhysicalPressure + if retry_safe + && postcondition == P2MutationPostcondition::NoEffect + && self.attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => + { + if self + .runtime + .recover_physical_pressure( + self.operation, + self.started + FILESYSTEM_MUTATION_RETRY_TIMEOUT, + ) + .await + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT + { + P2MutationAction::Retry + } else { + P2MutationAction::Error(ErrorCode::InsufficientSpace.into()) + } + } + MutationDecision::PhysicalPressure => { + P2MutationAction::Error(ErrorCode::InsufficientSpace.into()) + } + MutationDecision::BoundedRetry + if retry_safe + && postcondition == P2MutationPostcondition::NoEffect + && self.attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => + { + P2MutationAction::Retry + } + MutationDecision::BoundedRetry | MutationDecision::PreserveRaw => { + let error = raw_os_error.map_or_else( + || std::io::Error::new(error_kind, error_message), + std::io::Error::from_raw_os_error, + ); + P2MutationAction::Error(error.into()) + } + MutationDecision::Success => P2MutationAction::Success, + MutationDecision::Invalidate => P2MutationAction::Trap, + } + } +} + +fn p2_mutation_action_result(action: P2MutationAction) -> Result<(), FsError> { + match action { + P2MutationAction::Success => Ok(()), + P2MutationAction::Error(error) => Err(error), + P2MutationAction::Trap => Err(FsError::trap(wasmtime::Error::msg( + "agent filesystem mutation invalidated the runtime", + ))), + P2MutationAction::Retry => unreachable!("retry must be handled by the operation loop"), + } +} + +async fn p2_initial_probe( + runtime: &AgentFilesystemRuntime, + result: Result, +) -> Result { + match result { + Ok(value) => Ok(value), + Err(error) => { + let raw_os_error = error.raw_os_error(); + let error_kind = error.kind(); + let message = error.to_string(); + match runtime + .classify_mutation_failure_for::( + MutationOperation::Metadata, + MutationFailure::Io(error), + MutationEffect::ProvenNoEffect, + ) + .await + { + MutationDecision::Quota => Err(ErrorCode::Quota.into()), + MutationDecision::InsufficientSpace | MutationDecision::PhysicalPressure => { + Err(ErrorCode::InsufficientSpace.into()) + } + MutationDecision::BoundedRetry | MutationDecision::PreserveRaw => { + let error = raw_os_error.map_or_else( + || std::io::Error::new(error_kind, message), + std::io::Error::from_raw_os_error, + ); + Err(error.into()) + } + MutationDecision::PreserveGuest(error) => Err(error.into()), + MutationDecision::Success => unreachable!("failed probe cannot be satisfied"), + MutationDecision::Invalidate => Err(FsError::trap(wasmtime::Error::msg( + "agent filesystem mutation precondition probe invalidated the runtime", + ))), + } + } + } +} + +fn p2_directory(descriptor: &Descriptor) -> Result { + match descriptor { + Descriptor::Dir(directory) => Ok(directory.clone()), + Descriptor::File(_) => Err(ErrorCode::NotDirectory.into()), + } +} + +async fn p2_finish_native_mutation( + adapter: &P2MutationAdapter, + error: std::io::Error, + postcondition: P2MutationPostcondition, + retry_safe: bool, +) -> Result { + match adapter.io_failure(error, postcondition, retry_safe).await { + P2MutationAction::Retry => Ok(true), + action => p2_mutation_action_result(action).map(|()| false), + } +} + +fn p2_native_guest(error: NativeMutationGuestError) -> FsError { + match error { + NativeMutationGuestError::Invalid => ErrorCode::Invalid.into(), + NativeMutationGuestError::NotDirectory => ErrorCode::NotDirectory.into(), + NativeMutationGuestError::NotPermitted => ErrorCode::NotPermitted.into(), + NativeMutationGuestError::Unsupported => ErrorCode::Unsupported.into(), + } +} + +fn p2_requested_time(requested: NewTimestamp) -> RequestedTime { + match requested { + NewTimestamp::NoChange => RequestedTime::NoChange, + NewTimestamp::Now => RequestedTime::Now, + NewTimestamp::Timestamp(timestamp) => RequestedTime::Timestamp { + seconds: i128::from(timestamp.seconds), + nanoseconds: timestamp.nanoseconds, + }, + } +} + +fn p2_native_time(requested: NewTimestamp) -> Result, FsError> { + match requested { + NewTimestamp::NoChange => Ok(None), + NewTimestamp::Now => Ok(Some(SystemTime::now())), + NewTimestamp::Timestamp(timestamp) => SystemTime::UNIX_EPOCH + .checked_add(Duration::new(timestamp.seconds, timestamp.nanoseconds)) + .map(Some) + .ok_or_else(|| ErrorCode::Overflow.into()), + } +} + +fn p2_times_postcondition( + current: Result, + before: crate::services::agent_filesystem::TimesState, + accessed: NewTimestamp, + modified: NewTimestamp, + identity_required: bool, +) -> P2MutationPostcondition { + times_postcondition( + current, + before, + p2_requested_time(accessed), + p2_requested_time(modified), + identity_required, + ) +} + +async fn classified_positioned_write( + filesystem_runtime: crate::services::agent_filesystem::AgentFilesystemRuntime, + file: wasmtime_wasi::filesystem::File, + buffer: Vec, + offset: Filesize, + effect: crate::services::agent_filesystem::AgentFilesystemEffectLease, +) -> Result { + let file = Arc::clone(&file.file); + let buffer = Bytes::from(buffer); + let effect = Arc::new(effect); + let started = Instant::now(); + + for attempt in 0..FILESYSTEM_MUTATION_MAX_ATTEMPTS { + let file = Arc::clone(&file); + let buffer = buffer.clone(); + let effect = Arc::clone(&effect); + match spawn_blocking(move || { + let _effect = effect; + file.write_at(&buffer, offset) + }) + .await + { + Ok(written) => { + return Ok(Filesize::try_from(written).expect("usize fits in Filesize")); + } + Err(error) => { + let raw_os_error = error.raw_os_error(); + let error_kind = error.kind(); + let error_message = error.to_string(); + let effect = native_write_failure_effect(&error, 0); + let decision = filesystem_runtime + .classify_mutation_failure_for::( + MutationOperation::Write, + MutationFailure::Io(error), + effect, + ) + .await; + match decision { + MutationDecision::BoundedRetry + if attempt + 1 < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => {} + MutationDecision::BoundedRetry => { + let error = raw_os_error.map_or_else( + || std::io::Error::new(error_kind, error_message), + std::io::Error::from_raw_os_error, + ); + return Err(error.into()); + } + MutationDecision::PreserveRaw => { + let error = raw_os_error.map_or_else( + || std::io::Error::new(error_kind, error_message), + std::io::Error::from_raw_os_error, + ); + return Err(error.into()); + } + MutationDecision::PreserveGuest(error) => return Err(error.into()), + MutationDecision::Quota => return Err(ErrorCode::Quota.into()), + MutationDecision::InsufficientSpace => { + return Err(ErrorCode::InsufficientSpace.into()); + } + MutationDecision::PhysicalPressure + if attempt + 1 < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT + && filesystem_runtime + .recover_physical_pressure( + MutationOperation::Write, + started + FILESYSTEM_MUTATION_RETRY_TIMEOUT, + ) + .await + && started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => {} + MutationDecision::PhysicalPressure => { + return Err(ErrorCode::InsufficientSpace.into()); + } + MutationDecision::Success => return Ok(0), + MutationDecision::Invalidate => { + return Err(FsError::trap(wasmtime::Error::msg( + "agent filesystem mutation invalidated the runtime", + ))); + } + } + } + } + } + + unreachable!("positioned write loop always returns") +} + impl HostDescriptor for DurableWorkerCtx { fn read_via_stream( &mut self, @@ -64,15 +491,24 @@ impl HostDescriptor for DurableWorkerCtx { ) -> Result, FsError> { self.fail_if_read_only(&fd)?; self.observe_function_call("filesystem::types::descriptor", "write_via_stream"); - let descriptor_rep = fd.rep(); - let stream = - HostDescriptor::write_via_stream(&mut self.as_wasi_view().filesystem(), fd, offset)?; + let file = match self.table().get(&fd)? { + Descriptor::File(file) if file.perms.contains(FilePerms::WRITE) => file.clone(), + Descriptor::File(_) => return Err(ErrorCode::NotPermitted.into()), + Descriptor::Dir(_) => return Err(ErrorCode::BadDescriptor.into()), + }; + let filesystem_runtime = self.filesystem_runtime(); + let stream = self.table().push( + ClassifiedFileOutputStream::new( + file, + filesystem_runtime, + FilesystemStreamMode::Position(offset), + ) + .into_dyn(), + )?; self.state.open_filesystem_output_streams.insert( stream.rep(), FilesystemOutputStreamState { - descriptor_rep, position: Some(offset), - pending_reservation: None, }, ); Ok(stream) @@ -80,21 +516,23 @@ impl HostDescriptor for DurableWorkerCtx { fn append_via_stream( &mut self, - self_: Resource, + fd: Resource, ) -> Result, FsError> { - self.fail_if_read_only(&self_)?; + self.fail_if_read_only(&fd)?; self.observe_function_call("filesystem::types::descriptor", "append_via_stream"); - let descriptor_rep = self_.rep(); - let stream = - HostDescriptor::append_via_stream(&mut self.as_wasi_view().filesystem(), self_)?; - self.state.open_filesystem_output_streams.insert( - stream.rep(), - FilesystemOutputStreamState { - descriptor_rep, - position: None, - pending_reservation: None, - }, - ); + let file = match self.table().get(&fd)? { + Descriptor::File(file) if file.perms.contains(FilePerms::WRITE) => file.clone(), + Descriptor::File(_) => return Err(ErrorCode::NotPermitted.into()), + Descriptor::Dir(_) => return Err(ErrorCode::BadDescriptor.into()), + }; + let filesystem_runtime = self.filesystem_runtime(); + let stream = self.table().push( + ClassifiedFileOutputStream::new(file, filesystem_runtime, FilesystemStreamMode::Append) + .into_dyn(), + )?; + self.state + .open_filesystem_output_streams + .insert(stream.rep(), FilesystemOutputStreamState { position: None }); Ok(stream) } @@ -112,8 +550,27 @@ impl HostDescriptor for DurableWorkerCtx { async fn sync_data(&mut self, self_: Resource) -> Result<(), FsError> { self.observe_function_call("filesystem::types::descriptor", "sync_data"); - let mut view = self.as_wasi_view(); - HostDescriptor::sync_data(&mut view.filesystem(), self_).await + let effect = Arc::new( + self.filesystem_runtime() + .begin_effect() + .await + .map_err(FsError::trap)?, + ); + let descriptor = self.table().get(&self_)?.clone(); + let mut adapter = P2MutationAdapter::new(self.filesystem_runtime()); + adapter.begin_attempt(); + match run_blocking_filesystem_mutation(effect, move || { + native_sync_descriptor(&descriptor, true) + }) + .await + { + Ok(()) => Ok(()), + Err(error) => p2_mutation_action_result( + adapter + .io_failure(error, P2MutationPostcondition::Unknown, false) + .await, + ), + } } async fn get_flags(&mut self, fd: Resource) -> Result { @@ -137,47 +594,46 @@ impl HostDescriptor for DurableWorkerCtx { } async fn set_size(&mut self, fd: Resource, size: Filesize) -> Result<(), FsError> { + let effect = Arc::new( + self.filesystem_runtime() + .begin_update_effect() + .await + .map_err(FsError::trap)?, + ); self.fail_if_read_only(&fd)?; - // Determine whether this is a growth and charge the delta. - // We borrow fd to stat before consuming it in set_size. - let current_size = { - let fd_borrow = Resource::new_borrow(fd.rep()); - let mut view = self.as_wasi_view(); - match HostDescriptor::stat(&mut view.filesystem(), fd_borrow).await { - Ok(s) => s.size, - Err(_) => 0, // if we can't stat, treat current as 0 (conservative: charges full size) - } + let descriptor = self.table().get(&fd)?.clone(); + let file = match &descriptor { + Descriptor::File(file) => file.clone(), + Descriptor::Dir(_) => return Err(ErrorCode::BadDescriptor.into()), }; - - if size > current_size { - let delta = size - current_size; - self.reserve_filesystem_storage(delta) - .await - .map_err(|e| FsError::trap(wasmtime::Error::from_anyhow(e)))?; - } - // size == current_size: no-op + validate_resize(&file).map_err(p2_native_guest)?; + let runtime = self.filesystem_runtime(); + let before = p2_initial_probe(&runtime, p2_descriptor_state(&descriptor).await).await?; self.observe_function_call("filesystem::types::descriptor", "set_size"); - - let result = { - let mut view = self.as_wasi_view(); - HostDescriptor::set_size(&mut view.filesystem(), fd, size).await - }; - - if size > current_size { - // Growth path: release permits if the operation failed. - let delta = size - current_size; - if result.is_err() { - self.release_filesystem_storage_space(delta).await; + let mut adapter = P2MutationAdapter::for_operation(runtime, MutationOperation::Resize); + loop { + adapter.begin_attempt(); + let file = file.clone(); + let effect = Arc::clone(&effect); + let result = + run_blocking_filesystem_mutation(effect, move || native_resize_file(&file, size)) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = p2_resize_postcondition( + before, + p2_descriptor_state(&descriptor).await, + size, + ); + if !p2_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } } - } else if result.is_ok() && size < current_size { - // Shrink path: release permits for the freed space on success. - self.release_filesystem_storage_space(current_size - size) - .await; } - - result } async fn set_times( @@ -186,18 +642,46 @@ impl HostDescriptor for DurableWorkerCtx { data_access_timestamp: NewTimestamp, data_modification_timestamp: NewTimestamp, ) -> Result<(), FsError> { + let effect = Arc::new( + self.filesystem_runtime() + .begin_update_effect() + .await + .map_err(FsError::trap)?, + ); self.fail_if_read_only(&fd)?; self.observe_function_call("filesystem::types::descriptor", "set_times"); - - let mut view = self.as_wasi_view(); - HostDescriptor::set_times( - &mut view.filesystem(), - fd, - data_access_timestamp, - data_modification_timestamp, - ) - .await + let descriptor = self.table().get(&fd)?.clone(); + validate_descriptor_times(&descriptor).map_err(p2_native_guest)?; + let accessed = p2_native_time(data_access_timestamp)?; + let modified = p2_native_time(data_modification_timestamp)?; + let runtime = self.filesystem_runtime(); + let before = p2_initial_probe(&runtime, p2_descriptor_times(&descriptor).await).await?; + let mut adapter = P2MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let descriptor_for_attempt = descriptor.clone(); + let effect = Arc::clone(&effect); + let result = run_blocking_filesystem_mutation(effect, move || { + native_set_descriptor_times(&descriptor_for_attempt, accessed, modified) + }) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = p2_times_postcondition( + p2_descriptor_times(&descriptor).await, + before, + data_access_timestamp, + data_modification_timestamp, + false, + ); + if !p2_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn read( @@ -218,50 +702,19 @@ impl HostDescriptor for DurableWorkerCtx { offset: Filesize, ) -> Result { self.fail_if_read_only(&fd)?; - - let current_size = { - let fd_borrow = Resource::new_borrow(fd.rep()); - let mut view = self.as_wasi_view(); - match HostDescriptor::stat(&mut view.filesystem(), fd_borrow).await { - Ok(s) => s.size, - Err(_) => 0, - } + let file = match self.table().get(&fd)? { + Descriptor::File(file) if file.perms.contains(FilePerms::WRITE) => file.clone(), + Descriptor::File(_) => return Err(ErrorCode::NotPermitted.into()), + Descriptor::Dir(_) => return Err(ErrorCode::BadDescriptor.into()), }; - - let requested_end = offset.saturating_add(buffer.len() as u64); - let requested_growth = requested_end.saturating_sub(current_size); - if requested_growth > 0 { - self.reserve_filesystem_storage(requested_growth) - .await - .map_err(|e| FsError::trap(wasmtime::Error::from_anyhow(e)))?; - } + let effect = self + .filesystem_runtime() + .begin_effect() + .await + .map_err(FsError::trap)?; self.observe_function_call("filesystem::types::descriptor", "write"); - let result = { - let mut view = self.as_wasi_view(); - HostDescriptor::write(&mut view.filesystem(), fd, buffer, offset).await - }; - - if requested_growth > 0 { - match result { - Ok(written) => { - let actual_end = offset.saturating_add(written); - let actual_growth = actual_end.saturating_sub(current_size); - let over_reserved = requested_growth.saturating_sub(actual_growth); - if over_reserved > 0 { - self.release_filesystem_storage_space(over_reserved).await; - } - Ok(written) - } - Err(err) => { - self.release_filesystem_storage_space(requested_growth) - .await; - Err(err) - } - } - } else { - result - } + classified_positioned_write(self.filesystem_runtime(), file, buffer, offset, effect).await } async fn read_directory( @@ -286,8 +739,27 @@ impl HostDescriptor for DurableWorkerCtx { async fn sync(&mut self, self_: Resource) -> Result<(), FsError> { self.observe_function_call("filesystem::types::descriptor", "sync"); - let mut view = self.as_wasi_view(); - HostDescriptor::sync(&mut view.filesystem(), self_).await + let effect = Arc::new( + self.filesystem_runtime() + .begin_effect() + .await + .map_err(FsError::trap)?, + ); + let descriptor = self.table().get(&self_)?.clone(); + let mut adapter = P2MutationAdapter::new(self.filesystem_runtime()); + adapter.begin_attempt(); + match run_blocking_filesystem_mutation(effect, move || { + native_sync_descriptor(&descriptor, false) + }) + .await + { + Ok(()) => Ok(()), + Err(error) => p2_mutation_action_result( + adapter + .io_failure(error, P2MutationPostcondition::Unknown, false) + .await, + ), + } } async fn create_directory_at( @@ -296,8 +768,40 @@ impl HostDescriptor for DurableWorkerCtx { path: String, ) -> Result<(), FsError> { self.observe_function_call("filesystem::types::descriptor", "create_directory_at"); - let mut view = self.as_wasi_view(); - HostDescriptor::create_directory_at(&mut view.filesystem(), self_, path).await + let effect = Arc::new( + self.filesystem_runtime() + .begin_path_effect() + .await + .map_err(FsError::trap)?, + ); + self.fail_if_read_only_path(&self_, &path, false)?; + let directory = p2_directory(self.table().get(&self_)?)?; + validate_directory_mutation(&directory).map_err(p2_native_guest)?; + let runtime = self.filesystem_runtime(); + let before = p2_initial_probe(&runtime, p2_path_state(&directory, &path).await).await?; + let mut adapter = P2MutationAdapter::for_operation(runtime, MutationOperation::Create); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(&effect); + let result = run_blocking_filesystem_mutation(effect, move || { + native_create_directory(&directory_for_attempt, &path_for_attempt) + }) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = p2_create_directory_postcondition( + before, + p2_path_state(&directory, &path).await, + ); + if !p2_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn stat(&mut self, self_: Resource) -> Result { @@ -305,6 +809,11 @@ impl HostDescriptor for DurableWorkerCtx { Descriptor::File(f) => f.path.clone(), Descriptor::Dir(d) => d.path.clone(), }; + let _effect = self + .filesystem_runtime() + .begin_effect() + .await + .map_err(FsError::trap)?; // `ReadLocal`: the local stat always runs (its timestamps are then overridden by the durable // value), so only the file-times are made durable via `CallHandle::run`. @@ -386,6 +895,11 @@ impl HostDescriptor for DurableWorkerCtx { Descriptor::File(f) => f.path.join(path.clone()), Descriptor::Dir(d) => d.path.join(path.clone()), }; + let _effect = self + .filesystem_runtime() + .begin_effect() + .await + .map_err(FsError::trap)?; // `ReadLocal`: the local stat always runs (its timestamps are then overridden by the durable // value), so only the file-times are made durable via `CallHandle::run`. @@ -465,19 +979,56 @@ impl HostDescriptor for DurableWorkerCtx { data_access_timestamp: NewTimestamp, data_modification_timestamp: NewTimestamp, ) -> Result<(), FsError> { + let effect = Arc::new( + self.filesystem_runtime() + .begin_update_effect() + .await + .map_err(FsError::trap)?, + ); self.fail_if_read_only(&fd)?; + self.fail_if_read_only_path(&fd, &path, path_flags.contains(PathFlags::SYMLINK_FOLLOW))?; self.observe_function_call("filesystem::types::descriptor", "set_times_at"); - let mut view = self.as_wasi_view(); - HostDescriptor::set_times_at( - &mut view.filesystem(), - fd, - path_flags, - path, - data_access_timestamp, - data_modification_timestamp, - ) - .await + let directory = p2_directory(self.table().get(&fd)?)?; + validate_directory_mutation(&directory).map_err(p2_native_guest)?; + let follow = path_flags.contains(PathFlags::SYMLINK_FOLLOW); + let accessed = p2_native_time(data_access_timestamp)?; + let modified = p2_native_time(data_modification_timestamp)?; + let runtime = self.filesystem_runtime(); + let before = + p2_initial_probe(&runtime, p2_path_times(&directory, &path, follow).await).await?; + let mut adapter = P2MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(&effect); + let result = run_blocking_filesystem_mutation(effect, move || { + native_set_path_times( + &directory_for_attempt, + &path_for_attempt, + follow, + accessed, + modified, + ) + }) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = p2_times_postcondition( + p2_path_times(&directory, &path, follow).await, + before, + data_access_timestamp, + data_modification_timestamp, + true, + ); + if !p2_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn link_at( @@ -489,16 +1040,67 @@ impl HostDescriptor for DurableWorkerCtx { new_path: String, ) -> Result<(), FsError> { self.observe_function_call("filesystem::types::descriptor", "link_at"); - let mut view = self.as_wasi_view(); - HostDescriptor::link_at( - &mut view.filesystem(), - self_, - old_path_flags, - old_path, - new_descriptor, - new_path.clone(), + let effect = Arc::new( + self.filesystem_runtime() + .begin_path_effect() + .await + .map_err(FsError::trap)?, + ); + self.fail_if_read_only(&self_)?; + self.fail_if_read_only(&new_descriptor)?; + self.fail_if_read_only_path( + &self_, + &old_path, + old_path_flags.contains(PathFlags::SYMLINK_FOLLOW), + )?; + self.fail_if_read_only_path(&new_descriptor, &new_path, false)?; + let source_directory = p2_directory(self.table().get(&self_)?)?; + let destination_directory = p2_directory(self.table().get(&new_descriptor)?)?; + validate_two_directory_mutation(&source_directory, &destination_directory) + .map_err(p2_native_guest)?; + if old_path_flags.contains(PathFlags::SYMLINK_FOLLOW) { + return Err(ErrorCode::Invalid.into()); + } + let runtime = self.filesystem_runtime(); + let source_before = + p2_initial_probe(&runtime, p2_path_state(&source_directory, &old_path).await).await?; + let destination_before = p2_initial_probe( + &runtime, + p2_path_state(&destination_directory, &new_path).await, ) - .await + .await?; + let mut adapter = P2MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let source_directory_for_attempt = source_directory.clone(); + let destination_directory_for_attempt = destination_directory.clone(); + let old_path_for_attempt = old_path.clone(); + let new_path_for_attempt = new_path.clone(); + let effect = Arc::clone(&effect); + let result = run_blocking_filesystem_mutation(effect, move || { + native_hard_link( + &source_directory_for_attempt, + &old_path_for_attempt, + &destination_directory_for_attempt, + &new_path_for_attempt, + ) + }) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = p2_link_postcondition( + source_before, + destination_before, + p2_path_state(&source_directory, &old_path).await, + p2_path_state(&destination_directory, &new_path).await, + ); + if !p2_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn open_at( @@ -509,28 +1111,28 @@ impl HostDescriptor for DurableWorkerCtx { open_flags: OpenFlags, flags: DescriptorFlags, ) -> Result, FsError> { - let truncated_size = if open_flags.contains(OpenFlags::TRUNCATE) { - let fd_borrow = Resource::new_borrow(self_.rep()); - let mut view = self.as_wasi_view(); - match HostDescriptor::stat_at( - &mut view.filesystem(), - fd_borrow, - path_flags, - path.clone(), - ) - .await - { - Ok(s) => s.size, - Err(_) => 0, - } + let mutating = open_flags.intersects(OpenFlags::CREATE | OpenFlags::TRUNCATE); + let effect = if mutating { + Some(Arc::new( + self.filesystem_runtime() + .begin_update_effect() + .await + .map_err(FsError::trap)?, + )) } else { - 0 + None }; - + if open_flags.contains(OpenFlags::TRUNCATE) || flags.contains(DescriptorFlags::WRITE) { + self.fail_if_read_only_path( + &self_, + &path, + path_flags.contains(PathFlags::SYMLINK_FOLLOW), + )?; + } self.observe_function_call("filesystem::types::descriptor", "open_at"); - let result = { + if !mutating { let mut view = self.as_wasi_view(); - HostDescriptor::open_at( + return HostDescriptor::open_at( &mut view.filesystem(), self_, path_flags, @@ -538,14 +1140,100 @@ impl HostDescriptor for DurableWorkerCtx { open_flags, flags, ) - .await - }; - - if result.is_ok() && truncated_size > 0 { - self.release_filesystem_storage_space(truncated_size).await; + .await; } - result + let directory = p2_directory(self.table().get(&self_)?)?; + let follow = path_flags.contains(PathFlags::SYMLINK_FOLLOW); + let native_options = NativeOpenOptions { + create: open_flags.contains(OpenFlags::CREATE), + directory: open_flags.contains(OpenFlags::DIRECTORY), + exclusive: open_flags.contains(OpenFlags::EXCLUSIVE), + truncate: open_flags.contains(OpenFlags::TRUNCATE), + follow, + read: flags.contains(DescriptorFlags::READ), + write: flags.contains(DescriptorFlags::WRITE), + }; + validate_open( + &directory, + native_options, + flags.intersects( + DescriptorFlags::FILE_INTEGRITY_SYNC + | DescriptorFlags::DATA_INTEGRITY_SYNC + | DescriptorFlags::REQUESTED_WRITE_SYNC, + ), + ) + .map_err(p2_native_guest)?; + let runtime = self.filesystem_runtime(); + let before = p2_initial_probe( + &runtime, + p2_path_state_with_follow(&directory, &path, follow).await, + ) + .await?; + let requested_type = if open_flags.contains(OpenFlags::DIRECTORY) { + P2PathObjectType::Directory + } else { + P2PathObjectType::RegularFile + }; + let operation = if open_flags.contains(OpenFlags::CREATE) { + MutationOperation::Create + } else { + MutationOperation::Resize + }; + let mut adapter = P2MutationAdapter::for_operation(runtime, operation); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(effect.as_ref().expect("mutating open has an effect lease")); + let result = run_blocking_filesystem_mutation(effect, move || { + native_open(&directory_for_attempt, &path_for_attempt, native_options) + }) + .await; + match result { + Ok(NativeOpenResult::Descriptor(descriptor)) => { + return Ok(self.table().push(descriptor)?); + } + #[cfg(windows)] + Ok(NativeOpenResult::IsDirectory) => { + return Err(ErrorCode::IsDirectory.into()); + } + Ok(NativeOpenResult::NotDirectory) => { + return Err(p2_native_guest(NativeMutationGuestError::NotDirectory)); + } + Err(error) => { + let postcondition = p2_open_postcondition( + before, + p2_path_state_with_follow(&directory, &path, follow).await, + requested_type, + open_flags.contains(OpenFlags::TRUNCATE), + open_flags.contains(OpenFlags::EXCLUSIVE), + ); + match adapter.io_failure(error, postcondition, true).await { + P2MutationAction::Retry => {} + P2MutationAction::Success => { + let safe_flags = open_flags & OpenFlags::DIRECTORY; + let mut view = self.as_wasi_view(); + return HostDescriptor::open_at( + &mut view.filesystem(), + Resource::new_borrow(self_.rep()), + path_flags, + path.clone(), + safe_flags, + flags, + ) + .await; + } + P2MutationAction::Error(error) => return Err(error), + P2MutationAction::Trap => { + return Err(FsError::trap(wasmtime::Error::msg( + "agent filesystem mutation invalidated the runtime", + ))); + } + } + } + } + } } async fn readlink_at( @@ -564,8 +1252,38 @@ impl HostDescriptor for DurableWorkerCtx { path: String, ) -> Result<(), FsError> { self.observe_function_call("filesystem::types::descriptor", "remove_directory_at"); - let mut view = self.as_wasi_view(); - HostDescriptor::remove_directory_at(&mut view.filesystem(), self_, path.clone()).await + let effect = Arc::new( + self.filesystem_runtime() + .begin_path_effect() + .await + .map_err(FsError::trap)?, + ); + self.fail_if_contains_read_only_path(&self_, &path, false)?; + let directory = p2_directory(self.table().get(&self_)?)?; + validate_directory_mutation(&directory).map_err(p2_native_guest)?; + let runtime = self.filesystem_runtime(); + let before = p2_initial_probe(&runtime, p2_path_state(&directory, &path).await).await?; + let mut adapter = P2MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(&effect); + let result = run_blocking_filesystem_mutation(effect, move || { + native_remove_directory(&directory_for_attempt, &path_for_attempt) + }) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = + p2_remove_postcondition(before, p2_path_state(&directory, &path).await); + if !p2_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn rename_at( @@ -575,19 +1293,62 @@ impl HostDescriptor for DurableWorkerCtx { new_fd: Resource, new_path: String, ) -> Result<(), FsError> { + let effect = Arc::new( + self.filesystem_runtime() + .begin_path_effect() + .await + .map_err(FsError::trap)?, + ); self.fail_if_read_only(&old_fd)?; self.fail_if_read_only(&new_fd)?; + self.fail_if_contains_read_only_path(&old_fd, &old_path, false)?; + self.fail_if_contains_read_only_path(&new_fd, &new_path, false)?; self.observe_function_call("filesystem::types::descriptor", "rename_at"); - let mut view = self.as_wasi_view(); - HostDescriptor::rename_at( - &mut view.filesystem(), - old_fd, - old_path.clone(), - new_fd, - new_path.clone(), + let source_directory = p2_directory(self.table().get(&old_fd)?)?; + let destination_directory = p2_directory(self.table().get(&new_fd)?)?; + validate_two_directory_mutation(&source_directory, &destination_directory) + .map_err(p2_native_guest)?; + let runtime = self.filesystem_runtime(); + let source_before = + p2_initial_probe(&runtime, p2_path_state(&source_directory, &old_path).await).await?; + let destination_before = p2_initial_probe( + &runtime, + p2_path_state(&destination_directory, &new_path).await, ) - .await + .await?; + let mut adapter = P2MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let source_directory_for_attempt = source_directory.clone(); + let destination_directory_for_attempt = destination_directory.clone(); + let old_path_for_attempt = old_path.clone(); + let new_path_for_attempt = new_path.clone(); + let effect = Arc::clone(&effect); + let result = run_blocking_filesystem_mutation(effect, move || { + native_rename( + &source_directory_for_attempt, + &old_path_for_attempt, + &destination_directory_for_attempt, + &new_path_for_attempt, + ) + }) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = p2_rename_postcondition( + source_before, + destination_before, + p2_path_state(&source_directory, &old_path).await, + p2_path_state(&destination_directory, &new_path).await, + ); + if !p2_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn symlink_at( @@ -596,11 +1357,50 @@ impl HostDescriptor for DurableWorkerCtx { old_path: String, new_path: String, ) -> Result<(), FsError> { + let effect = Arc::new( + self.filesystem_runtime() + .begin_path_effect() + .await + .map_err(FsError::trap)?, + ); self.fail_if_read_only(&fd)?; + self.fail_if_read_only_path(&fd, &new_path, false)?; self.observe_function_call("filesystem::types::descriptor", "symlink_at"); - let mut view = self.as_wasi_view(); - HostDescriptor::symlink_at(&mut view.filesystem(), fd, old_path, new_path.clone()).await + let directory = p2_directory(self.table().get(&fd)?)?; + validate_directory_mutation(&directory).map_err(p2_native_guest)?; + let runtime = self.filesystem_runtime(); + let before = + p2_initial_probe(&runtime, p2_symlink_state(&directory, &new_path).await).await?; + let mut adapter = P2MutationAdapter::for_operation(runtime, MutationOperation::Create); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let old_path_for_attempt = old_path.clone(); + let new_path_for_attempt = new_path.clone(); + let effect = Arc::clone(&effect); + let result = run_blocking_filesystem_mutation(effect, move || { + native_symlink( + &directory_for_attempt, + &old_path_for_attempt, + &new_path_for_attempt, + ) + }) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = p2_symlink_postcondition( + &before, + p2_symlink_state(&directory, &new_path).await, + &old_path, + ); + if !p2_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn unlink_file_at( @@ -608,38 +1408,41 @@ impl HostDescriptor for DurableWorkerCtx { fd: Resource, path: String, ) -> Result<(), FsError> { + let effect = Arc::new( + self.filesystem_runtime() + .begin_path_effect() + .await + .map_err(FsError::trap)?, + ); self.fail_if_read_only(&fd)?; - - // Stat the target file before unlinking to know how many bytes to release. - // Use the upstream (non-durable) stat_at to avoid oplog side effects. - let file_size = { - let fd_borrow = Resource::new_borrow(fd.rep()); - let mut view = self.as_wasi_view(); - match HostDescriptor::stat_at( - &mut view.filesystem(), - fd_borrow, - PathFlags::empty(), - path.clone(), - ) - .await - { - Ok(s) => s.size, - Err(_) => 0, - } - }; + self.fail_if_read_only_path(&fd, &path, false)?; self.observe_function_call("filesystem::types::descriptor", "unlink_file_at"); - let result = { - let mut view = self.as_wasi_view(); - HostDescriptor::unlink_file_at(&mut view.filesystem(), fd, path.clone()).await - }; - - // Only release permits if the unlink actually succeeded. - if result.is_ok() { - self.release_filesystem_storage_space(file_size).await; + let directory = p2_directory(self.table().get(&fd)?)?; + validate_directory_mutation(&directory).map_err(p2_native_guest)?; + let runtime = self.filesystem_runtime(); + let before = p2_initial_probe(&runtime, p2_path_state(&directory, &path).await).await?; + let mut adapter = P2MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(&effect); + let result = run_blocking_filesystem_mutation(effect, move || { + native_unlink_file(&directory_for_attempt, &path_for_attempt) + }) + .await; + match result { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = + p2_remove_postcondition(before, p2_path_state(&directory, &path).await); + if !p2_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } } - - result } async fn is_same_object( @@ -707,6 +1510,13 @@ impl Host for DurableWorkerCtx { &mut self, err: Resource, ) -> wasmtime::Result> { + if let Some(error) = + crate::services::agent_filesystem::classified_filesystem_stream_error_code( + self.table().get(&err)?, + ) + { + return Ok(Some(error)); + } Host::filesystem_error_code(&mut self.as_wasi_view().filesystem(), err) } @@ -738,3 +1548,247 @@ pub(crate) fn calculate_metadata_hash_parts(modified: Option<(u64, u32)>, size: hasher.finish128() } + +#[cfg(test)] +mod p2_mutation_tests { + use super::*; + use crate::services::agent_filesystem::{ + AgentFilesystemUsage, FilesystemCapacity, ResolvedAgentFilesystemLimits, + }; + use test_r::test; + + fn healthy_capacity() -> FilesystemCapacity { + FilesystemCapacity { + total_bytes: 100, + available_bytes: 50, + total_filesystem_objects: 100, + available_filesystem_objects: 50, + } + } + + fn path_state(identity: Option) -> P2PathState { + P2PathState { + identity, + type_: P2PathObjectType::RegularFile, + size: 17, + } + } + + #[test] + fn p2_object_comparison_requires_authoritative_identity() { + assert!(!p2_same_object(path_state(None), path_state(None))); + assert!(!p2_same_optional_object( + Some(path_state(None)), + Some(path_state(None)), + )); + assert!(p2_same_optional_object(None, None)); + + let identity = P2ObjectIdentity { + device: 3, + inode: 5, + }; + assert!(p2_same_object( + path_state(Some(identity)), + path_state(Some(identity)), + )); + } + + #[test] + async fn p2_adapter_retries_only_proven_no_effect_and_at_most_once() { + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); + let mut adapter = P2MutationAdapter::new(runtime); + + adapter.begin_attempt(); + assert!(matches!( + adapter + .failure( + ErrorCode::Busy.into(), + P2MutationPostcondition::NoEffect, + true + ) + .await, + P2MutationAction::Retry + )); + adapter.begin_attempt(); + assert!(matches!( + adapter + .failure(ErrorCode::Busy.into(), P2MutationPostcondition::NoEffect, true) + .await, + P2MutationAction::Error(error) + if error.downcast_ref() == Some(&ErrorCode::Busy) + )); + } + + #[test_r::test] + async fn p2_adapter_accepts_satisfied_postcondition_without_retry() { + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); + let mut adapter = P2MutationAdapter::new(runtime); + adapter.begin_attempt(); + + assert!(matches!( + adapter + .failure( + ErrorCode::Interrupted.into(), + P2MutationPostcondition::Satisfied, + true, + ) + .await, + P2MutationAction::Success + )); + + let mut adapter = P2MutationAdapter::new( + crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(), + ); + adapter.begin_attempt(); + assert!(matches!( + adapter + .failure( + ErrorCode::NoEntry.into(), + P2MutationPostcondition::Satisfied, + true, + ) + .await, + P2MutationAction::Success + )); + } + + #[test_r::test] + async fn p2_adapter_invalidates_changed_or_unknown_effect() { + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); + let mut adapter = P2MutationAdapter::new(runtime.clone()); + adapter.begin_attempt(); + + assert!(matches!( + adapter + .failure( + ErrorCode::Busy.into(), + P2MutationPostcondition::Unknown, + true + ) + .await, + P2MutationAction::Trap + )); + assert!(runtime.begin_effect().await.is_err()); + } + + #[cfg(target_os = "linux")] + #[test_r::test] + async fn p2_native_eio_reaches_terminal_classifier() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let mut adapter = P2MutationAdapter::new(runtime.clone()); + adapter.begin_attempt(); + + assert!(matches!( + adapter + .io_failure( + std::io::Error::from_raw_os_error(libc::EIO), + P2MutationPostcondition::NoEffect, + true, + ) + .await, + P2MutationAction::Trap + )); + assert!(runtime.begin_effect().await.is_err()); + } + + #[test_r::test] + async fn p2_adapter_distinguishes_quota_from_physical_exhaustion() { + let exhausted = FilesystemCapacity { + available_bytes: 0, + available_filesystem_objects: 0, + ..healthy_capacity() + }; + let quota_runtime = + crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test_with_observations( + Some(AgentFilesystemUsage { + allocated_bytes: 50, + filesystem_objects: 10, + }), + Some(ResolvedAgentFilesystemLimits { + allocated_bytes: 50, + filesystem_objects: 10, + filesystem_object_limit_policy_version: 2, + }), + exhausted, + ); + let mut quota = P2MutationAdapter::new(quota_runtime); + quota.begin_attempt(); + assert!(matches!( + quota + .failure( + ErrorCode::InsufficientSpace.into(), + P2MutationPostcondition::NoEffect, + true, + ) + .await, + P2MutationAction::Error(error) + if error.downcast_ref() == Some(&ErrorCode::Quota) + )); + + let physical_runtime = + crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test_with_observations( + None, None, exhausted, + ); + let recovery_attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + physical_runtime.set_pressure_recovery_callback(Some({ + let recovery_attempts = Arc::clone(&recovery_attempts); + Arc::new(move |operation, _deadline| { + let recovery_attempts = Arc::clone(&recovery_attempts); + Box::pin(async move { + assert_eq!(operation, MutationOperation::Metadata); + recovery_attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + true + }) + }) + })); + let mut physical = P2MutationAdapter::new(physical_runtime); + physical.begin_attempt(); + assert!(matches!( + physical + .failure( + ErrorCode::InsufficientSpace.into(), + P2MutationPostcondition::NoEffect, + true, + ) + .await, + P2MutationAction::Retry + )); + physical.begin_attempt(); + assert!(matches!( + physical + .failure( + ErrorCode::InsufficientSpace.into(), + P2MutationPostcondition::NoEffect, + true, + ) + .await, + P2MutationAction::Error(error) + if error.downcast_ref() == Some(&ErrorCode::InsufficientSpace) + )); + assert_eq!( + recovery_attempts.load(std::sync::atomic::Ordering::Acquire), + 1 + ); + } + + #[test_r::test] + async fn p2_adapter_preserves_access_error_when_backend_is_healthy() { + let runtime = + crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test_with_observations( + None, + None, + healthy_capacity(), + ); + let mut adapter = P2MutationAdapter::new(runtime.clone()); + adapter.begin_attempt(); + + assert!(matches!( + adapter + .failure(ErrorCode::Access.into(), P2MutationPostcondition::NoEffect, true) + .await, + P2MutationAction::Error(error) + if error.downcast_ref() == Some(&ErrorCode::Access) + )); + assert!(runtime.begin_effect().await.is_ok()); + } +} diff --git a/golem-worker-executor/src/durable_host/io/streams.rs b/golem-worker-executor/src/durable_host/io/streams.rs index ab7b073f74..e8b3fe4d15 100644 --- a/golem-worker-executor/src/durable_host/io/streams.rs +++ b/golem-worker-executor/src/durable_host/io/streams.rs @@ -21,7 +21,6 @@ use crate::durable_host::http::{continue_http_request, end_http_request}; use crate::durable_host::io::{ManagedStdErr, ManagedStdOut}; use crate::durable_host::{ DurabilityHost, DurableWorkerCtx, HttpOutputStreamState, HttpRequestCloseOwner, - PendingFilesystemReservation, }; use crate::model::event::InternalWorkerEvent; use crate::workerctx::WorkerCtx; @@ -41,10 +40,6 @@ use golem_common::model::oplog::{ HostResponseStreamWriteWithBytes, HostResponseStreamWriteZeroes, OplogIndex, }; use golem_service_base::error::worker_executor::WorkerExecutorError; -use wasmtime_wasi::filesystem::WasiFilesystemView as _; -use wasmtime_wasi::p2::bindings::filesystem::types::{ - Descriptor as FsDescriptor, HostDescriptor as FsHostDescriptor, -}; use wasmtime_wasi::p2::bindings::io::streams::{ Host, HostInputStream, HostOutputStream, InputStream, OutputStream, Pollable, }; @@ -489,7 +484,7 @@ impl HostOutputStream for DurableWorkerCtx { } else { self.observe_function_call("io::streams::output_stream", "check_write"); let stream_rep = self_.rep(); - let result = if self + if self .state .open_filesystem_output_streams .contains_key(&stream_rep) @@ -549,15 +544,7 @@ impl HostOutputStream for DurableWorkerCtx { } } else { HostOutputStream::check_write(self.table(), self_).await - }; - if let Ok(permit) = result.as_ref() { - if *permit > 0 { - reconcile_pending_filesystem_stream_reservation(self, stream_rep).await; - } - } else { - reconcile_pending_filesystem_stream_reservation(self, stream_rep).await; } - result } } @@ -631,22 +618,19 @@ impl HostOutputStream for DurableWorkerCtx { if let Some(event) = event { self.emit_log_event(event).await; Ok::<(), StreamError>(()) + } else if !self.state.open_filesystem_output_streams.contains_key(&rep) { + HostOutputStream::write(self.table(), self_, contents).await } else { let stream_rep = self_.rep(); - let write_len = contents.len() as u64; - reserve_filesystem_stream_growth(self, stream_rep, write_len).await?; + if filesystem_stream_effect_is_active(self, stream_rep)? { + return HostOutputStream::write(self.table(), self_, contents).await; + } + let effect = begin_filesystem_stream_effect(self, stream_rep).await?; + prepare_filesystem_stream_effect(self, stream_rep, effect)?; let result = HostOutputStream::write(self.table(), self_, contents).await; - match result { - Ok(()) => { - mark_filesystem_stream_write_enqueued(self, stream_rep, write_len); - Ok(()) - } - Err(err) => { - rollback_pending_filesystem_stream_reservation(self, stream_rep).await; - Err(err) - } - } + clear_unused_filesystem_stream_effect(self, stream_rep)?; + result } } } @@ -939,28 +923,25 @@ impl HostOutputStream for DurableWorkerCtx { } else { self.observe_function_call("io::streams::output_stream", "write_zeroes"); - // Exclude console streams from quota — only file-backed streams consume storage. + // Only file-backed streams participate in filesystem effect coordination. let is_console = { let output = self.table().get(&self_)?; output.as_any().downcast_ref::().is_some() || output.as_any().downcast_ref::().is_some() }; - if !is_console { + let is_file_stream = self.state.open_filesystem_output_streams.contains_key(&rep); + if !is_console && is_file_stream { let stream_rep = self_.rep(); - reserve_filesystem_stream_growth(self, stream_rep, len).await?; + if filesystem_stream_effect_is_active(self, stream_rep)? { + return HostOutputStream::write_zeroes(self.table(), self_, len).await; + } + let effect = begin_filesystem_stream_effect(self, stream_rep).await?; + prepare_filesystem_stream_effect(self, stream_rep, effect)?; let result = HostOutputStream::write_zeroes(self.table(), self_, len).await; - return match result { - Ok(()) => { - mark_filesystem_stream_write_enqueued(self, stream_rep, len); - Ok(()) - } - Err(err) => { - rollback_pending_filesystem_stream_reservation(self, stream_rep).await; - Err(err) - } - }; + clear_unused_filesystem_stream_effect(self, stream_rep)?; + return result; } HostOutputStream::write_zeroes(self.table(), self_, len).await @@ -1028,7 +1009,7 @@ impl HostOutputStream for DurableWorkerCtx { result.result.map(|_| ()).map_err(StreamError::from) } else { // Composed from write_zeroes + blocking_flush, both of which handle - // quota enforcement individually — mirrors blocking_write_and_flush. + // filesystem effect coordination individually — mirrors blocking_write_and_flush. let self2 = Resource::new_borrow(self_.rep()); self.write_zeroes(self_, len).await?; self.blocking_flush(self2).await?; @@ -1088,18 +1069,21 @@ impl HostOutputStream for DurableWorkerCtx { self.observe_function_call("io::streams::output_stream", "splice"); let stream_rep = self_.rep(); - reserve_filesystem_stream_growth(self, stream_rep, len).await?; + if !self + .state + .open_filesystem_output_streams + .contains_key(&stream_rep) + { + return HostOutputStream::splice(self.table(), self_, src, len).await; + } + if filesystem_stream_effect_is_active(self, stream_rep)? { + return HostOutputStream::splice(self.table(), self_, src, len).await; + } + let effect = begin_filesystem_stream_effect(self, stream_rep).await?; + prepare_filesystem_stream_effect(self, stream_rep, effect)?; let result = HostOutputStream::splice(self.table(), self_, src, len).await; - match &result { - Ok(spliced) => { - mark_filesystem_stream_write_enqueued(self, stream_rep, *spliced); - reconcile_pending_filesystem_stream_reservation(self, stream_rep).await; - } - Err(_) => { - rollback_pending_filesystem_stream_reservation(self, stream_rep).await; - } - } + clear_unused_filesystem_stream_effect(self, stream_rep)?; result } } @@ -1157,18 +1141,22 @@ impl HostOutputStream for DurableWorkerCtx { self.observe_function_call("io::streams::output_stream", "blocking_splice"); let stream_rep = self_.rep(); - reserve_filesystem_stream_growth(self, stream_rep, len).await?; + if !self + .state + .open_filesystem_output_streams + .contains_key(&stream_rep) + { + return HostOutputStream::blocking_splice(self.table(), self_, src, len).await; + } + if filesystem_stream_effect_is_active(self, stream_rep)? { + let readiness = self.table().get_mut(&self_)?.write_ready().await; + readiness?; + } + let effect = begin_filesystem_stream_effect(self, stream_rep).await?; + prepare_filesystem_stream_effect(self, stream_rep, effect)?; let result = HostOutputStream::blocking_splice(self.table(), self_, src, len).await; - match &result { - Ok(spliced) => { - mark_filesystem_stream_write_enqueued(self, stream_rep, *spliced); - reconcile_pending_filesystem_stream_reservation(self, stream_rep).await; - } - Err(_) => { - rollback_pending_filesystem_stream_reservation(self, stream_rep).await; - } - } + clear_unused_filesystem_stream_effect(self, stream_rep)?; result } } @@ -1182,7 +1170,6 @@ impl HostOutputStream for DurableWorkerCtx { state.output_stream_rep = None; } let result = HostOutputStream::drop(self.table(), rep).await; - reconcile_pending_filesystem_stream_reservation(self, handle).await; if result.is_ok() { // Only unclassify after the resource is really gone: reps are recycled by the // resource table, and a failed drop leaves the file stream live. @@ -1543,134 +1530,70 @@ async fn blocking_write_zeroes_and_flush_chunked( Ok(()) } -async fn reserve_filesystem_stream_growth( +fn filesystem_stream_effect_is_active( ctx: &mut DurableWorkerCtx, stream_rep: u32, - write_len: u64, -) -> Result<(), StreamError> { - let Some(stream_state) = ctx.state.open_filesystem_output_streams.get(&stream_rep) else { - if write_len > 0 { - ctx.reserve_filesystem_storage(write_len) - .await - .map_err(|e| StreamError::Trap(wasmtime::Error::from_anyhow(e)))?; - } - return Ok(()); - }; - - if stream_state.pending_reservation.is_some() { - return Ok(()); - } - - let stream_state = stream_state.clone(); - - let current_size = { - let fd_borrow = Resource::::new_borrow(stream_state.descriptor_rep); - let mut view = ctx.as_wasi_view(); - match FsHostDescriptor::stat(&mut view.filesystem(), fd_borrow).await { - Ok(stat) => stat.size, - Err(_) => { - if write_len > 0 { - ctx.reserve_filesystem_storage(write_len) - .await - .map_err(|e| StreamError::Trap(wasmtime::Error::from_anyhow(e)))?; - } - return Ok(()); - } - } - }; - - let requested_end = match stream_state.position { - Some(position) => position.saturating_add(write_len), - None => current_size.saturating_add(write_len), - }; - - let requested_growth = requested_end.saturating_sub(current_size); - - if requested_growth > 0 { - ctx.reserve_filesystem_storage(requested_growth) - .await - .map_err(|e| StreamError::Trap(wasmtime::Error::from_anyhow(e)))?; - } - - if let Some(state) = ctx - .state - .open_filesystem_output_streams - .get_mut(&stream_rep) - { - state.pending_reservation = Some(PendingFilesystemReservation { - base_size: current_size, - reserved_growth: requested_growth, - }); - } - - Ok(()) +) -> Result { + let stream = Resource::::new_borrow(stream_rep); + let output = ctx.table().get(&stream)?; + Ok(output + .as_any() + .downcast_ref::() + .is_some_and(|stream| stream.is_active())) } -fn mark_filesystem_stream_write_enqueued( - ctx: &mut DurableWorkerCtx, +async fn begin_filesystem_stream_effect( + ctx: &DurableWorkerCtx, stream_rep: u32, - write_len: u64, -) { - if let Some(state) = ctx +) -> Result { + let append = ctx .state .open_filesystem_output_streams - .get_mut(&stream_rep) - && let Some(position) = &mut state.position - { - *position = position.saturating_add(write_len); + .get(&stream_rep) + .is_some_and(|state| state.position.is_none()); + if append { + ctx.filesystem_runtime() + .begin_append_effect() + .await + .map_err(StreamError::Trap) + } else { + ctx.filesystem_runtime() + .begin_effect() + .await + .map_err(StreamError::Trap) } } -async fn rollback_pending_filesystem_stream_reservation( +fn prepare_filesystem_stream_effect( ctx: &mut DurableWorkerCtx, stream_rep: u32, -) { - let reserved_growth = ctx - .state - .open_filesystem_output_streams - .get_mut(&stream_rep) - .and_then(|state| state.pending_reservation.take()) - .map(|pending| pending.reserved_growth) - .unwrap_or(0); - - if reserved_growth > 0 { - ctx.release_filesystem_storage_space(reserved_growth).await; - } + effect: crate::services::agent_filesystem::AgentFilesystemEffectLease, +) -> Result<(), StreamError> { + let stream = Resource::::new_borrow(stream_rep); + let output = ctx.table().get(&stream)?; + let stream = output + .as_any() + .downcast_ref::() + .ok_or_else(|| { + StreamError::Trap(wasmtime::Error::msg( + "filesystem output stream is not coordinated", + )) + })?; + stream.prepare_effect(effect); + Ok(()) } -async fn reconcile_pending_filesystem_stream_reservation( +fn clear_unused_filesystem_stream_effect( ctx: &mut DurableWorkerCtx, stream_rep: u32, -) { - let Some((descriptor_rep, pending)) = ctx - .state - .open_filesystem_output_streams - .get_mut(&stream_rep) - .and_then(|state| { - state - .pending_reservation - .take() - .map(|pending| (state.descriptor_rep, pending)) - }) - else { - return; - }; - - if pending.reserved_growth == 0 { - return; - } - - let actual_growth = { - let fd_borrow = Resource::::new_borrow(descriptor_rep); - let mut view = ctx.as_wasi_view(); - match FsHostDescriptor::stat(&mut view.filesystem(), fd_borrow).await { - Ok(stat) => stat.size.saturating_sub(pending.base_size), - Err(_) => pending.reserved_growth, - } - }; - - let over_reserved = pending.reserved_growth.saturating_sub(actual_growth); - if over_reserved > 0 { - ctx.release_filesystem_storage_space(over_reserved).await; +) -> Result<(), StreamError> { + let stream = Resource::::new_borrow(stream_rep); + let output = ctx.table().get(&stream)?; + if let Some(stream) = output + .as_any() + .downcast_ref::( + ) { + stream.clear_unused_effect(); } + Ok(()) } diff --git a/golem-worker-executor/src/durable_host/mod.rs b/golem-worker-executor/src/durable_host/mod.rs index 88084a4b8b..5621fddadc 100644 --- a/golem-worker-executor/src/durable_host/mod.rs +++ b/golem-worker-executor/src/durable_host/mod.rs @@ -45,16 +45,13 @@ use crate::durable_host::durability::collect_named_retry_policies; use crate::durable_host::io::{ManagedStdErr, ManagedStdIn, ManagedStdOut}; use crate::durable_host::replay_state::{OplogEntryLookupResult, ReplayState}; use crate::metrics::ephemeral::record_non_suspending_failure; -use crate::metrics::storage::{ - STORAGE_TYPE_FILESYSTEM, record_storage_bytes_deleted, record_storage_bytes_written, -}; use crate::metrics::wasm::{record_number_of_replayed_functions, record_resume_worker}; use crate::model::event::InternalWorkerEvent; use crate::model::{ AgentConfig, ExecutionStatus, InvocationContext, LastError, ReadFileResult, TrapType, }; use crate::services::active_workers::MemoryGrant; -use crate::services::agent_storage_meter::AgentStorageMeter; +use crate::services::agent_resource_billing::AgentResourceBilling; use crate::services::agent_types::AgentTypesService; use crate::services::agent_webhooks::AgentWebhooksService; use crate::services::blob_store::BlobStoreService; @@ -62,7 +59,7 @@ use crate::services::card::{CardService, CardState}; use crate::services::card_interest::CardInterestIndex; use crate::services::component::ComponentService; use crate::services::environment_state::EnvironmentStateService; -use crate::services::file_loader::{FileLoader, FileUseToken}; +use crate::services::file_loader::FileLoader; use crate::services::golem_config::GolemConfig; use crate::services::key_value::KeyValueService; use crate::services::linear_memory::{ @@ -104,16 +101,14 @@ use chrono::{DateTime, Utc}; pub use durability::*; use futures::TryFutureExt; use futures::TryStreamExt; -use futures::future::try_join_all; use golem_common::base_model::oplog::{CardInstallFailure, QueuedCardEvent}; use golem_common::model::TransactionId; use golem_common::model::account::{AccountEmail, AccountId}; use golem_common::model::agent::{AgentMode, ParsedAgentId, Principal}; use golem_common::model::card::{CardId, StoredCard}; use golem_common::model::component::{ - AgentFilePermissions, CanonicalFilePath, ComponentId, ComponentRevision, InitialAgentFile, + AgentFilePermissions, CanonicalFilePath, ComponentId, ComponentRevision, }; -use golem_common::model::environment::EnvironmentId; use golem_common::model::invocation_context::{ AttributeValue, InvocationContextSpan, InvocationContextStack, SpanId, }; @@ -148,45 +143,13 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::error::Error; use std::fmt::{Debug, Display, Formatter}; use std::future::Future; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, RwLock, Weak}; use std::time::{Duration, Instant, SystemTime}; use std::vec; -use tempfile::TempDir; use tokio::sync::RwLock as TRwLock; -/// A worker's filesystem root directory. Either a random OS temp directory -/// (the default) or a deterministic path derived from the agent id. -/// -/// In both cases the directory is removed when this value is dropped. -enum WorkerDir { - /// Random temp dir created by `tempfile`. Auto-deleted on drop. - Temp(TempDir), - /// Deterministic directory. Deleted explicitly on drop. - Deterministic(PathBuf), -} - -impl WorkerDir { - fn path(&self) -> &Path { - match self { - WorkerDir::Temp(td) => td.path(), - WorkerDir::Deterministic(p) => p, - } - } -} - -impl Drop for WorkerDir { - fn drop(&mut self) { - if let WorkerDir::Deterministic(p) = self - && p.exists() - { - let _ = std::fs::remove_dir_all(p); - } - // WorkerDir::Temp is dropped automatically by TempDir's own Drop impl - } -} - use golem_common::base_model::component_metadata::AgentTypeProvisionConfig; use golem_service_base::model::auth::AuthCtx; use tokio_util::codec::{BytesCodec, FramedRead}; @@ -374,12 +337,13 @@ pub struct DurableWorkerCtx { pub owned_agent_id: OwnedAgentId, pub public_state: PublicDurableWorkerState, state: PrivateDurableWorkerState, - worker_dir: Arc, + filesystem_root: PathBuf, + filesystem_runtime: crate::services::agent_filesystem::AgentFilesystemRuntime, execution_status: Arc>, pub websocket_connection_pool: websocket::WebSocketConnectionPool, resource_limits: Arc, linear_memory: LinearMemoryTracker, - storage_meter: AgentStorageMeter, + resource_billing: AgentResourceBilling, /// Per-instance cache of resolved typed guest export handles, populated /// lazily on first use during invocation dispatch. agent_export_funcs: AgentExportFuncs, @@ -417,12 +381,10 @@ pub trait DurableResourceLimiter { impl Drop for DurableWorkerCtx { fn drop(&mut self) { - self.linear_memory.stop(Instant::now()); - self.resource_limits - .unregister_memory_meter(&self.owned_agent_id, self.linear_memory.meter()); - self.storage_meter.flush(Instant::now()); + self.filesystem_runtime.set_usage_observer(None); + self.resource_billing.abort(); self.resource_limits - .unregister_storage_meter(&self.owned_agent_id, &self.storage_meter); + .unregister_resource_billing(&self.owned_agent_id, &self.resource_billing); } } @@ -518,6 +480,12 @@ fn validate_unshared_memory_growth( } impl DurableWorkerCtx { + pub(crate) fn filesystem_runtime( + &self, + ) -> crate::services::agent_filesystem::AgentFilesystemRuntime { + self.filesystem_runtime.clone() + } + pub(crate) fn derive_idempotency_key(&mut self, oplog_index: OplogIndex) -> IdempotencyKey { let current_idempotency_key = self .state @@ -562,7 +530,9 @@ impl DurableWorkerCtx { component_service: Arc, resource_limits: Arc, config: Arc, - mut worker_config: AgentConfig, + filesystem_root: PathBuf, + filesystem_runtime: crate::services::agent_filesystem::AgentFilesystemRuntime, + worker_config: AgentConfig, execution_status: Arc>, file_loader: Arc, worker_fork: Arc, @@ -577,30 +547,7 @@ impl DurableWorkerCtx { per_invocation_http_call_limit: u64, per_invocation_rpc_call_limit: u64, ) -> Result { - let worker_dir = Arc::new( - if let Some(root) = &config.filesystem_storage.deterministic_root_dir { - let dir = root - .join(owned_agent_id.environment_id.to_string()) - .join(owned_agent_id.agent_id.component_id.to_string()) - .join(owned_agent_id.agent_id.agent_name_encoded()); - std::fs::create_dir_all(&dir).map_err(|e| { - WorkerExecutorError::runtime(format!( - "Failed to create deterministic directory {}: {e}", - dir.display() - )) - })?; - WorkerDir::Deterministic(dir) - } else { - WorkerDir::Temp(tempfile::Builder::new().prefix("golem").tempdir().map_err( - |e| { - WorkerExecutorError::runtime(format!( - "Failed to create temporary directory: {e}", - )) - }, - )?) - }, - ); - debug!("Created file system root at {:?}", worker_dir.path()); + debug!("Created file system root at {:?}", filesystem_root); debug!( "Worker {} initialized with deleted regions {}", @@ -643,58 +590,6 @@ impl DurableWorkerCtx { .get(&agent_id.agent_type) .cloned() }); - let initial_read_write_file_bytes: u64 = agent_type_provision_configs - .as_ref() - .map(|c| c.files.as_slice()) - .unwrap_or_default() - .iter() - .filter(|f| f.permissions == AgentFilePermissions::ReadWrite) - .map(|f| f.size) - .fold(0u64, u64::saturating_add); - let initial_filesystem_storage_usage = worker_config - .current_filesystem_storage_usage - .saturating_add(initial_read_write_file_bytes); - if initial_filesystem_storage_usage > resource_limits.max_disk_space_limit() { - return Err(WorkerExecutorError::worker_creation_failed( - owned_agent_id.agent_id.clone(), - format!( - "Provisioned read-write files require {initial_filesystem_storage_usage} bytes, exceeding the per-agent disk limit of {} bytes", - resource_limits.max_disk_space_limit() - ), - )); - } - - let files = prepare_filesystem( - &file_loader, - owned_agent_id.environment_id, - worker_dir.path(), - agent_type_provision_configs - .as_ref() - .map(|c| c.files.as_slice()) - .unwrap_or_default(), - ) - .await?; - - // Acquire storage semaphore permits for read-write initial component files. - // - // Read-only files are hardlinked from the FileLoader shared cache, so - // they occupy disk space only once per unique content hash regardless of - // how many workers reference them. FileLoader acquires the semaphore - // permit on the first cache miss and releases it when the last - // FileUseToken for that entry is dropped — no per-worker charge here. - // - // Read-write files are copied per-worker (each worker gets its own - // private inode and data blocks), so they must be charged individually. - if let Some(worker) = invocation_queue.upgrade() - && initial_read_write_file_bytes > 0 - { - worker - .acquire_initial_filesystem_storage(initial_read_write_file_bytes) - .await - .map_err(|trap| WorkerExecutorError::runtime(trap.to_string()))?; - } - worker_config.current_filesystem_storage_usage = initial_filesystem_storage_usage; - let agent_config = if agent_id.is_some() { effective_agent_config( worker_config.initial_agent_config.clone(), @@ -716,7 +611,7 @@ impl DurableWorkerCtx { }; let (wasi, io_ctx, table) = wasi_host::create_context( &[] as &[&str], - worker_dir.path().to_path_buf(), + filesystem_root.clone(), stdin.clone(), stdout, stderr, @@ -730,28 +625,29 @@ impl DurableWorkerCtx { connection_pool: http_connection_pool, is_replay: Arc::new(AtomicBool::new(false)), }; - let storage_meter = AgentStorageMeter::new( - execution_status.read().unwrap().agent_mode(), - worker_config.current_filesystem_storage_usage, - resource_limits.clone(), - Instant::now(), - ); - resource_limits.register_storage_meter(owned_agent_id.clone(), storage_meter.clone()); let worker = invocation_queue .upgrade() .expect("worker must remain alive while creating its context"); let retained_memory_grant = worker.linear_memory_grant(); let canonical_startup_bytes = worker.startup_linear_memory_bytes(); let admitted_startup_bytes = retained_memory_grant.lock().unwrap().bytes(); + let agent_mode = execution_status.read().unwrap().agent_mode(); let linear_memory = LinearMemoryTracker::new( canonical_startup_bytes, admitted_startup_bytes, - execution_status.read().unwrap().agent_mode(), + agent_mode, true, resource_limits.clone(), retained_memory_grant, Instant::now(), ); + let resource_billing = AgentResourceBilling::new( + agent_mode, + linear_memory.clone(), + resource_limits.clone(), + Instant::now(), + ); + filesystem_runtime.set_usage_observer(Some(Arc::new(resource_billing.clone()))); let weak_worker = Arc::downgrade(&worker); let memory_meter = linear_memory.meter().clone(); linear_memory.set_limit_exceeded_callback(Arc::new(move || { @@ -783,11 +679,8 @@ impl DurableWorkerCtx { worker_proxy, worker_config.deleted_regions.clone(), component_metadata, - worker_config.current_filesystem_storage_usage, worker_config.agent_effective_surface, worker_fork, - RwLock::new(compute_read_only_paths(&files)), - TRwLock::new(files), file_loader, worker_config.created_by, worker_config.created_by_email, @@ -805,8 +698,29 @@ impl DurableWorkerCtx { if state.is_live() { linear_memory.switch_to_live(); } - resource_limits - .register_memory_meter(owned_agent_id.clone(), linear_memory.meter().clone()); + resource_limits.register_resource_billing(owned_agent_id.clone(), resource_billing.clone()); + + filesystem_runtime.set_retry_callback(Some({ + let worker = Arc::downgrade(&worker); + let invocation_deadline_exceeded = state.invocation_deadline_exceeded.clone(); + let tail_work_deadline_exceeded = state.tail_work_deadline_exceeded.clone(); + Arc::new(move || { + let worker = worker.clone(); + let invocation_deadline_exceeded = invocation_deadline_exceeded.clone(); + let tail_work_deadline_exceeded = tail_work_deadline_exceeded.clone(); + Box::pin(async move { + if invocation_deadline_exceeded.load(Ordering::Acquire) + || tail_work_deadline_exceeded.load(Ordering::Acquire) + { + return false; + } + let Some(worker) = worker.upgrade() else { + return false; + }; + worker.filesystem_retry_permitted().await + }) + }) + })); Ok(DurableWorkerCtx { table: Arc::new(Mutex::new(table)), @@ -824,11 +738,12 @@ impl DurableWorkerCtx { oplog: oplog.clone(), }, state, - worker_dir, + filesystem_root, + filesystem_runtime, execution_status, resource_limits, linear_memory, - storage_meter, + resource_billing, agent_export_funcs: AgentExportFuncs::default(), _store_alive_guard: StoreAliveGuard::new(), }) @@ -851,9 +766,8 @@ impl DurableWorkerCtx { /// Records one outgoing HTTP call against the monthly account quota. /// /// Returns `Err(WorkerMonthlyHttpCallBudgetExhausted)` if the monthly budget - /// is exhausted. This trap maps to `RetryDecision::TryStop` — the worker is - /// suspended (same as filesystem `NodeOutOfFilesystemStorage` -> `ReacquirePermits`), - /// and will be resumed when the registry replenishes the budget. + /// is exhausted. This trap maps to `RetryDecision::TryStop`; the worker is + /// suspended and resumed when the registry replenishes the budget. pub fn record_monthly_http_call(&mut self) -> anyhow::Result<()> { if self.state.is_live() && !self.state.resource_limit_entry.record_http_call() { Err(anyhow!( @@ -889,7 +803,7 @@ impl DurableWorkerCtx { match table.get(fd)? { Descriptor::File(f) => { - let read_only = self.state.read_only_paths.read().unwrap().contains(&f.path); + let read_only = self.filesystem_runtime.is_read_only(&f.path); Ok(read_only) } @@ -897,6 +811,54 @@ impl DurableWorkerCtx { } } + fn descriptor_path( + &mut self, + fd: &Resource, + ) -> Result { + let table = Arc::get_mut(&mut self.table) + .expect("ResourceTable is shared and cannot be borrowed mutably") + .get_mut() + .expect("ResourceTable mutex must never fail"); + Ok(match table.get(fd)? { + Descriptor::File(file) => file.path.clone(), + Descriptor::Dir(directory) => directory.path.clone(), + }) + } + + fn fail_if_read_only_path( + &mut self, + fd: &Resource, + path: &str, + follow_final_symlink: bool, + ) -> FsResult<()> { + let target = self.descriptor_path(fd)?.join(path); + if self + .filesystem_runtime + .is_read_only_path(&target, follow_final_symlink) + { + Err(wasmtime_wasi::p2::bindings::filesystem::types::ErrorCode::NotPermitted.into()) + } else { + Ok(()) + } + } + + fn fail_if_contains_read_only_path( + &mut self, + fd: &Resource, + path: &str, + follow_final_symlink: bool, + ) -> FsResult<()> { + let target = self.descriptor_path(fd)?.join(path); + if self + .filesystem_runtime + .contains_read_only_path(&target, follow_final_symlink) + { + Err(wasmtime_wasi::p2::bindings::filesystem::types::ErrorCode::NotPermitted.into()) + } else { + Ok(()) + } + } + fn fail_if_read_only(&mut self, fd: &Resource) -> FsResult<()> { if self.check_if_file_is_readonly(fd)? { Err(wasmtime_wasi::p2::bindings::filesystem::types::ErrorCode::NotPermitted.into()) @@ -1371,6 +1333,12 @@ impl DurableWorkerCtx { self.linear_memory.clone() } + /// Returns an owned handle so callers can release the store lock before awaiting + /// resource-window transitions. Cloning only increments the handle's `Arc` count. + pub(crate) fn resource_billing(&self) -> AgentResourceBilling { + self.resource_billing.clone() + } + async fn switch_to_live(&self) { self.state.replay_state.switch_to_live().await; self.linear_memory.switch_to_live(); @@ -1390,186 +1358,6 @@ impl DurableWorkerCtx { } self.state.active_custom_invocations.clear(); } - - pub fn current_filesystem_storage_usage(&self) -> u64 { - self.state.current_filesystem_storage_usage - } - - pub fn max_disk_space(&self) -> u64 { - self.resource_limits.max_disk_space_limit() - } - - /// Check whether acquiring `new_bytes` would breach the per-plan storage - /// limit. Returns `WorkerAgentExceededFilesystemStorageLimit` (permanent) if so. - /// Does NOT check the executor semaphore pool — that is done by - /// `acquire_filesystem_space`. - /// - /// No-op during replay. - pub fn check_filesystem_storage_quota(&self, new_bytes: u64) -> anyhow::Result<()> { - if self.state.is_replay() { - return Ok(()); - } - let after = self - .state - .current_filesystem_storage_usage - .saturating_add(new_bytes); - if after > self.resource_limits.max_disk_space_limit() { - Err(anyhow!( - GolemSpecificWasmTrap::WorkerAgentExceededFilesystemStorageLimit - )) - } else { - Ok(()) - } - } - - /// Acquire `new_bytes` of storage from the executor semaphore pool. - /// - /// - During replay: no-op (permits were pre-acquired at startup). - /// - During live execution: calls `Worker::acquire_filesystem_space`, which - /// tries the semaphore non-blockingly. On failure returns - /// `NodeOutOfFilesystemStorage` (retriable via `ReacquirePermits`). - /// - /// Call `check_filesystem_quota` before calling this to enforce the per-plan - /// limit (`WorkerAgentExceededFilesystemStorageLimit`). This method only checks the - /// executor-wide semaphore pool (`NodeOutOfFilesystemStorage`). - pub async fn acquire_filesystem_storage_space(&mut self, new_bytes: u64) -> anyhow::Result<()> { - if self.state.is_replay() { - return Ok(()); - } - // Acquire the semaphore permit first (non-blocking try). Writing the - // oplog entry after a confirmed acquire ensures the oplog accurately - // reflects only committed storage changes — a failed acquire leaves no - // phantom delta that would inflate `current_filesystem_storage_usage` on restart. - self.public_state - .worker() - .acquire_filesystem_storage_space(new_bytes) - .await?; - self.public_state - .worker() - .add_to_oplog(OplogEntry::filesystem_storage_usage_update( - new_bytes as i64, - )) - .await; - self.storage_meter.on_acquire(new_bytes, Instant::now()); - self.state.current_filesystem_storage_usage += new_bytes; - let account_id = self.created_by().to_string(); - let environment_id = self.state.owned_agent_id.environment_id().to_string(); - record_storage_bytes_written( - STORAGE_TYPE_FILESYSTEM, - &account_id, - &environment_id, - new_bytes, - ); - Ok(()) - } - - /// Release `freed_bytes` back to the executor semaphore pool. - /// Called when files are deleted or truncated. - /// During replay this is a no-op. - pub async fn release_filesystem_storage_space(&mut self, freed_bytes: u64) { - if self.state.is_replay() { - return; - } - let freed_bytes = freed_bytes.min(self.state.current_filesystem_storage_usage); - if freed_bytes == 0 { - return; - } - self.public_state - .worker() - .add_to_oplog(OplogEntry::filesystem_storage_usage_update( - -(freed_bytes as i64), - )) - .await; - self.public_state - .worker() - .release_filesystem_storage_space(freed_bytes) - .await; - self.storage_meter.on_release(freed_bytes, Instant::now()); - self.state.current_filesystem_storage_usage -= freed_bytes; - let account_id = self.created_by().to_string(); - let environment_id = self.state.owned_agent_id.environment_id().to_string(); - record_storage_bytes_deleted( - STORAGE_TYPE_FILESYSTEM, - &account_id, - &environment_id, - freed_bytes, - ); - } - - /// Check the per-agent storage quota and acquire permits from the - /// executor-wide semaphore pool in a single step. - /// - /// This combines `check_filesystem_storage_quota` (per-plan limit) and - /// `acquire_filesystem_storage_space` (executor semaphore) — the two must - /// always be called together in this order. - pub async fn reserve_filesystem_storage(&mut self, new_bytes: u64) -> anyhow::Result<()> { - self.check_filesystem_storage_quota(new_bytes)?; - self.acquire_filesystem_storage_space(new_bytes).await - } - - pub(crate) fn prepare_filesystem_storage_reservation( - &mut self, - new_bytes: u64, - ) -> anyhow::Result>>> { - if new_bytes == 0 || self.state.is_replay() { - return Ok(None); - } - self.check_filesystem_storage_quota(new_bytes)?; - self.state.current_filesystem_storage_usage += new_bytes; - Ok(Some(self.public_state.worker())) - } - - pub(crate) fn rollback_filesystem_storage_reservation(&mut self, new_bytes: u64) { - if new_bytes == 0 || self.state.is_replay() { - return; - } - self.state.current_filesystem_storage_usage -= new_bytes; - } - - pub(crate) fn finish_filesystem_storage_reservation(&mut self, new_bytes: u64) { - if new_bytes == 0 || self.state.is_replay() { - return; - } - let account_id = self.created_by().to_string(); - let environment_id = self.state.owned_agent_id.environment_id().to_string(); - record_storage_bytes_written( - STORAGE_TYPE_FILESYSTEM, - &account_id, - &environment_id, - new_bytes, - ); - } - - pub(crate) fn prepare_filesystem_storage_release( - &mut self, - freed_bytes: u64, - ) -> Option<(Arc>, u64)> { - if freed_bytes == 0 || self.state.is_replay() { - return None; - } - let freed_bytes = freed_bytes.min(self.state.current_filesystem_storage_usage); - if freed_bytes == 0 { - None - } else { - self.state.current_filesystem_storage_usage -= freed_bytes; - Some((self.public_state.worker(), freed_bytes)) - } - } - - pub(crate) fn finish_filesystem_storage_release(&mut self, freed_bytes: u64) { - if freed_bytes == 0 || self.state.is_replay() { - return; - } - let account_id = self.created_by().to_string(); - let environment_id = self.state.owned_agent_id.environment_id().to_string(); - record_storage_bytes_deleted( - STORAGE_TYPE_FILESYSTEM, - &account_id, - &environment_id, - freed_bytes, - ); - } - pub fn increase_memory(&mut self, delta: u64) { let (_, reconciling) = self.linear_memory.grow(delta, Instant::now()); if self.state.is_live() && !reconciling { @@ -1633,7 +1421,7 @@ impl DurableWorkerCtx { /// the caller falls through to policy-based resolution. pub(crate) fn fixed_decision_for_trap_type(trap_type: &TrapType) -> Option { match trap_type { - TrapType::Interrupt(InterruptKind::Interrupt(ts)) => Some(RetryDecision::TryStop(*ts)), + TrapType::Interrupt(InterruptKind::Interrupt(_)) => Some(RetryDecision::None), TrapType::Interrupt(InterruptKind::Suspend(ts)) => Some(RetryDecision::TryStop(*ts)), TrapType::Interrupt(InterruptKind::Restart) => Some(RetryDecision::Immediate), TrapType::Interrupt(InterruptKind::Jump) => Some(RetryDecision::Immediate), @@ -1658,14 +1446,6 @@ impl DurableWorkerCtx { error: AgentError::ExceededTableLimit, .. } => Some(RetryDecision::None), - TrapType::Error { - error: AgentError::NodeOutOfFilesystemStorage, - .. - } => Some(RetryDecision::ReacquirePermits), - TrapType::Error { - error: AgentError::AgentExceededFilesystemStorageLimit, - .. - } => Some(RetryDecision::None), TrapType::Error { error: AgentError::AgentTerminatedByQuota(_), .. @@ -1726,10 +1506,6 @@ impl DurableWorkerCtx { AgentError::ExceededTableLimit => "exceeded-table-limit", AgentError::ExceededHttpCallLimit => "exceeded-http-call-limit", AgentError::ExceededRpcCallLimit => "exceeded-rpc-call-limit", - AgentError::NodeOutOfFilesystemStorage => "node-out-of-filesystem-storage", - AgentError::AgentExceededFilesystemStorageLimit => { - "agent-exceeded-filesystem-storage-limit" - } AgentError::InternalError(_) => "internal-error", AgentError::DeterministicTrap(_) => "deterministic-trap", AgentError::PermanentError(_) => "permanent-error", @@ -3487,23 +3263,18 @@ impl DurableWorkerCtx { None }; - { - let mut current_files = self.state.files.write().await; - update_filesystem( - &mut current_files, + let _filesystem_update = self + .filesystem_runtime + .update_initial_files( &self.state.file_loader, self.owned_agent_id.environment_id, - self.worker_dir.path(), new_agent_type_provision_configs .as_ref() .map(|c| c.files.as_slice()) .unwrap_or_default(), ) - .await?; - - let mut read_only_paths = self.state.read_only_paths.write().unwrap(); - *read_only_paths = compute_read_only_paths(¤t_files); - } + .await + .map_err(|error| WorkerExecutorError::runtime(error.to_string()))?; if let Some((updated_agent_config, agent_effective_surface, initial_wallet_cards)) = updated_agent_state @@ -5453,7 +5224,7 @@ impl FileSystemReading for DurableWorkerCtx { &self, path: &CanonicalFilePath, ) -> Result { - let root = self.worker_dir.path(); + let root = &self.filesystem_root; let target = root.join(PathBuf::from(path.to_rel_string())); { @@ -5477,7 +5248,7 @@ impl FileSystemReading for DurableWorkerCtx { if metadata.is_file() { let is_readonly_by_host = metadata.permissions().readonly(); - let is_readonly_by_us = self.state.read_only_paths.read().unwrap().contains(&target); + let is_readonly_by_us = self.filesystem_runtime.is_read_only(&target); let permissions = if is_readonly_by_host || is_readonly_by_us { AgentFilePermissions::ReadOnly @@ -5528,12 +5299,7 @@ impl FileSystemReading for DurableWorkerCtx { if metadata.is_file() { let is_readonly_by_host = metadata.permissions().readonly(); // additionally consider permissions we maintain ourselves - let is_readonly_by_us = self - .state - .read_only_paths - .read() - .unwrap() - .contains(&entry.path()); + let is_readonly_by_us = self.filesystem_runtime.is_read_only(&entry.path()); let permissions = if is_readonly_by_host || is_readonly_by_us { AgentFilePermissions::ReadOnly @@ -5564,7 +5330,7 @@ impl FileSystemReading for DurableWorkerCtx { &self, path: &CanonicalFilePath, ) -> Result { - let root = self.worker_dir.path(); + let root = &self.filesystem_root; let target = root.join(PathBuf::from(path.to_rel_string())); { @@ -6196,17 +5962,9 @@ struct ActiveDurableScope { replay_end: Option, } -#[derive(Debug, Clone)] +#[derive(Debug)] pub(crate) struct FilesystemOutputStreamState { - pub descriptor_rep: u32, pub position: Option, - pub pending_reservation: Option, -} - -#[derive(Debug, Clone)] -pub(crate) struct PendingFilesystemReservation { - pub base_size: u64, - pub reserved_growth: u64, } /// Direction of a P3 TCP one-shot stream acquisition (`send` vs `receive`). @@ -6399,11 +6157,6 @@ struct PrivateDurableWorkerState { agent_wallet_cards: BTreeMap, card_event_boundary_scan: Option, - /// Running total of storage bytes acquired from the executor semaphore pool - /// by this worker since it last started. Incremented on every successful - /// write; decremented when files are deleted or truncated. - current_filesystem_storage_usage: u64, - invocation_context: InvocationContext, current_span_id: SpanId, forward_trace_context_headers: bool, @@ -6411,8 +6164,6 @@ struct PrivateDurableWorkerState { worker_fork: Arc, - read_only_paths: RwLock>, - files: TRwLock>, file_loader: Arc, shard_service: Arc, @@ -6613,11 +6364,8 @@ impl PrivateDurableWorkerState { worker_proxy: Arc, deleted_regions: DeletedRegions, component_metadata: Component, - current_filesystem_storage_usage: u64, _agent_effective_surface: golem_common::model::card::EffectiveSurface, worker_fork: Arc, - read_only_paths: RwLock>, - files: TRwLock>, file_loader: Arc, created_by: AccountId, created_by_email: AccountEmail, @@ -6734,15 +6482,12 @@ impl PrivateDurableWorkerState { agent_effective_surface, agent_wallet_cards, card_event_boundary_scan: None, - current_filesystem_storage_usage, replay_state, invocation_context, current_span_id, forward_trace_context_headers: true, set_outgoing_http_idempotency_key: true, worker_fork, - read_only_paths, - files, file_loader, created_by, created_by_email, @@ -7544,229 +7289,6 @@ impl WasiHttpView for DurableWorkerCtx { } } -/// File that was provisioned due to metadata. There might be additional files that the -/// worker created itself. -/// Ro files are symlinked to the proper location and might be garbage collected when the token is dropped. -/// Rw files are directly copied to the target location. -enum IFSWorkerFile { - Ro { - file: InitialAgentFile, - _token: FileUseToken, - }, - Rw, -} - -async fn prepare_filesystem( - file_loader: &Arc, - environment_id: EnvironmentId, - root: &Path, - files: &[InitialAgentFile], -) -> Result, WorkerExecutorError> { - let futures = files.iter().map(|file| { - let path = root.join(PathBuf::from(file.path.to_rel_string())); - let file = file.clone(); - let permissions = file.permissions; - let file_loader = file_loader.clone(); - async move { - match permissions { - AgentFilePermissions::ReadOnly => { - debug!("Loading read-only file {}", path.display()); - let token = file_loader - .get_read_only_to(environment_id, file.content_hash, &path, file.size) - .await?; - Ok::<_, WorkerExecutorError>(( - path, - IFSWorkerFile::Ro { - file, - _token: token, - }, - )) - } - AgentFilePermissions::ReadWrite => { - debug!("Loading read-write file {}", path.display()); - file_loader - .get_read_write_to(environment_id, file.content_hash, &path) - .await?; - Ok((path, IFSWorkerFile::Rw)) - } - } - } - }); - Ok(HashMap::from_iter(try_join_all(futures).await?)) -} - -async fn update_filesystem( - current_state: &mut HashMap, - file_loader: &Arc, - environment_id: EnvironmentId, - root: &Path, - files: &[InitialAgentFile], -) -> Result<(), WorkerExecutorError> { - enum UpdateFileSystemResult { - NoChanges, - Remove(PathBuf), - Replace { path: PathBuf, value: IFSWorkerFile }, - } - - let desired_paths: HashSet = HashSet::from_iter( - files - .iter() - .map(|f| root.join(PathBuf::from(f.path.to_rel_string()))), - ); - - // We do this in two phases to make errors less likely. First, delete all files that are no longer needed and then create - // new ones. - let futures_phase_1 = current_state.iter().map(|(path, file)| { - let path = path.clone(); - let should_keep = desired_paths.contains(&path); - async move { - match file { - IFSWorkerFile::Ro { file, .. } if !should_keep => { - tokio::fs::remove_dir(&path).await.map_err(|e| { - WorkerExecutorError::FileSystemError { - path: file.path.to_rel_string(), - reason: format!("Failed deleting file during update: {e}"), - } - })?; - Ok::<_, WorkerExecutorError>(UpdateFileSystemResult::Remove(path)) - } - _ => Ok(UpdateFileSystemResult::NoChanges), - } - } - }); - - let futures_phase_2 = files.iter().map(|file| { - let path = root.join(PathBuf::from(file.path.to_rel_string())); - let file = file.clone(); - let permissions = file.permissions; - let file_loader = file_loader.clone(); - - let existing = current_state.get(&path); - - async move { - match (permissions, existing) { - (AgentFilePermissions::ReadOnly, None) => { - debug!("Loading read-only file {}", path.display()); - - let exists = tokio::fs::try_exists(&path).map_err(|e| WorkerExecutorError::FileSystemError { path: file.path.to_rel_string(), reason: format!("Failed checking whether path exists: {e}") }).await?; - - if exists { - // Try removing it if it's an empty directory; this will fail otherwise, and we can report the error. - tokio::fs::remove_dir(&path).await.map_err(|e| - WorkerExecutorError::FileSystemError { - path: file.path.to_rel_string(), - reason: format!("Tried replacing an existing non-empty path with ro file during update: {e}"), - } - )?; - }; - - let token = file_loader - .get_read_only_to(environment_id, file.content_hash, &path, file.size) - .await?; - - Ok::<_, WorkerExecutorError>(UpdateFileSystemResult::Replace { path, value: IFSWorkerFile::Ro { file, _token: token } }) - } - (AgentFilePermissions::ReadOnly, Some(IFSWorkerFile::Ro { file: existing_file, .. })) => { - if existing_file.content_hash == file.content_hash { - Ok(UpdateFileSystemResult::NoChanges) - } else { - debug!("updating ro file {}", path.display()); - tokio::fs::remove_file(&path).await.map_err(|e| - WorkerExecutorError::FileSystemError { - path: file.path.to_rel_string(), - reason: format!("Failed deleting file during update: {e}"), - } - )?; - let token = file_loader - .get_read_only_to(environment_id, file.content_hash, &path, file.size) - .await?; - Ok::<_, WorkerExecutorError>(UpdateFileSystemResult::Replace { path, value: IFSWorkerFile::Ro { file, _token: token } }) - } - } - (AgentFilePermissions::ReadOnly, Some(IFSWorkerFile::Rw)) => { - Err(WorkerExecutorError::FileSystemError { - path: file.path.to_rel_string(), - reason: "Tried updating rw file to ro during update".to_string(), - }) - } - (AgentFilePermissions::ReadWrite, None) => { - debug!("Loading rw file {}", path.display()); - - let exists = tokio::fs::try_exists(&path).map_err(|e| WorkerExecutorError::FileSystemError { path: file.path.to_rel_string(), reason: format!("Failed checking whether path exists: {e}") }).await?; - - if exists { - let metadata = tokio::fs::metadata(&path).await.map_err(|e| - WorkerExecutorError::FileSystemError { - path: file.path.to_rel_string(), - reason: format!("Failed getting metadata of path: {e}"), - } - )?; - - if metadata.is_file() { - return Ok(UpdateFileSystemResult::NoChanges); - } - - // Try removing it if it's an empty directory, this will fail otherwise, and we can report the error. - tokio::fs::remove_dir(&path).await.map_err(|e| - WorkerExecutorError::FileSystemError { - path: file.path.to_rel_string(), - reason: format!("Tried replacing an existing non-empty path with rw file during update: {e}"), - } - )?; - } - - file_loader - .get_read_write_to(environment_id, file.content_hash, &path) - .await?; - Ok::<_, WorkerExecutorError>(UpdateFileSystemResult::Replace { path, value: IFSWorkerFile::Rw }) - } - (AgentFilePermissions::ReadWrite, Some(IFSWorkerFile::Ro { .. })) => { - debug!("Updating ro file to rw {}", path.display()); - tokio::fs::remove_file(&path).await.map_err(|e| - WorkerExecutorError::FileSystemError { - path: file.path.to_rel_string(), - reason: format!("Failed deleting file during update: {e}"), - } - )?; - file_loader - .get_read_write_to(environment_id, file.content_hash, &path) - .await?; - Ok::<_, WorkerExecutorError>(UpdateFileSystemResult::Replace { path, value: IFSWorkerFile::Rw }) - } - (AgentFilePermissions::ReadWrite, Some(IFSWorkerFile::Rw)) => { - debug!("Updating rw file {}", path.display()); - Ok(UpdateFileSystemResult::NoChanges) - } - } - } - }); - - let mut results = try_join_all(futures_phase_1).await?; - results.extend(try_join_all(futures_phase_2).await?); - - for result in results { - match result { - UpdateFileSystemResult::NoChanges => {} - UpdateFileSystemResult::Remove(path) => { - current_state.remove(&path); - } - UpdateFileSystemResult::Replace { path, value } => { - current_state.insert(path, value); - } - } - } - - Ok(()) -} - -fn compute_read_only_paths(files: &HashMap) -> HashSet { - let ro_paths = files.iter().filter_map(|(p, f)| match f { - IFSWorkerFile::Ro { .. } => Some(p.clone()), - _ => None, - }); - HashSet::from_iter(ro_paths) -} - /// Helper macro for expecting a given type of OplogEntry as the next entry in the oplog during /// replay, while skipping hint entries. /// The macro expression's type is `Result<(OplogIndex, OplogEntry), WorkerExecutorError>` and it fails if the next non-hint diff --git a/golem-worker-executor/src/durable_host/p3/filesystem.rs b/golem-worker-executor/src/durable_host/p3/filesystem.rs index 9b01694600..602b5ef7f7 100644 --- a/golem-worker-executor/src/durable_host/p3/filesystem.rs +++ b/golem-worker-executor/src/durable_host/p3/filesystem.rs @@ -18,8 +18,8 @@ use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use std::sync::Mutex; -use std::sync::OnceLock; use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; use crate::durable_host::filesystem::types::calculate_metadata_hash_parts; use crate::durable_host::p3::{ @@ -27,14 +27,33 @@ use crate::durable_host::p3::{ observe_function_call_store, run_read_access, wasi_filesystem_view, }; use crate::durable_host::tail_work::TailActivity; +#[cfg(test)] +use crate::services::agent_filesystem::state_postcondition; +use crate::services::agent_filesystem::{ + AgentFilesystemRuntime, FILESYSTEM_MUTATION_MAX_ATTEMPTS, FILESYSTEM_MUTATION_RETRY_TIMEOUT, + MutationDecision, MutationEffect, MutationFailure, MutationOperation, MutationPostcondition, + NativeMutationGuestError, NativeOpenOptions, NativeOpenResult, PathObjectType, RequestedTime, + create_directory as native_create_directory, create_directory_postcondition, descriptor_state, + descriptor_times, hard_link as native_hard_link, link_postcondition, + native_write_failure_effect, open as native_open, open_postcondition, path_state, + path_state_with_follow, path_times, proven_write_progress_effect, + remove_directory as native_remove_directory, remove_postcondition, rename as native_rename, + rename_postcondition, resize_file as native_resize_file, resize_postcondition, + run_blocking_filesystem_mutation, set_descriptor_times as native_set_descriptor_times, + set_path_times as native_set_path_times, symlink as native_symlink, symlink_postcondition, + symlink_state, sync_descriptor as native_sync_descriptor, times_postcondition, + unlink_file as native_unlink_file, validate_descriptor_times, validate_directory_mutation, + validate_open, validate_resize, validate_two_directory_mutation, +}; use crate::workerctx::WorkerCtx; +use bytes::Bytes; use cap_std::fs::FileExt; use golem_common::model::oplog::host_functions::{ P3FilesystemTypesDescriptorStat, P3FilesystemTypesDescriptorStatAt, }; use golem_common::model::oplog::types::{SerializableFileTimes, SerializableP3FileSystemError}; use golem_common::model::oplog::{ - DurableFunctionType, HostRequestFileSystemPath, HostResponseP3FileSystemStat, OplogEntry, + DurableFunctionType, HostRequestFileSystemPath, HostResponseP3FileSystemStat, }; use wasmtime::AsContextMut; use wasmtime::StoreContextMut; @@ -46,28 +65,293 @@ use wasmtime_wasi::filesystem::{Descriptor, Dir, File, WasiFilesystem, WasiFiles use wasmtime_wasi::p3::bindings::filesystem::{preopens, types}; use wasmtime_wasi::p3::filesystem::{FilesystemError, FilesystemResult}; use wasmtime_wasi::runtime::spawn_blocking; -use wasmtime_wasi::{DirPerms, FilePerms}; - -static FILESYSTEM_APPEND_LOCK: OnceLock> = OnceLock::new(); +use wasmtime_wasi::{DirPerms, FilePerms, ResourceTableError}; struct FilesystemWriteChunk { contents: Vec, - result_tx: tokio::sync::oneshot::Sender>, + result_tx: tokio::sync::oneshot::Sender<(usize, FilesystemWriteResult)>, + admission: crate::services::agent_filesystem::AgentFilesystemEffectAdmission, + cancellation: tokio_util::sync::CancellationToken, +} + +#[derive(Clone, Debug)] +enum FilesystemWriteFailure { + Guest(types::ErrorCode), + Trap(String), +} + +type FilesystemWriteResult = Result<(), FilesystemWriteFailure>; + +enum P3MutationAction { + Retry, + Success, + Error(FilesystemError), + Trap, +} + +struct P3MutationAdapter { + runtime: AgentFilesystemRuntime, + operation: MutationOperation, + started: Instant, + attempts: usize, +} + +impl P3MutationAdapter { + fn new(runtime: AgentFilesystemRuntime) -> Self { + Self::for_operation(runtime, MutationOperation::Metadata) + } + + fn for_operation(runtime: AgentFilesystemRuntime, operation: MutationOperation) -> Self { + Self { + runtime, + operation, + started: Instant::now(), + attempts: 0, + } + } + + fn begin_attempt(&mut self) { + self.attempts += 1; + } + + #[cfg(test)] + async fn failure( + &self, + error: FilesystemError, + postcondition: MutationPostcondition, + retry_safe: bool, + ) -> P3MutationAction { + let Some(guest) = error.downcast_ref().cloned() else { + let _ = self + .runtime + .classify_mutation_failure::( + MutationFailure::Infrastructure(std::io::Error::other(error.to_string())), + MutationEffect::Unknown, + ) + .await; + return P3MutationAction::Trap; + }; + let effect = match (postcondition, &guest) { + (MutationPostcondition::Satisfied, _) => MutationEffect::DesiredPostconditionSatisfied, + (MutationPostcondition::NoEffect, _) => MutationEffect::ProvenNoEffect, + (MutationPostcondition::Unknown, _) => MutationEffect::Unknown, + }; + let failure = match guest.clone() { + types::ErrorCode::Quota => MutationFailure::StorageExhaustion { + guest: guest.clone(), + quota_hint: true, + }, + types::ErrorCode::InsufficientSpace => MutationFailure::StorageExhaustion { + guest: guest.clone(), + quota_hint: false, + }, + types::ErrorCode::Busy + | types::ErrorCode::Interrupted + | types::ErrorCode::InProgress + | types::ErrorCode::Already => MutationFailure::TransientGuest(guest.clone()), + types::ErrorCode::Access | types::ErrorCode::NotPermitted => { + MutationFailure::AccessGuest(guest.clone()) + } + types::ErrorCode::Io => MutationFailure::UnclassifiedGuest(guest.clone()), + _ => MutationFailure::Guest(guest.clone()), + }; + match self + .runtime + .classify_mutation_failure_for(self.operation, failure, effect) + .await + { + MutationDecision::PreserveGuest(error) => P3MutationAction::Error(error.into()), + MutationDecision::Quota => P3MutationAction::Error(types::ErrorCode::Quota.into()), + MutationDecision::InsufficientSpace => { + P3MutationAction::Error(types::ErrorCode::InsufficientSpace.into()) + } + MutationDecision::PhysicalPressure + if retry_safe + && postcondition == MutationPostcondition::NoEffect + && self.attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => + { + if self + .runtime + .recover_physical_pressure( + self.operation, + self.started + FILESYSTEM_MUTATION_RETRY_TIMEOUT, + ) + .await + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT + { + P3MutationAction::Retry + } else { + P3MutationAction::Error(types::ErrorCode::InsufficientSpace.into()) + } + } + MutationDecision::PhysicalPressure => { + P3MutationAction::Error(types::ErrorCode::InsufficientSpace.into()) + } + MutationDecision::BoundedRetry + if retry_safe + && postcondition == MutationPostcondition::NoEffect + && self.attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => + { + P3MutationAction::Retry + } + MutationDecision::BoundedRetry | MutationDecision::PreserveRaw => { + P3MutationAction::Error(guest.into()) + } + MutationDecision::Success => P3MutationAction::Success, + MutationDecision::Invalidate => P3MutationAction::Trap, + } + } + + async fn io_failure( + &self, + error: std::io::Error, + postcondition: MutationPostcondition, + retry_safe: bool, + ) -> P3MutationAction { + let raw_os_error = error.raw_os_error(); + let error_kind = error.kind(); + let error_message = error.to_string(); + let effect = match postcondition { + MutationPostcondition::Satisfied => MutationEffect::DesiredPostconditionSatisfied, + MutationPostcondition::NoEffect => MutationEffect::ProvenNoEffect, + MutationPostcondition::Unknown => MutationEffect::Unknown, + }; + match self + .runtime + .classify_mutation_failure_for::( + self.operation, + MutationFailure::Io(error), + effect, + ) + .await + { + MutationDecision::PreserveGuest(error) => P3MutationAction::Error(error.into()), + MutationDecision::Quota => P3MutationAction::Error(types::ErrorCode::Quota.into()), + MutationDecision::InsufficientSpace => { + P3MutationAction::Error(types::ErrorCode::InsufficientSpace.into()) + } + MutationDecision::PhysicalPressure + if retry_safe + && postcondition == MutationPostcondition::NoEffect + && self.attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => + { + if self + .runtime + .recover_physical_pressure( + self.operation, + self.started + FILESYSTEM_MUTATION_RETRY_TIMEOUT, + ) + .await + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT + { + P3MutationAction::Retry + } else { + P3MutationAction::Error(types::ErrorCode::InsufficientSpace.into()) + } + } + MutationDecision::PhysicalPressure => { + P3MutationAction::Error(types::ErrorCode::InsufficientSpace.into()) + } + MutationDecision::BoundedRetry + if retry_safe + && postcondition == MutationPostcondition::NoEffect + && self.attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && self.started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => + { + P3MutationAction::Retry + } + MutationDecision::BoundedRetry | MutationDecision::PreserveRaw => { + let error = raw_os_error.map_or_else( + || std::io::Error::new(error_kind, error_message), + std::io::Error::from_raw_os_error, + ); + P3MutationAction::Error(types::ErrorCode::from(&error).into()) + } + MutationDecision::Success => P3MutationAction::Success, + MutationDecision::Invalidate => P3MutationAction::Trap, + } + } +} + +fn p3_mutation_action_result(action: P3MutationAction) -> FilesystemResult<()> { + match action { + P3MutationAction::Success => Ok(()), + P3MutationAction::Error(error) => Err(error), + P3MutationAction::Trap => Err(FilesystemError::trap(wasmtime::Error::msg( + "agent filesystem mutation invalidated the runtime", + ))), + P3MutationAction::Retry => unreachable!("retry must be handled by the operation loop"), + } } struct FilesystemWriteConsumer { chunks_tx: Option>, - pending_chunk: Option<( - usize, - tokio::sync::oneshot::Receiver>, - )>, + pending_chunk: Option, + pending_invalidation: Option + Send>>>, + filesystem_runtime: crate::services::agent_filesystem::AgentFilesystemRuntime, +} + +struct PendingFilesystemWriteChunk { + result_rx: tokio::sync::oneshot::Receiver<(usize, FilesystemWriteResult)>, + cancellation: tokio_util::sync::CancellationToken, } impl FilesystemWriteConsumer { - fn new(chunks_tx: tokio::sync::mpsc::UnboundedSender) -> Self { + fn new( + chunks_tx: tokio::sync::mpsc::UnboundedSender, + filesystem_runtime: crate::services::agent_filesystem::AgentFilesystemRuntime, + ) -> Self { Self { chunks_tx: Some(chunks_tx), pending_chunk: None, + pending_invalidation: None, + filesystem_runtime, + } + } + + fn cancel(&mut self) { + if let Some(pending) = &self.pending_chunk { + pending.cancellation.cancel(); + } + self.chunks_tx.take(); + } + + fn poll_pending_result( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>> { + if let Some(invalidation) = &mut self.pending_invalidation { + return match invalidation.as_mut().poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(()) => { + self.pending_invalidation = None; + Poll::Ready(Err(wasmtime::Error::msg( + "filesystem write task dropped before reporting its effect", + ))) + } + }; + } + let Some(pending) = &mut self.pending_chunk else { + return Poll::Ready(Ok(None)); + }; + match Pin::new(&mut pending.result_rx).poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(Ok(result)) => { + self.pending_chunk = None; + Poll::Ready(Ok(Some(result))) + } + Poll::Ready(Err(_)) => { + self.pending_chunk = None; + self.chunks_tx.take(); + let runtime = self.filesystem_runtime.clone(); + self.pending_invalidation = Some(Box::pin(async move { + runtime.invalidate_runtime().await; + })); + self.poll_pending_result(cx) + } } } } @@ -80,31 +364,53 @@ impl StreamConsumer for FilesystemWriteConsumer { cx: &mut Context<'_>, store: StoreContextMut, src: Source, - _finish: bool, + finish: bool, ) -> Poll> { let mut src = src.as_direct(store); + if self.pending_invalidation.is_some() { + return match self.poll_pending_result(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Ready(Ok(_)) => unreachable!("pending invalidation disappeared"), + }; + } + + if finish { + self.cancel(); + if self.pending_chunk.is_none() { + return Poll::Ready(Ok(StreamResult::Cancelled)); + } + } + loop { // Wait for the in-flight chunk to be persisted before reading more. // The receiver must be polled here (not just stored) so its waker is // registered; otherwise the write task's completion notification // could be missed, hanging the stream. - if let Some((len, result_rx)) = &mut self.pending_chunk { - match Pin::new(result_rx).poll(cx) { + if self.pending_chunk.is_some() { + match self.poll_pending_result(cx) { Poll::Pending => return Poll::Pending, - Poll::Ready(Ok(Ok(()))) => { - let len = *len; - self.pending_chunk = None; - src.mark_read(len); - return Poll::Ready(Ok(StreamResult::Completed)); + Poll::Ready(Ok(Some((written, Ok(()))))) => { + src.mark_read(written); + return Poll::Ready(Ok(if finish { + StreamResult::Cancelled + } else { + StreamResult::Completed + })); } - Poll::Ready(Ok(Err(_))) | Poll::Ready(Err(_)) => { - let len = *len; - self.pending_chunk = None; + Poll::Ready(Ok(Some((written, Err(FilesystemWriteFailure::Guest(_)))))) => { self.chunks_tx.take(); - src.mark_read(len); + src.mark_read(written); return Poll::Ready(Ok(StreamResult::Dropped)); } + Poll::Ready(Ok(Some((written, Err(FilesystemWriteFailure::Trap(error)))))) => { + self.chunks_tx.take(); + src.mark_read(written); + return Poll::Ready(Err(wasmtime::Error::msg(error))); + } + Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), + Poll::Ready(Ok(None)) => unreachable!("pending result disappeared"), } } @@ -113,20 +419,25 @@ impl StreamConsumer for FilesystemWriteConsumer { return Poll::Ready(Ok(StreamResult::Completed)); } - let len = bytes.len(); let Some(chunks_tx) = &self.chunks_tx else { - src.mark_read(len); return Poll::Ready(Ok(StreamResult::Dropped)); }; let (result_tx, result_rx) = tokio::sync::oneshot::channel(); + let admission = self.filesystem_runtime.admit_effect()?; + let cancellation = tokio_util::sync::CancellationToken::new(); chunks_tx .send(FilesystemWriteChunk { contents: bytes.to_vec(), result_tx, + admission, + cancellation: cancellation.clone(), }) .map_err(|_| wasmtime::Error::msg("filesystem write task dropped"))?; - self.pending_chunk = Some((len, result_rx)); + self.pending_chunk = Some(PendingFilesystemWriteChunk { + result_rx, + cancellation, + }); // Loop back to poll the freshly created receiver and register its waker. } } @@ -134,7 +445,7 @@ impl StreamConsumer for FilesystemWriteConsumer { impl Drop for FilesystemWriteConsumer { fn drop(&mut self) { - self.chunks_tx.take(); + self.cancel(); } } @@ -149,6 +460,7 @@ struct FilesystemWriteTask { mode: FilesystemWriteMode, chunks_rx: tokio::sync::mpsc::UnboundedReceiver, result_tx: tokio::sync::oneshot::Sender>>, + filesystem_runtime: crate::services::agent_filesystem::AgentFilesystemRuntime, activity: TailActivity, _phantom: PhantomData Ctx>, } @@ -159,6 +471,7 @@ impl FilesystemWriteTask { mode: FilesystemWriteMode, chunks_rx: tokio::sync::mpsc::UnboundedReceiver, result_tx: tokio::sync::oneshot::Sender>>, + filesystem_runtime: crate::services::agent_filesystem::AgentFilesystemRuntime, activity: TailActivity, ) -> Self { Self { @@ -166,6 +479,7 @@ impl FilesystemWriteTask { mode, chunks_rx, result_tx, + filesystem_runtime, activity, _phantom: PhantomData, } @@ -177,20 +491,21 @@ where Ctx: WorkerCtx, U: 'static, { - async fn run(self, accessor: &Accessor>) -> wasmtime::Result<()> { + async fn run(self, _accessor: &Accessor>) -> wasmtime::Result<()> { let FilesystemWriteTask { file, mode, mut chunks_rx, result_tx, + filesystem_runtime, activity, _phantom, } = self; - let result = run_streaming_filesystem_write::( - accessor, + let result = run_streaming_filesystem_write( &file, mode, &mut chunks_rx, + &filesystem_runtime, &activity, ) .await; @@ -257,6 +572,31 @@ where } } +fn descriptor_from_access( + store: &mut Access<'_, U, DurableP3>, + fd: &Resource, +) -> wasmtime::Result +where + U: 'static, +{ + let mut filesystem = + Access::::new(store.as_context_mut(), wasi_filesystem_view::); + Ok(filesystem.get().table.get(fd)?.clone()) +} + +fn directory_from_access( + store: &mut Access<'_, U, DurableP3>, + fd: &Resource, +) -> FilesystemResult +where + U: 'static, +{ + match descriptor_from_access::(store, fd).map_err(FilesystemError::trap)? { + Descriptor::Dir(directory) => Ok(directory), + Descriptor::File(_) => Err(types::ErrorCode::NotDirectory.into()), + } +} + fn dir_result_from_access( store: &mut Access<'_, U, DurableP3>, fd: &Resource, @@ -295,6 +635,179 @@ where } } +fn fail_if_read_only_path_from_accessor( + accessor: &Accessor>, + fd: &Resource, + path: &str, + include_descendants: bool, + follow_final_symlink: bool, +) -> FilesystemResult<()> +where + Ctx: WorkerCtx, + U: 'static, +{ + let read_only = accessor + .with(|mut access| { + let ctx = durable_worker_ctx::(access.data_mut()); + let target = ctx.descriptor_path(fd)?.join(path); + Ok::<_, ResourceTableError>(if include_descendants { + ctx.filesystem_runtime + .contains_read_only_path(&target, follow_final_symlink) + } else { + ctx.filesystem_runtime + .is_read_only_path(&target, follow_final_symlink) + }) + }) + .map_err(|error| FilesystemError::trap(wasmtime::Error::from(error)))?; + if read_only { + Err(types::ErrorCode::NotPermitted.into()) + } else { + Ok(()) + } +} + +async fn begin_filesystem_effect( + accessor: &Accessor>, +) -> FilesystemResult +where + Ctx: WorkerCtx, + U: 'static, +{ + let runtime = accessor + .with(|mut access| durable_worker_ctx::(access.data_mut()).filesystem_runtime()); + runtime.begin_effect().await.map_err(FilesystemError::trap) +} + +async fn begin_filesystem_path_effect( + accessor: &Accessor>, +) -> FilesystemResult +where + Ctx: WorkerCtx, + U: 'static, +{ + let runtime = accessor + .with(|mut access| durable_worker_ctx::(access.data_mut()).filesystem_runtime()); + runtime + .begin_path_effect() + .await + .map_err(FilesystemError::trap) +} + +async fn begin_filesystem_update_effect( + accessor: &Accessor>, +) -> FilesystemResult +where + Ctx: WorkerCtx, + U: 'static, +{ + let runtime = accessor + .with(|mut access| durable_worker_ctx::(access.data_mut()).filesystem_runtime()); + runtime + .begin_update_effect() + .await + .map_err(FilesystemError::trap) +} + +async fn p3_initial_probe( + runtime: &AgentFilesystemRuntime, + result: Result, +) -> FilesystemResult { + match result { + Ok(value) => Ok(value), + Err(error) => { + let guest = types::ErrorCode::from(&error); + match runtime + .classify_mutation_failure_for( + MutationOperation::Metadata, + MutationFailure::::Io(error), + MutationEffect::ProvenNoEffect, + ) + .await + { + MutationDecision::Quota => Err(types::ErrorCode::Quota.into()), + MutationDecision::InsufficientSpace | MutationDecision::PhysicalPressure => { + Err(types::ErrorCode::InsufficientSpace.into()) + } + MutationDecision::BoundedRetry | MutationDecision::PreserveRaw => Err(guest.into()), + MutationDecision::PreserveGuest(error) => Err(error.into()), + MutationDecision::Success => unreachable!("failed probe cannot be satisfied"), + MutationDecision::Invalidate => Err(FilesystemError::trap(wasmtime::Error::msg( + "agent filesystem mutation precondition probe invalidated the runtime", + ))), + } + } + } +} + +async fn p3_finish_native_mutation( + adapter: &P3MutationAdapter, + error: std::io::Error, + postcondition: MutationPostcondition, + retry_safe: bool, +) -> FilesystemResult { + match adapter.io_failure(error, postcondition, retry_safe).await { + P3MutationAction::Retry => Ok(true), + action => p3_mutation_action_result(action).map(|()| false), + } +} + +fn p3_native_guest(error: NativeMutationGuestError) -> FilesystemError { + match error { + NativeMutationGuestError::Invalid => types::ErrorCode::Invalid.into(), + NativeMutationGuestError::NotDirectory => types::ErrorCode::NotDirectory.into(), + NativeMutationGuestError::NotPermitted => types::ErrorCode::NotPermitted.into(), + NativeMutationGuestError::Unsupported => types::ErrorCode::Unsupported.into(), + } +} + +fn p3_requested_time(requested: types::NewTimestamp) -> RequestedTime { + match requested { + types::NewTimestamp::NoChange => RequestedTime::NoChange, + types::NewTimestamp::Now => RequestedTime::Now, + types::NewTimestamp::Timestamp(timestamp) => RequestedTime::Timestamp { + seconds: i128::from(timestamp.seconds), + nanoseconds: timestamp.nanoseconds, + }, + } +} + +fn p3_native_time( + requested: types::NewTimestamp, +) -> Result, FilesystemError> { + match requested { + types::NewTimestamp::NoChange => Ok(None), + types::NewTimestamp::Now => Ok(Some(std::time::SystemTime::now())), + types::NewTimestamp::Timestamp(timestamp) => { + let time = if let Ok(seconds) = timestamp.seconds.try_into() { + std::time::SystemTime::UNIX_EPOCH + .checked_add(Duration::new(seconds, timestamp.nanoseconds)) + } else { + std::time::SystemTime::UNIX_EPOCH.checked_sub(Duration::new( + timestamp.seconds.unsigned_abs(), + timestamp.nanoseconds, + )) + }; + time.map(Some) + .ok_or_else(|| types::ErrorCode::Overflow.into()) + } + } +} + +fn push_descriptor( + accessor: &Accessor>, + descriptor: Descriptor, +) -> FilesystemResult> { + accessor + .with(|mut access| { + let mut filesystem = Access::::new( + access.as_context_mut(), + wasi_filesystem_view::, + ); + filesystem.get().table.push(descriptor) + }) + .map_err(FilesystemError::trap) +} + fn write_validation_error_from_access( store: &mut Access<'_, U, DurableP3>, fd: &Resource, @@ -425,53 +938,6 @@ where } } -/// Returns the size of the file referenced by `fd`, or `0` if it cannot be -/// stat-ed. Uses the underlying (non-durable) host stat so it produces no oplog -/// side effects; it is only used to compute storage-quota deltas, mirroring the -/// WASI P2 implementation. -async fn descriptor_size( - accessor: &Accessor>, - fd: Resource, -) -> u64 -where - Ctx: WorkerCtx, - U: Send + 'static, -{ - let filesystem = accessor.with_getter::(wasi_filesystem_view::); - match >::stat(&filesystem, fd).await { - Ok(stat) => stat.size, - Err(_) => 0, - } -} - -/// Returns the size of the file at `path` relative to `fd`, or `0` if it cannot -/// be stat-ed. Uses the underlying (non-durable) host stat so it produces no -/// oplog side effects; it is only used to compute storage-quota deltas, -/// mirroring the WASI P2 implementation. -async fn descriptor_size_at( - accessor: &Accessor>, - fd: Resource, - path_flags: types::PathFlags, - path: String, -) -> u64 -where - Ctx: WorkerCtx, - U: Send + 'static, -{ - let filesystem = accessor.with_getter::(wasi_filesystem_view::); - match >::stat_at( - &filesystem, - fd, - path_flags, - path, - ) - .await - { - Ok(stat) => stat.size, - Err(_) => 0, - } -} - /// Computes the metadata hash from a durable stat result, using the same hash /// inputs and function as the WASI P2 implementation so P2 and P3 report /// identical hashes for the same file state. P3 datetimes use signed seconds @@ -516,293 +982,348 @@ async fn wait_filesystem_task_result( .unwrap_or_else(|_| Err(wasmtime::Error::msg("filesystem stream task dropped"))) } -#[derive(Clone, Copy)] -struct FilesystemStorageReservation { - base_size: Option, - reserved_growth: u64, -} - -async fn filesystem_file_size(file: &File) -> Option { - let file = Arc::clone(&file.file); - spawn_blocking(move || file.metadata().map(|metadata| metadata.len()).ok()).await -} - -async fn reserve_filesystem_write_storage( - accessor: &Accessor>, +// Drains and writes the guest's data stream to the worker filesystem chunk by +// chunk, mirroring the WASI P2 behavior: the file effect is driven entirely by +// the input stream finishing or erroring, never by the liveness of the returned +// result future. The bytes themselves are not recorded in the oplog; on replay +// the guest re-issues the same writes which deterministically rebuild the +// transient worker filesystem. +async fn run_streaming_filesystem_write( file: &File, mode: FilesystemWriteMode, - write_len: u64, -) -> wasmtime::Result -where - Ctx: WorkerCtx, - U: 'static, -{ - let base_size = filesystem_file_size(file).await; - let reserved_growth = match (base_size, mode) { - (Some(current_size), FilesystemWriteMode::At(offset)) => offset - .saturating_add(write_len) - .saturating_sub(current_size), - (Some(current_size), FilesystemWriteMode::Append) => current_size - .saturating_add(write_len) - .saturating_sub(current_size), - (None, _) => write_len, + chunks_rx: &mut tokio::sync::mpsc::UnboundedReceiver, + filesystem_runtime: &crate::services::agent_filesystem::AgentFilesystemRuntime, + activity: &TailActivity, +) -> wasmtime::Result> { + let mut result: FilesystemWriteResult = Ok(()); + let mut position = match mode { + FilesystemWriteMode::At(offset) => Some(offset), + FilesystemWriteMode::Append => None, }; + // Safe park: each chunk is guest-produced stream data. + while let Some(chunk) = activity.park(chunks_rx.recv()).await { + let written_len = if result.is_ok() { + let effect = match position { + Some(_) => chunk.admission.begin().await?, + None => chunk.admission.begin_append().await?, + }; + let chunk_mode = match position { + Some(offset) => FilesystemWriteMode::At(offset), + None => FilesystemWriteMode::Append, + }; + let (written_len, mut write_result) = run_live_filesystem_write_chunk( + file.clone(), + filesystem_runtime, + chunk_mode, + chunk.contents, + effect, + chunk.cancellation, + ) + .await; + if let Some(offset) = &mut position { + let next_offset = u64::try_from(written_len) + .ok() + .and_then(|written_len| offset.checked_add(written_len)); + match next_offset { + Some(next_offset) => *offset = next_offset, + None if write_result.is_ok() => { + (_, write_result) = invariant_write_failure( + filesystem_runtime, + written_len, + "filesystem stream write offset overflowed", + ) + .await; + } + None => {} + } + } + result = write_result; + written_len + } else { + 0 + }; - reserve_filesystem_storage_bytes::(accessor, reserved_growth).await?; + let _ = chunk.result_tx.send((written_len, result.clone())); + } - Ok(FilesystemStorageReservation { - base_size, - reserved_growth, - }) + filesystem_write_result_to_wasi(result) } -/// Reserve `bytes` of filesystem storage quota: check the per-agent limit, -/// acquire executor-wide permits, and record the growth in the oplog. No-op for -/// `bytes == 0` and during replay (the helpers short-circuit). On acquisition -/// failure the optimistic reservation is rolled back and the error is returned. -async fn reserve_filesystem_storage_bytes( - accessor: &Accessor>, - bytes: u64, -) -> wasmtime::Result<()> -where - Ctx: WorkerCtx, - U: 'static, -{ - if bytes == 0 { - return Ok(()); - } - - if let Some(worker) = accessor - .with(|mut access| { - durable_worker_ctx::(access.data_mut()) - .prepare_filesystem_storage_reservation(bytes) - }) - .map_err(wasmtime::Error::from_anyhow)? - { - if let Err(error) = worker.acquire_filesystem_storage_space(bytes).await { - accessor.with(|mut access| { - durable_worker_ctx::(access.data_mut()) - .rollback_filesystem_storage_reservation(bytes); - }); - return Err(wasmtime::Error::from_anyhow(error)); - } - worker - .add_to_oplog(OplogEntry::filesystem_storage_usage_update(bytes as i64)) - .await; - accessor.with(|mut access| { - durable_worker_ctx::(access.data_mut()) - .finish_filesystem_storage_reservation(bytes); - }); - } - - Ok(()) +async fn run_live_filesystem_write_chunk( + file: File, + filesystem_runtime: &crate::services::agent_filesystem::AgentFilesystemRuntime, + mode: FilesystemWriteMode, + contents: Vec, + effect: crate::services::agent_filesystem::AgentFilesystemEffectLease, + cancellation: tokio_util::sync::CancellationToken, +) -> (usize, FilesystemWriteResult) { + run_classified_filesystem_write_chunk( + &LiveFilesystemChunkWriter { file }, + filesystem_runtime, + mode, + contents, + effect, + cancellation, + ) + .await } -async fn release_filesystem_write_storage( - accessor: &Accessor>, - bytes: u64, -) -> wasmtime::Result<()> -where - Ctx: WorkerCtx, - U: 'static, -{ - if bytes == 0 { - return Ok(()); - } - - if let Some((worker, bytes)) = accessor.with(|mut access| { - durable_worker_ctx::(access.data_mut()).prepare_filesystem_storage_release(bytes) - }) { - worker - .add_to_oplog(OplogEntry::filesystem_storage_usage_update(-(bytes as i64))) - .await; - worker.release_filesystem_storage_space(bytes).await; - accessor.with(|mut access| { - durable_worker_ctx::(access.data_mut()) - .finish_filesystem_storage_release(bytes); - }); - } - - Ok(()) +struct FilesystemWriteAttempt { + written: usize, + result: std::io::Result<()>, } -async fn reconcile_filesystem_write_storage( - accessor: &Accessor>, - file: &File, - reservation: FilesystemStorageReservation, - write_result: &Result<(), types::ErrorCode>, -) -> wasmtime::Result<()> -where - Ctx: WorkerCtx, - U: 'static, -{ - if reservation.reserved_growth == 0 { - return Ok(()); - } - - if write_result.is_err() { - return release_filesystem_write_storage::(accessor, reservation.reserved_growth) - .await; - } - - let Some(base_size) = reservation.base_size else { - return Ok(()); - }; - let Some(actual_end) = filesystem_file_size(file).await else { - return Ok(()); - }; - - let actual_growth = actual_end.saturating_sub(base_size); - let over_reserved = reservation.reserved_growth.saturating_sub(actual_growth); - release_filesystem_write_storage::(accessor, over_reserved).await +#[async_trait::async_trait] +trait FilesystemChunkWriter: Sync { + async fn write( + &self, + mode: FilesystemWriteMode, + contents: Bytes, + start: usize, + effect: Arc, + ) -> FilesystemWriteAttempt; } -// Drains and writes the guest's data stream to the worker filesystem chunk by -// chunk, mirroring the WASI P2 behavior: the file effect is driven entirely by -// the input stream finishing or erroring, never by the liveness of the returned -// result future. The bytes themselves are not recorded in the oplog; on replay -// the guest re-issues the same writes which deterministically rebuild the -// transient worker filesystem. Storage-quota deltas are reserved and reconciled -// per chunk (no-ops during replay). -async fn run_streaming_filesystem_write( - accessor: &Accessor>, - file: &File, - mode: FilesystemWriteMode, - chunks_rx: &mut tokio::sync::mpsc::UnboundedReceiver, - activity: &TailActivity, -) -> wasmtime::Result> -where - Ctx: WorkerCtx, - U: 'static, -{ - let mut result = Ok(()); - let mut position = match mode { - FilesystemWriteMode::At(offset) => Some(offset), - FilesystemWriteMode::Append => None, - }; +struct LiveFilesystemChunkWriter { + file: File, +} - // Safe park: each chunk is guest-produced stream data. - while let Some(chunk) = activity.park(chunks_rx.recv()).await { - if result.is_ok() { - let chunk_mode = match position { - Some(offset) => FilesystemWriteMode::At(offset), - None => FilesystemWriteMode::Append, +#[async_trait::async_trait] +impl FilesystemChunkWriter for LiveFilesystemChunkWriter { + async fn write( + &self, + mode: FilesystemWriteMode, + contents: Bytes, + start: usize, + effect: Arc, + ) -> FilesystemWriteAttempt { + let file = Arc::clone(&self.file.file); + spawn_blocking(move || { + let _effect = effect; + let suffix = &contents[start..]; + let result = match mode { + FilesystemWriteMode::At(offset) => file.write_at(suffix, offset), + FilesystemWriteMode::Append => { + let mut file = file.as_ref(); + file.seek(SeekFrom::End(0)).and_then(|_| file.write(suffix)) + } }; - let write_len = chunk.contents.len() as u64; - let reservation = - reserve_filesystem_write_storage::(accessor, file, chunk_mode, write_len) - .await?; - - let (written_len, write_result) = - run_live_filesystem_write_chunk(file.clone(), chunk_mode, chunk.contents).await; - reconcile_filesystem_write_storage::( - accessor, - file, - reservation, - &write_result, - ) - .await?; - if let Some(offset) = &mut position { - *offset = offset.saturating_add(written_len); + match result { + Ok(written) => FilesystemWriteAttempt { + written, + result: Ok(()), + }, + Err(error) => FilesystemWriteAttempt { + written: 0, + result: Err(error), + }, } - result = write_result; - } - - let _ = chunk.result_tx.send(result.clone()); + }) + .await } - - Ok(result) } -async fn run_live_filesystem_write_chunk( - file: File, +async fn run_classified_filesystem_write_chunk( + writer: &W, + filesystem_runtime: &crate::services::agent_filesystem::AgentFilesystemRuntime, mode: FilesystemWriteMode, contents: Vec, -) -> (u64, Result<(), types::ErrorCode>) { - let _append_guard = if matches!(mode, FilesystemWriteMode::Append) { - Some( - FILESYSTEM_APPEND_LOCK - .get_or_init(|| tokio::sync::Mutex::new(())) - .lock() - .await, - ) - } else { - None - }; - let file = Arc::clone(&file.file); - let (contents, written, result) = spawn_blocking(move || match mode { - FilesystemWriteMode::At(mut offset) => { - let mut written = 0; - while written < contents.len() { - match file.write_at(&contents[written..], offset) { - Ok(0) => { - return ( - contents, - written, - Err(std::io::Error::from(std::io::ErrorKind::WriteZero)), - ); + effect: crate::services::agent_filesystem::AgentFilesystemEffectLease, + cancellation: tokio_util::sync::CancellationToken, +) -> (usize, FilesystemWriteResult) { + let effect = Arc::new(effect); + let contents = Bytes::from(contents); + let started = Instant::now(); + let mut completed = 0usize; + let mut failed_attempts = 0; + + while completed < contents.len() { + if cancellation.is_cancelled() { + return (completed, Ok(())); + } + let attempt_mode = match mode { + FilesystemWriteMode::At(offset) => { + let completed_offset = match u64::try_from(completed) { + Ok(completed) => completed, + Err(_) => { + return invariant_write_failure( + filesystem_runtime, + completed, + "filesystem write length exceeds u64", + ) + .await; } - Ok(n) => { - written += n; - let n = match u64::try_from(n) { - Ok(n) => n, - Err(_) => { - return ( - contents, - written, - Err(std::io::Error::from(std::io::ErrorKind::InvalidData)), - ); - } - }; - offset = match offset.checked_add(n) { - Some(offset) => offset, - None => { - return ( - contents, - written, - Err(std::io::Error::from(std::io::ErrorKind::InvalidData)), - ); - } - }; + }; + match offset.checked_add(completed_offset) { + Some(offset) => FilesystemWriteMode::At(offset), + None => { + return invariant_write_failure( + filesystem_runtime, + completed, + "filesystem write offset overflowed", + ) + .await; } - Err(error) => return (contents, written, Err(error)), } } - (contents, written, Ok(())) + FilesystemWriteMode::Append => FilesystemWriteMode::Append, + }; + let attempt = writer + .write( + attempt_mode, + contents.clone(), + completed, + Arc::clone(&effect), + ) + .await; + let remaining = contents.len() - completed; + if attempt.written > remaining { + return invariant_write_failure( + filesystem_runtime, + completed, + "filesystem writer reported more bytes than requested", + ) + .await; } - FilesystemWriteMode::Append => { - let mut file = file.as_ref(); - if let Err(error) = file.seek(SeekFrom::End(0)) { - return (contents, 0, Err(error)); - } - let mut written = 0; - while written < contents.len() { - match file.write(&contents[written..]) { - Ok(0) => { - return ( - contents, - written, - Err(std::io::Error::from(std::io::ErrorKind::WriteZero)), - ); - } - Ok(n) => { - written += n; - } - Err(error) => return (contents, written, Err(error)), + completed += attempt.written; + + let (error, effect) = match attempt.result { + Ok(()) if attempt.written != 0 => { + if cancellation.is_cancelled() { + return (completed, Ok(())); } + continue; + } + Ok(()) => ( + std::io::Error::from(std::io::ErrorKind::WriteZero), + proven_write_progress_effect(completed), + ), + Err(error) => { + let effect = native_write_failure_effect(&error, completed); + (error, effect) + } + }; + failed_attempts += 1; + let raw_os_error = error.raw_os_error(); + let error_kind = error.kind(); + let error_message = error.to_string(); + let decision = filesystem_runtime + .classify_mutation_failure_for( + MutationOperation::Write, + crate::services::agent_filesystem::MutationFailure::::Io(error), + effect, + ) + .await; + + use crate::services::agent_filesystem::MutationDecision; + match decision { + MutationDecision::BoundedRetry if cancellation.is_cancelled() => { + return (completed, Ok(())); + } + MutationDecision::BoundedRetry + if failed_attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => {} + MutationDecision::BoundedRetry => { + let error = raw_os_error.map_or_else( + || std::io::Error::new(error_kind, error_message), + std::io::Error::from_raw_os_error, + ); + return (completed, Err(FilesystemWriteFailure::Guest(error.into()))); + } + MutationDecision::PreserveRaw => { + let error = raw_os_error.map_or_else( + || std::io::Error::new(error_kind, error_message), + std::io::Error::from_raw_os_error, + ); + return (completed, Err(FilesystemWriteFailure::Guest(error.into()))); + } + MutationDecision::PreserveGuest(error) => { + return (completed, Err(FilesystemWriteFailure::Guest(error))); + } + MutationDecision::Quota => { + return ( + completed, + Err(FilesystemWriteFailure::Guest(types::ErrorCode::Quota)), + ); + } + MutationDecision::InsufficientSpace => { + return ( + completed, + Err(FilesystemWriteFailure::Guest( + types::ErrorCode::InsufficientSpace, + )), + ); + } + MutationDecision::PhysicalPressure if cancellation.is_cancelled() => { + return (completed, Ok(())); + } + MutationDecision::PhysicalPressure + if failed_attempts < FILESYSTEM_MUTATION_MAX_ATTEMPTS + && started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT + && filesystem_runtime + .recover_physical_pressure( + MutationOperation::Write, + started + FILESYSTEM_MUTATION_RETRY_TIMEOUT, + ) + .await + && started.elapsed() <= FILESYSTEM_MUTATION_RETRY_TIMEOUT => {} + MutationDecision::PhysicalPressure => { + return ( + completed, + Err(FilesystemWriteFailure::Guest( + types::ErrorCode::InsufficientSpace, + )), + ); + } + MutationDecision::Success => return (completed, Ok(())), + MutationDecision::Invalidate => { + return (completed, Err(FilesystemWriteFailure::Trap(error_message))); } - (contents, written, Ok(())) } - }) - .await; + } + + (completed, Ok(())) +} - let written = written.min(contents.len()); +async fn invariant_write_failure( + filesystem_runtime: &crate::services::agent_filesystem::AgentFilesystemRuntime, + completed: usize, + message: &'static str, +) -> (usize, FilesystemWriteResult) { + let effect = u64::try_from(completed).map_or( + crate::services::agent_filesystem::MutationEffect::Unknown, + |bytes| { + if bytes == 0 { + crate::services::agent_filesystem::MutationEffect::ProvenNoEffect + } else { + crate::services::agent_filesystem::MutationEffect::KnownCompletedPrefix { bytes } + } + }, + ); + let _ = filesystem_runtime + .classify_mutation_failure::( + crate::services::agent_filesystem::MutationFailure::Infrastructure( + std::io::Error::new(std::io::ErrorKind::InvalidData, message), + ), + effect, + ) + .await; ( - written as u64, - result.map_err(|error: std::io::Error| error.into()), + completed, + Err(FilesystemWriteFailure::Trap(message.to_string())), ) } +fn filesystem_write_result_to_wasi( + result: FilesystemWriteResult, +) -> wasmtime::Result> { + match result { + Ok(()) => Ok(Ok(())), + Err(FilesystemWriteFailure::Guest(error)) => Ok(Err(error)), + Err(FilesystemWriteFailure::Trap(error)) => Err(wasmtime::Error::msg(error)), + } +} + impl types::Host for DurableP3View<'_, Ctx> { fn convert_error_code(&mut self, error: FilesystemError) -> wasmtime::Result { observe_function_call(&*self.0, "filesystem::types", "convert-error-code"); @@ -872,8 +1393,15 @@ impl types::HostDescriptorWithStore for Du let file = accessor.with(|mut store| file_from_access::(&mut store, &fd))?; let (chunks_tx, chunks_rx) = tokio::sync::mpsc::unbounded_channel(); - accessor - .with(|mut store| data.pipe(&mut store, FilesystemWriteConsumer::new(chunks_tx)))?; + let filesystem_runtime = accessor.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + accessor.with(|mut store| { + data.pipe( + &mut store, + FilesystemWriteConsumer::new(chunks_tx, filesystem_runtime.clone()), + ) + })?; let (result_tx, result_rx) = tokio::sync::oneshot::channel(); accessor.with(|mut store| { @@ -885,6 +1413,7 @@ impl types::HostDescriptorWithStore for Du FilesystemWriteMode::At(offset), chunks_rx, result_tx, + filesystem_runtime, activity, )); @@ -918,8 +1447,15 @@ impl types::HostDescriptorWithStore for Du let file = accessor.with(|mut store| file_from_access::(&mut store, &fd))?; let (chunks_tx, chunks_rx) = tokio::sync::mpsc::unbounded_channel(); - accessor - .with(|mut store| data.pipe(&mut store, FilesystemWriteConsumer::new(chunks_tx)))?; + let filesystem_runtime = accessor.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + accessor.with(|mut store| { + data.pipe( + &mut store, + FilesystemWriteConsumer::new(chunks_tx, filesystem_runtime.clone()), + ) + })?; let (result_tx, result_rx) = tokio::sync::oneshot::channel(); accessor.with(|mut store| { @@ -931,6 +1467,7 @@ impl types::HostDescriptorWithStore for Du FilesystemWriteMode::Append, chunks_rx, result_tx, + filesystem_runtime, activity, )); @@ -970,8 +1507,26 @@ impl types::HostDescriptorWithStore for Du "sync-data", ) }); - let store = store.with_getter::(wasi_filesystem_view::); - >::sync_data(&store, fd).await + let effect = Arc::new(begin_filesystem_effect::(store).await?); + let descriptor = store + .with(|mut access| descriptor_from_access::(&mut access, &fd)) + .map_err(FilesystemError::trap)?; + let mut adapter = P3MutationAdapter::new(store.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + })); + adapter.begin_attempt(); + match run_blocking_filesystem_mutation(effect, move || { + native_sync_descriptor(&descriptor, true) + }) + .await + { + Ok(()) => Ok(()), + Err(error) => p3_mutation_action_result( + adapter + .io_failure(error, MutationPostcondition::Unknown, false) + .await, + ), + } } async fn get_flags( @@ -1030,37 +1585,38 @@ impl types::HostDescriptorWithStore for Du "set-size", ) }); + let effect = Arc::new(begin_filesystem_update_effect::(accessor).await?); fail_if_read_only_from_accessor::(accessor, &fd)?; - // Charge growth before resizing and credit shrink afterwards, matching - // the WASI P2 storage-quota accounting. The quota helpers are no-ops - // during replay, so the storage usage is rebuilt purely from the oplog. - let current_size = - descriptor_size::(accessor, Resource::new_borrow(fd.rep())).await; - let growth = size.saturating_sub(current_size); - if growth > 0 { - reserve_filesystem_storage_bytes::(accessor, growth) - .await - .map_err(FilesystemError::trap)?; - } - - let result = { - let store = accessor.with_getter::(wasi_filesystem_view::); - >::set_size(&store, fd, size).await + let descriptor = accessor + .with(|mut access| descriptor_from_access::(&mut access, &fd)) + .map_err(FilesystemError::trap)?; + let file = match &descriptor { + Descriptor::File(file) => file.clone(), + Descriptor::Dir(_) => return Err(types::ErrorCode::BadDescriptor.into()), }; - - if growth > 0 { - if result.is_err() { - release_filesystem_write_storage::(accessor, growth) - .await - .map_err(FilesystemError::trap)?; - } - } else if result.is_ok() && size < current_size { - release_filesystem_write_storage::(accessor, current_size - size) + validate_resize(&file).map_err(p3_native_guest)?; + let runtime = accessor.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let before = p3_initial_probe(&runtime, descriptor_state(&descriptor).await).await?; + let mut adapter = P3MutationAdapter::for_operation(runtime, MutationOperation::Resize); + loop { + adapter.begin_attempt(); + let file = file.clone(); + let effect = Arc::clone(&effect); + match run_blocking_filesystem_mutation(effect, move || native_resize_file(&file, size)) .await - .map_err(FilesystemError::trap)?; + { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = + resize_postcondition(before, descriptor_state(&descriptor).await, size); + if !p3_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } } - - result } async fn set_times( @@ -1069,6 +1625,7 @@ impl types::HostDescriptorWithStore for Du data_access_timestamp: types::NewTimestamp, data_modification_timestamp: types::NewTimestamp, ) -> FilesystemResult<()> { + let effect = Arc::new(begin_filesystem_update_effect::(accessor).await?); fail_if_read_only_from_accessor::(accessor, &fd)?; accessor.with(|mut access| { observe_function_call_store::( @@ -1077,14 +1634,41 @@ impl types::HostDescriptorWithStore for Du "set-times", ) }); - let store = accessor.with_getter::(wasi_filesystem_view::); - >::set_times( - &store, - fd, - data_access_timestamp, - data_modification_timestamp, - ) - .await + let descriptor = accessor + .with(|mut access| descriptor_from_access::(&mut access, &fd)) + .map_err(FilesystemError::trap)?; + validate_descriptor_times(&descriptor).map_err(p3_native_guest)?; + let accessed = p3_native_time(data_access_timestamp)?; + let modified = p3_native_time(data_modification_timestamp)?; + let runtime = accessor.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let before = p3_initial_probe(&runtime, descriptor_times(&descriptor).await).await?; + let mut adapter = P3MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let descriptor_for_attempt = descriptor.clone(); + let effect = Arc::clone(&effect); + match run_blocking_filesystem_mutation(effect, move || { + native_set_descriptor_times(&descriptor_for_attempt, accessed, modified) + }) + .await + { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = times_postcondition( + descriptor_times(&descriptor).await, + before, + p3_requested_time(data_access_timestamp), + p3_requested_time(data_modification_timestamp), + false, + ); + if !p3_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn read_directory( @@ -1153,8 +1737,26 @@ impl types::HostDescriptorWithStore for Du "sync", ) }); - let store = store.with_getter::(wasi_filesystem_view::); - >::sync(&store, fd).await + let effect = Arc::new(begin_filesystem_effect::(store).await?); + let descriptor = store + .with(|mut access| descriptor_from_access::(&mut access, &fd)) + .map_err(FilesystemError::trap)?; + let mut adapter = P3MutationAdapter::new(store.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + })); + adapter.begin_attempt(); + match run_blocking_filesystem_mutation(effect, move || { + native_sync_descriptor(&descriptor, false) + }) + .await + { + Ok(()) => Ok(()), + Err(error) => p3_mutation_action_result( + adapter + .io_failure(error, MutationPostcondition::Unknown, false) + .await, + ), + } } async fn create_directory_at( @@ -1169,9 +1771,36 @@ impl types::HostDescriptorWithStore for Du "create-directory-at", ) }); - let store = store.with_getter::(wasi_filesystem_view::); - >::create_directory_at(&store, fd, path) + let effect = Arc::new(begin_filesystem_path_effect::(store).await?); + fail_if_read_only_path_from_accessor::(store, &fd, &path, false, false)?; + let directory = + store.with(|mut access| directory_from_access::(&mut access, &fd))?; + validate_directory_mutation(&directory).map_err(p3_native_guest)?; + let runtime = store.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let before = p3_initial_probe(&runtime, path_state(&directory, &path).await).await?; + let mut adapter = P3MutationAdapter::for_operation(runtime, MutationOperation::Create); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(&effect); + match run_blocking_filesystem_mutation(effect, move || { + native_create_directory(&directory_for_attempt, &path_for_attempt) + }) .await + { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = + create_directory_postcondition(before, path_state(&directory, &path).await); + if !p3_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn stat( @@ -1263,7 +1892,15 @@ impl types::HostDescriptorWithStore for Du data_access_timestamp: types::NewTimestamp, data_modification_timestamp: types::NewTimestamp, ) -> FilesystemResult<()> { + let effect = Arc::new(begin_filesystem_update_effect::(accessor).await?); fail_if_read_only_from_accessor::(accessor, &fd)?; + fail_if_read_only_path_from_accessor::( + accessor, + &fd, + &path, + false, + path_flags.contains(types::PathFlags::SYMLINK_FOLLOW), + )?; accessor.with(|mut access| { observe_function_call_store::( access.data_mut(), @@ -1271,16 +1908,49 @@ impl types::HostDescriptorWithStore for Du "set-times-at", ) }); - let store = accessor.with_getter::(wasi_filesystem_view::); - >::set_times_at( - &store, - fd, - path_flags, - path, - data_access_timestamp, - data_modification_timestamp, - ) - .await + let directory = + accessor.with(|mut access| directory_from_access::(&mut access, &fd))?; + validate_directory_mutation(&directory).map_err(p3_native_guest)?; + let follow = path_flags.contains(types::PathFlags::SYMLINK_FOLLOW); + let accessed = p3_native_time(data_access_timestamp)?; + let modified = p3_native_time(data_modification_timestamp)?; + let runtime = accessor.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let before = + p3_initial_probe(&runtime, path_times(&directory, &path, follow).await).await?; + let mut adapter = P3MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(&effect); + match run_blocking_filesystem_mutation(effect, move || { + native_set_path_times( + &directory_for_attempt, + &path_for_attempt, + follow, + accessed, + modified, + ) + }) + .await + { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = times_postcondition( + path_times(&directory, &path, follow).await, + before, + p3_requested_time(data_access_timestamp), + p3_requested_time(data_modification_timestamp), + true, + ); + if !p3_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn link_at( @@ -1298,16 +1968,68 @@ impl types::HostDescriptorWithStore for Du "link-at", ) }); - let store = store.with_getter::(wasi_filesystem_view::); - >::link_at( - &store, - fd, - old_path_flags, - old_path, - new_fd, - new_path, + let effect = Arc::new(begin_filesystem_path_effect::(store).await?); + fail_if_read_only_from_accessor::(store, &fd)?; + fail_if_read_only_from_accessor::(store, &new_fd)?; + fail_if_read_only_path_from_accessor::( + store, + &fd, + &old_path, + false, + old_path_flags.contains(types::PathFlags::SYMLINK_FOLLOW), + )?; + fail_if_read_only_path_from_accessor::(store, &new_fd, &new_path, false, false)?; + let source_directory = + store.with(|mut access| directory_from_access::(&mut access, &fd))?; + let destination_directory = + store.with(|mut access| directory_from_access::(&mut access, &new_fd))?; + validate_two_directory_mutation(&source_directory, &destination_directory) + .map_err(p3_native_guest)?; + if old_path_flags.contains(types::PathFlags::SYMLINK_FOLLOW) { + return Err(types::ErrorCode::Invalid.into()); + } + let runtime = store.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let source_before = + p3_initial_probe(&runtime, path_state(&source_directory, &old_path).await).await?; + let destination_before = p3_initial_probe( + &runtime, + path_state(&destination_directory, &new_path).await, ) - .await + .await?; + let mut adapter = P3MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let source_directory_for_attempt = source_directory.clone(); + let destination_directory_for_attempt = destination_directory.clone(); + let old_path_for_attempt = old_path.clone(); + let new_path_for_attempt = new_path.clone(); + let effect = Arc::clone(&effect); + match run_blocking_filesystem_mutation(effect, move || { + native_hard_link( + &source_directory_for_attempt, + &old_path_for_attempt, + &destination_directory_for_attempt, + &new_path_for_attempt, + ) + }) + .await + { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = link_postcondition( + source_before, + destination_before, + path_state(&source_directory, &old_path).await, + path_state(&destination_directory, &new_path).await, + ); + if !p3_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn open_at( @@ -1325,36 +2047,132 @@ impl types::HostDescriptorWithStore for Du "open-at", ) }); - // Opening with TRUNCATE discards the existing file contents, so credit - // the freed bytes back to the storage quota on success, matching WASI - // P2. The release helper is a no-op during replay. - let truncated_size = if open_flags.contains(types::OpenFlags::TRUNCATE) { - descriptor_size_at::( + let mutating = open_flags.intersects(types::OpenFlags::CREATE | types::OpenFlags::TRUNCATE); + let effect = if mutating { + Some(Arc::new( + begin_filesystem_update_effect::(accessor).await?, + )) + } else { + None + }; + if open_flags.contains(types::OpenFlags::TRUNCATE) + || flags.contains(types::DescriptorFlags::WRITE) + { + fail_if_read_only_path_from_accessor::( accessor, - Resource::new_borrow(fd.rep()), + &fd, + &path, + false, + path_flags.contains(types::PathFlags::SYMLINK_FOLLOW), + )?; + } + if !mutating { + let filesystem = accessor.with_getter::(wasi_filesystem_view::); + return >::open_at( + &filesystem, + fd, path_flags, - path.clone(), + path, + open_flags, + flags, ) - .await + .await; + } + let directory = + accessor.with(|mut access| directory_from_access::(&mut access, &fd))?; + let follow = path_flags.contains(types::PathFlags::SYMLINK_FOLLOW); + let native_options = NativeOpenOptions { + create: open_flags.contains(types::OpenFlags::CREATE), + directory: open_flags.contains(types::OpenFlags::DIRECTORY), + exclusive: open_flags.contains(types::OpenFlags::EXCLUSIVE), + truncate: open_flags.contains(types::OpenFlags::TRUNCATE), + follow, + read: flags.contains(types::DescriptorFlags::READ), + write: flags.contains(types::DescriptorFlags::WRITE), + }; + validate_open( + &directory, + native_options, + flags.intersects( + types::DescriptorFlags::FILE_INTEGRITY_SYNC + | types::DescriptorFlags::DATA_INTEGRITY_SYNC + | types::DescriptorFlags::REQUESTED_WRITE_SYNC, + ), + ) + .map_err(p3_native_guest)?; + let runtime = accessor.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let before = p3_initial_probe( + &runtime, + path_state_with_follow(&directory, &path, follow).await, + ) + .await?; + let requested_type = if open_flags.contains(types::OpenFlags::DIRECTORY) { + PathObjectType::Directory } else { - 0 + PathObjectType::RegularFile }; - - let result = { - let store = accessor.with_getter::(wasi_filesystem_view::); - >::open_at( - &store, fd, path_flags, path, open_flags, flags, - ) - .await + let operation = if open_flags.contains(types::OpenFlags::CREATE) { + MutationOperation::Create + } else { + MutationOperation::Resize }; - - if result.is_ok() && truncated_size > 0 { - release_filesystem_write_storage::(accessor, truncated_size) - .await - .map_err(FilesystemError::trap)?; + let mut adapter = P3MutationAdapter::for_operation(runtime, operation); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(effect.as_ref().expect("mutating open has an effect lease")); + match run_blocking_filesystem_mutation(effect, move || { + native_open(&directory_for_attempt, &path_for_attempt, native_options) + }) + .await + { + Ok(NativeOpenResult::Descriptor(descriptor)) => { + return push_descriptor(accessor, descriptor); + } + #[cfg(windows)] + Ok(NativeOpenResult::IsDirectory) => { + return Err(types::ErrorCode::IsDirectory.into()); + } + Ok(NativeOpenResult::NotDirectory) => { + return Err(p3_native_guest(NativeMutationGuestError::NotDirectory)); + } + Err(error) => { + let postcondition = open_postcondition( + before, + path_state_with_follow(&directory, &path, follow).await, + requested_type, + open_flags.contains(types::OpenFlags::TRUNCATE), + open_flags.contains(types::OpenFlags::EXCLUSIVE), + ); + match adapter.io_failure(error, postcondition, true).await { + P3MutationAction::Retry => {} + P3MutationAction::Success => { + let safe_flags = open_flags & types::OpenFlags::DIRECTORY; + let filesystem = accessor + .with_getter::(wasi_filesystem_view::); + return >::open_at( + &filesystem, + Resource::new_borrow(fd.rep()), + path_flags, + path.clone(), + safe_flags, + flags, + ) + .await; + } + P3MutationAction::Error(error) => return Err(error), + P3MutationAction::Trap => { + return Err(FilesystemError::trap(wasmtime::Error::msg( + "agent filesystem mutation invalidated the runtime", + ))); + } + } + } + } } - - result } async fn readlink_at( @@ -1385,9 +2203,36 @@ impl types::HostDescriptorWithStore for Du "remove-directory-at", ) }); - let store = store.with_getter::(wasi_filesystem_view::); - >::remove_directory_at(&store, fd, path) + let effect = Arc::new(begin_filesystem_path_effect::(store).await?); + fail_if_read_only_path_from_accessor::(store, &fd, &path, true, false)?; + let directory = + store.with(|mut access| directory_from_access::(&mut access, &fd))?; + validate_directory_mutation(&directory).map_err(p3_native_guest)?; + let runtime = store.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let before = p3_initial_probe(&runtime, path_state(&directory, &path).await).await?; + let mut adapter = P3MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(&effect); + match run_blocking_filesystem_mutation(effect, move || { + native_remove_directory(&directory_for_attempt, &path_for_attempt) + }) .await + { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = + remove_postcondition(before, path_state(&directory, &path).await); + if !p3_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn rename_at( @@ -1397,8 +2242,11 @@ impl types::HostDescriptorWithStore for Du new_fd: Resource, new_path: String, ) -> FilesystemResult<()> { + let effect = Arc::new(begin_filesystem_path_effect::(accessor).await?); fail_if_read_only_from_accessor::(accessor, &fd)?; fail_if_read_only_from_accessor::(accessor, &new_fd)?; + fail_if_read_only_path_from_accessor::(accessor, &fd, &old_path, true, false)?; + fail_if_read_only_path_from_accessor::(accessor, &new_fd, &new_path, true, false)?; accessor.with(|mut access| { observe_function_call_store::( access.data_mut(), @@ -1406,11 +2254,54 @@ impl types::HostDescriptorWithStore for Du "rename-at", ) }); - let store = accessor.with_getter::(wasi_filesystem_view::); - >::rename_at( - &store, fd, old_path, new_fd, new_path, + let source_directory = + accessor.with(|mut access| directory_from_access::(&mut access, &fd))?; + let destination_directory = + accessor.with(|mut access| directory_from_access::(&mut access, &new_fd))?; + validate_two_directory_mutation(&source_directory, &destination_directory) + .map_err(p3_native_guest)?; + let runtime = accessor.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let source_before = + p3_initial_probe(&runtime, path_state(&source_directory, &old_path).await).await?; + let destination_before = p3_initial_probe( + &runtime, + path_state(&destination_directory, &new_path).await, ) - .await + .await?; + let mut adapter = P3MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let source_directory_for_attempt = source_directory.clone(); + let destination_directory_for_attempt = destination_directory.clone(); + let old_path_for_attempt = old_path.clone(); + let new_path_for_attempt = new_path.clone(); + let effect = Arc::clone(&effect); + match run_blocking_filesystem_mutation(effect, move || { + native_rename( + &source_directory_for_attempt, + &old_path_for_attempt, + &destination_directory_for_attempt, + &new_path_for_attempt, + ) + }) + .await + { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = rename_postcondition( + source_before, + destination_before, + path_state(&source_directory, &old_path).await, + path_state(&destination_directory, &new_path).await, + ); + if !p3_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn symlink_at( @@ -1419,7 +2310,9 @@ impl types::HostDescriptorWithStore for Du old_path: String, new_path: String, ) -> FilesystemResult<()> { + let effect = Arc::new(begin_filesystem_path_effect::(accessor).await?); fail_if_read_only_from_accessor::(accessor, &fd)?; + fail_if_read_only_path_from_accessor::(accessor, &fd, &new_path, false, false)?; accessor.with(|mut access| { observe_function_call_store::( access.data_mut(), @@ -1427,11 +2320,42 @@ impl types::HostDescriptorWithStore for Du "symlink-at", ) }); - let store = accessor.with_getter::(wasi_filesystem_view::); - >::symlink_at( - &store, fd, old_path, new_path, - ) - .await + let directory = + accessor.with(|mut access| directory_from_access::(&mut access, &fd))?; + validate_directory_mutation(&directory).map_err(p3_native_guest)?; + let runtime = accessor.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let before = p3_initial_probe(&runtime, symlink_state(&directory, &new_path).await).await?; + let mut adapter = P3MutationAdapter::for_operation(runtime, MutationOperation::Create); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let old_path_for_attempt = old_path.clone(); + let new_path_for_attempt = new_path.clone(); + let effect = Arc::clone(&effect); + match run_blocking_filesystem_mutation(effect, move || { + native_symlink( + &directory_for_attempt, + &old_path_for_attempt, + &new_path_for_attempt, + ) + }) + .await + { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = symlink_postcondition( + &before, + symlink_state(&directory, &new_path).await, + &old_path, + ); + if !p3_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } + } } async fn unlink_file_at( @@ -1439,18 +2363,9 @@ impl types::HostDescriptorWithStore for Du fd: Resource, path: String, ) -> FilesystemResult<()> { + let effect = Arc::new(begin_filesystem_path_effect::(accessor).await?); fail_if_read_only_from_accessor::(accessor, &fd)?; - // Stat the file before unlinking so the freed bytes can be credited back - // to the storage quota on success, matching WASI P2. The release helper - // is a no-op during replay. - let file_size = descriptor_size_at::( - accessor, - Resource::new_borrow(fd.rep()), - types::PathFlags::empty(), - path.clone(), - ) - .await; - + fail_if_read_only_path_from_accessor::(accessor, &fd, &path, false, false)?; accessor.with(|mut access| { observe_function_call_store::( access.data_mut(), @@ -1458,19 +2373,34 @@ impl types::HostDescriptorWithStore for Du "unlink-file-at", ) }); - let result = { - let store = accessor.with_getter::(wasi_filesystem_view::); - >::unlink_file_at(&store, fd, path) - .await - }; - - if result.is_ok() && file_size > 0 { - release_filesystem_write_storage::(accessor, file_size) - .await - .map_err(FilesystemError::trap)?; + let directory = + accessor.with(|mut access| directory_from_access::(&mut access, &fd))?; + validate_directory_mutation(&directory).map_err(p3_native_guest)?; + let runtime = accessor.with(|mut access| { + durable_worker_ctx::(access.data_mut()).filesystem_runtime() + }); + let before = p3_initial_probe(&runtime, path_state(&directory, &path).await).await?; + let mut adapter = P3MutationAdapter::new(runtime); + loop { + adapter.begin_attempt(); + let directory_for_attempt = directory.clone(); + let path_for_attempt = path.clone(); + let effect = Arc::clone(&effect); + match run_blocking_filesystem_mutation(effect, move || { + native_unlink_file(&directory_for_attempt, &path_for_attempt) + }) + .await + { + Ok(()) => return Ok(()), + Err(error) => { + let postcondition = + remove_postcondition(before, path_state(&directory, &path).await); + if !p3_finish_native_mutation(&adapter, error, postcondition, true).await? { + return Ok(()); + } + } + } } - - result } async fn is_same_object( @@ -1530,11 +2460,82 @@ impl types::HostDescriptorWithStore for Du #[cfg(test)] mod tests { use super::*; + use crate::services::agent_filesystem::{FilesystemCapacity, ObjectIdentity, PathState}; use fs_set_times::{SystemTimeSpec, set_symlink_times, set_times}; use golem_common::model::oplog::types::SerializableDateTime; + use std::collections::VecDeque; + use std::sync::Mutex as StdMutex; use std::time::{Duration, SystemTime}; use test_r::test; + struct InjectedWriter { + attempts: StdMutex)>>, + suffixes: StdMutex>>, + started: Option>, + release: Option>, + } + + impl InjectedWriter { + fn new(attempts: impl IntoIterator)>) -> Self { + Self { + attempts: StdMutex::new(attempts.into_iter().collect()), + suffixes: StdMutex::new(Vec::new()), + started: None, + release: None, + } + } + + fn delayed_first( + attempts: impl IntoIterator)>, + started: Arc, + release: Arc, + ) -> Self { + Self { + attempts: StdMutex::new(attempts.into_iter().collect()), + suffixes: StdMutex::new(Vec::new()), + started: Some(started), + release: Some(release), + } + } + + fn suffixes(&self) -> Vec> { + self.suffixes.lock().unwrap().clone() + } + } + + #[async_trait::async_trait] + impl FilesystemChunkWriter for InjectedWriter { + async fn write( + &self, + _mode: FilesystemWriteMode, + contents: Bytes, + start: usize, + _effect: Arc, + ) -> FilesystemWriteAttempt { + let attempt_index = { + let mut suffixes = self.suffixes.lock().unwrap(); + let attempt_index = suffixes.len(); + suffixes.push(contents[start..].to_vec()); + attempt_index + }; + if attempt_index == 0 { + if let Some(started) = &self.started { + started.notify_one(); + } + if let Some(release) = &self.release { + release.acquire().await.unwrap().forget(); + } + } + let (written, errno) = self.attempts.lock().unwrap().pop_front().unwrap(); + FilesystemWriteAttempt { + written, + result: errno + .map(std::io::Error::from_raw_os_error) + .map_or(Ok(()), Err), + } + } + } + fn test_file(path: std::path::PathBuf) -> File { File::new( cap_std::fs::File::from_std(std::fs::File::create(&path).unwrap()), @@ -1545,6 +2546,389 @@ mod tests { ) } + #[cfg(target_os = "linux")] + #[test] + async fn p3_mutation_adapter_retries_only_proven_no_effect_once() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let mut adapter = P3MutationAdapter::new(runtime.clone()); + + adapter.begin_attempt(); + assert!(matches!( + adapter + .failure( + types::ErrorCode::Busy.into(), + MutationPostcondition::NoEffect, + true, + ) + .await, + P3MutationAction::Retry + )); + adapter.begin_attempt(); + assert!(matches!( + adapter + .failure( + types::ErrorCode::Busy.into(), + MutationPostcondition::NoEffect, + true, + ) + .await, + P3MutationAction::Error(error) + if matches!(error.downcast_ref(), Some(types::ErrorCode::Busy)) + )); + assert!(runtime.begin_effect().await.is_ok()); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_mutation_adapter_accepts_satisfied_postcondition() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let mut adapter = P3MutationAdapter::new(runtime.clone()); + adapter.begin_attempt(); + + assert!(matches!( + adapter + .failure( + types::ErrorCode::Interrupted.into(), + MutationPostcondition::Satisfied, + true, + ) + .await, + P3MutationAction::Success + )); + assert!(runtime.begin_effect().await.is_ok()); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_mutation_adapter_seals_unknown_effect() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let mut adapter = P3MutationAdapter::new(runtime.clone()); + adapter.begin_attempt(); + + assert!(matches!( + adapter + .failure( + types::ErrorCode::Busy.into(), + MutationPostcondition::Unknown, + true, + ) + .await, + P3MutationAction::Trap + )); + assert!(runtime.begin_effect().await.is_err()); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_native_eio_reaches_terminal_classifier() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let mut adapter = P3MutationAdapter::new(runtime.clone()); + adapter.begin_attempt(); + + assert!(matches!( + adapter + .io_failure( + std::io::Error::from_raw_os_error(libc::EIO), + MutationPostcondition::NoEffect, + true, + ) + .await, + P3MutationAction::Trap + )); + assert!(runtime.begin_effect().await.is_err()); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_physical_pressure_runs_one_safe_recovery_cycle() { + let runtime = AgentFilesystemRuntime::new_for_test_with_observations( + None, + None, + FilesystemCapacity { + total_bytes: 100, + available_bytes: 0, + total_filesystem_objects: 100, + available_filesystem_objects: 100, + }, + ); + let recovery_attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + runtime.set_pressure_recovery_callback(Some({ + let recovery_attempts = Arc::clone(&recovery_attempts); + Arc::new(move |operation, _deadline| { + let recovery_attempts = Arc::clone(&recovery_attempts); + Box::pin(async move { + assert_eq!(operation, MutationOperation::Metadata); + recovery_attempts.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + true + }) + }) + })); + let mut adapter = P3MutationAdapter::new(runtime); + adapter.begin_attempt(); + assert!(matches!( + adapter + .failure( + types::ErrorCode::InsufficientSpace.into(), + MutationPostcondition::NoEffect, + true, + ) + .await, + P3MutationAction::Retry + )); + adapter.begin_attempt(); + assert!(matches!( + adapter + .failure( + types::ErrorCode::InsufficientSpace.into(), + MutationPostcondition::NoEffect, + true, + ) + .await, + P3MutationAction::Error(error) + if matches!(error.downcast_ref(), Some(types::ErrorCode::InsufficientSpace)) + )); + assert_eq!( + recovery_attempts.load(std::sync::atomic::Ordering::Acquire), + 1 + ); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_physical_pressure_respects_effect_and_time_bounds() { + fn pressure_runtime() -> AgentFilesystemRuntime { + AgentFilesystemRuntime::new_for_test_with_observations( + None, + None, + FilesystemCapacity { + total_bytes: 100, + available_bytes: 0, + total_filesystem_objects: 100, + available_filesystem_objects: 100, + }, + ) + } + + let completed_recoveries = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let completed = pressure_runtime(); + completed.set_pressure_recovery_callback(Some({ + let completed_recoveries = Arc::clone(&completed_recoveries); + Arc::new(move |_, _deadline| { + let completed_recoveries = Arc::clone(&completed_recoveries); + Box::pin(async move { + completed_recoveries.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + true + }) + }) + })); + let mut completed_adapter = P3MutationAdapter::new(completed); + completed_adapter.begin_attempt(); + assert!(matches!( + completed_adapter + .failure( + types::ErrorCode::InsufficientSpace.into(), + MutationPostcondition::Satisfied, + true, + ) + .await, + P3MutationAction::Success + )); + assert_eq!( + completed_recoveries.load(std::sync::atomic::Ordering::Acquire), + 0 + ); + + let unknown_recoveries = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let unknown = pressure_runtime(); + unknown.set_pressure_recovery_callback(Some({ + let unknown_recoveries = Arc::clone(&unknown_recoveries); + Arc::new(move |_, _deadline| { + let unknown_recoveries = Arc::clone(&unknown_recoveries); + Box::pin(async move { + unknown_recoveries.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + true + }) + }) + })); + let mut unknown_adapter = P3MutationAdapter::new(unknown); + unknown_adapter.begin_attempt(); + assert!(matches!( + unknown_adapter + .failure( + types::ErrorCode::InsufficientSpace.into(), + MutationPostcondition::Unknown, + true, + ) + .await, + P3MutationAction::Trap + )); + assert_eq!( + unknown_recoveries.load(std::sync::atomic::Ordering::Acquire), + 0 + ); + + let timed_out = pressure_runtime(); + timed_out.set_pressure_recovery_callback(Some(Arc::new(move |_, _deadline| { + Box::pin(async move { + tokio::time::sleep(Duration::from_millis(260)).await; + true + }) + }))); + let mut timed_out_adapter = P3MutationAdapter::new(timed_out); + timed_out_adapter.begin_attempt(); + assert!(matches!( + timed_out_adapter + .failure( + types::ErrorCode::InsufficientSpace.into(), + MutationPostcondition::NoEffect, + true, + ) + .await, + P3MutationAction::Error(error) + if matches!(error.downcast_ref(), Some(types::ErrorCode::InsufficientSpace)) + )); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_shared_path_probe_distinguishes_unchanged_and_satisfied_state() { + let tempdir = tempfile::TempDir::new().unwrap(); + let directory = Dir::new( + cap_std::fs::Dir::open_ambient_dir(tempdir.path(), cap_std::ambient_authority()) + .unwrap(), + DirPerms::all(), + FilePerms::all(), + wasmtime_wasi::filesystem::OpenMode::READ | wasmtime_wasi::filesystem::OpenMode::WRITE, + false, + tempdir.path().to_path_buf(), + ); + let before = path_state(&directory, "entry").await.unwrap(); + assert_eq!(before, None); + + let unchanged = state_postcondition( + path_state(&directory, "entry").await, + |state| state.is_some(), + |state| state == before, + ); + assert_eq!(unchanged, MutationPostcondition::NoEffect); + + std::fs::create_dir(tempdir.path().join("entry")).unwrap(); + let satisfied = state_postcondition( + path_state(&directory, "entry").await, + |state| state.is_some_and(|state| state.type_ == PathObjectType::Directory), + |_| false, + ); + assert_eq!(satisfied, MutationPostcondition::Satisfied); + } + + #[test] + fn p3_shared_operation_postconditions_cover_non_prefix_effects() { + let source = PathState { + identity: Some(ObjectIdentity { + device: 1, + inode: 10, + }), + type_: PathObjectType::RegularFile, + size: 17, + }; + let replacement = PathState { + identity: Some(ObjectIdentity { + device: 1, + inode: 11, + }), + type_: PathObjectType::RegularFile, + size: 8, + }; + + let open_cases = [ + ( + None, + Ok(None), + false, + false, + MutationPostcondition::NoEffect, + ), + ( + Some(source), + Ok(Some(source)), + false, + false, + MutationPostcondition::Satisfied, + ), + ( + Some(source), + Ok(Some(replacement)), + false, + false, + MutationPostcondition::Satisfied, + ), + ( + Some(source), + Ok(Some(PathState { size: 0, ..source })), + true, + false, + MutationPostcondition::Satisfied, + ), + ]; + for (before, current, truncate, exclusive, expected) in open_cases { + assert_eq!( + open_postcondition( + before, + current, + PathObjectType::RegularFile, + truncate, + exclusive, + ), + expected + ); + } + + assert_eq!( + rename_postcondition(Some(source), Some(replacement), Ok(None), Ok(Some(source))), + MutationPostcondition::Satisfied + ); + assert_eq!( + rename_postcondition( + Some(source), + Some(replacement), + Ok(Some(source)), + Ok(Some(replacement)), + ), + MutationPostcondition::NoEffect + ); + assert_eq!( + link_postcondition(Some(source), None, Ok(Some(source)), Ok(Some(source))), + MutationPostcondition::Satisfied + ); + assert_eq!( + create_directory_postcondition(Some(source), Ok(Some(source))), + MutationPostcondition::NoEffect + ); + assert_eq!( + remove_postcondition(None, Ok(None)), + MutationPostcondition::NoEffect + ); + assert_eq!( + link_postcondition(None, Some(replacement), Ok(None), Ok(Some(replacement))), + MutationPostcondition::NoEffect + ); + assert_eq!( + rename_postcondition(None, Some(replacement), Ok(None), Ok(Some(replacement))), + MutationPostcondition::NoEffect + ); + let existing_symlink = crate::services::agent_filesystem::SymlinkState { + object: Some(PathState { + type_: PathObjectType::SymbolicLink, + ..source + }), + target: Some("existing".into()), + }; + assert_eq!( + symlink_postcondition(&existing_symlink, Ok(existing_symlink.clone()), "requested",), + MutationPostcondition::NoEffect + ); + } + #[test] fn metadata_hash_from_stat_matches_p2_hash() { let stat = types::DescriptorStat { @@ -1593,19 +2977,29 @@ mod tests { let tempdir = tempfile::TempDir::new().unwrap(); let path = tempdir.path().join("out"); let file = test_file(path.clone()); + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); let (written, result) = run_live_filesystem_write_chunk( file.clone(), + &runtime, FilesystemWriteMode::At(0), b"hello".to_vec(), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), ) .await; assert_eq!(written, 5); assert!(result.is_ok()); - let (written, result) = - run_live_filesystem_write_chunk(file, FilesystemWriteMode::At(5), b" world".to_vec()) - .await; + let (written, result) = run_live_filesystem_write_chunk( + file, + &runtime, + FilesystemWriteMode::At(5), + b" world".to_vec(), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; assert_eq!(written, 6); assert!(result.is_ok()); @@ -1617,12 +3011,19 @@ mod tests { let tempdir = tempfile::TempDir::new().unwrap(); let path = tempdir.path().join("out"); let file = test_file(path.clone()); + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); for chunk in [b"foo".to_vec(), b"bar".to_vec(), b"baz".to_vec()] { - let len = chunk.len() as u64; - let (written, result) = - run_live_filesystem_write_chunk(file.clone(), FilesystemWriteMode::Append, chunk) - .await; + let len = chunk.len(); + let (written, result) = run_live_filesystem_write_chunk( + file.clone(), + &runtime, + FilesystemWriteMode::Append, + chunk, + runtime.begin_append_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; assert_eq!(written, len); assert!(result.is_ok()); } @@ -1630,6 +3031,321 @@ mod tests { assert_eq!(std::fs::read(&path).unwrap(), b"foobarbaz"); } + #[cfg(target_os = "linux")] + #[test] + async fn p3_fs_stream_write_retries_transient_before_effect_once() { + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); + let writer = InjectedWriter::new([(0, Some(libc::EAGAIN)), (5, None)]); + + let (written, result) = run_classified_filesystem_write_chunk( + &writer, + &runtime, + FilesystemWriteMode::At(7), + b"hello".to_vec(), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 5); + assert!(result.is_ok()); + assert_eq!(writer.suffixes(), [b"hello".to_vec(), b"hello".to_vec()]); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_fs_stream_write_retries_only_partial_failure_suffix() { + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); + let writer = InjectedWriter::new([(2, Some(libc::EBUSY)), (3, None)]); + + let (written, result) = run_classified_filesystem_write_chunk( + &writer, + &runtime, + FilesystemWriteMode::At(11), + b"hello".to_vec(), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 5); + assert!(result.is_ok()); + assert_eq!(writer.suffixes(), [b"hello".to_vec(), b"llo".to_vec()]); + } + + #[test] + async fn p3_fs_stream_write_continues_after_short_success_with_remainder() { + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); + let writer = InjectedWriter::new([(2, None), (3, None)]); + + let (written, result) = run_classified_filesystem_write_chunk( + &writer, + &runtime, + FilesystemWriteMode::At(11), + b"hello".to_vec(), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 5); + assert!(result.is_ok()); + assert_eq!(writer.suffixes(), [b"hello".to_vec(), b"llo".to_vec()]); + } + + #[cfg(target_os = "linux")] + #[test] + async fn cancelling_p3_fs_stream_stops_retry_and_releases_effect_lease() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let writer = Arc::new(InjectedWriter::delayed_first( + [(0, Some(libc::EAGAIN)), (5, None)], + Arc::clone(&started), + Arc::clone(&release), + )); + let cancellation = tokio_util::sync::CancellationToken::new(); + let (chunks_tx, _chunks_rx) = tokio::sync::mpsc::unbounded_channel(); + let (_result_tx, result_rx) = tokio::sync::oneshot::channel(); + let mut consumer = FilesystemWriteConsumer { + chunks_tx: Some(chunks_tx), + pending_chunk: Some(PendingFilesystemWriteChunk { + result_rx, + cancellation: cancellation.clone(), + }), + pending_invalidation: None, + filesystem_runtime: runtime.clone(), + }; + let write = tokio::spawn({ + let runtime = runtime.clone(); + let writer = Arc::clone(&writer); + let cancellation = cancellation.clone(); + async move { + run_classified_filesystem_write_chunk( + writer.as_ref(), + &runtime, + FilesystemWriteMode::At(0), + b"hello".to_vec(), + runtime.begin_effect().await.unwrap(), + cancellation, + ) + .await + } + }); + started.notified().await; + + consumer.cancel(); + assert!(cancellation.is_cancelled()); + let update = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.begin_update_effect().await } + }); + tokio::task::yield_now().await; + assert!(!write.is_finished()); + assert!(!update.is_finished()); + + release.add_permits(1); + let (written, result) = write.await.unwrap(); + assert_eq!(written, 0); + assert!(result.is_ok()); + assert!(update.await.unwrap().is_ok()); + assert_eq!(writer.suffixes(), [b"hello".to_vec()]); + assert!(runtime.begin_effect().await.is_ok()); + } + + #[test] + async fn dropping_p3_fs_consumer_stops_suffix_and_releases_append_lease() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let writer = Arc::new(InjectedWriter::delayed_first( + [(2, None), (3, None)], + Arc::clone(&started), + Arc::clone(&release), + )); + let cancellation = tokio_util::sync::CancellationToken::new(); + let (chunks_tx, _chunks_rx) = tokio::sync::mpsc::unbounded_channel(); + let (_result_tx, result_rx) = tokio::sync::oneshot::channel(); + let consumer = FilesystemWriteConsumer { + chunks_tx: Some(chunks_tx), + pending_chunk: Some(PendingFilesystemWriteChunk { + result_rx, + cancellation: cancellation.clone(), + }), + pending_invalidation: None, + filesystem_runtime: runtime.clone(), + }; + let write = tokio::spawn({ + let runtime = runtime.clone(); + let writer = Arc::clone(&writer); + async move { + run_classified_filesystem_write_chunk( + writer.as_ref(), + &runtime, + FilesystemWriteMode::Append, + b"hello".to_vec(), + runtime.begin_append_effect().await.unwrap(), + cancellation, + ) + .await + } + }); + started.notified().await; + + drop(consumer); + let next_append = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.begin_append_effect().await } + }); + tokio::task::yield_now().await; + assert!(!write.is_finished()); + assert!(!next_append.is_finished()); + + release.add_permits(1); + let (written, result) = write.await.unwrap(); + assert_eq!(written, 2); + assert!(result.is_ok()); + assert!(next_append.await.unwrap().is_ok()); + assert_eq!(writer.suffixes(), [b"hello".to_vec()]); + assert!(runtime.begin_effect().await.is_ok()); + } + + #[test] + async fn dropped_p3_host_result_sender_invalidates_runtime() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let invalidated = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + runtime.set_invalidation_callback(Some(Arc::new({ + let invalidated = Arc::clone(&invalidated); + let release = Arc::clone(&release); + move || { + let invalidated = Arc::clone(&invalidated); + let release = Arc::clone(&release); + Box::pin(async move { + release.acquire().await.unwrap().forget(); + invalidated.store(true, std::sync::atomic::Ordering::Release); + }) + } + }))); + let (chunks_tx, _chunks_rx) = tokio::sync::mpsc::unbounded_channel(); + let (result_tx, result_rx) = tokio::sync::oneshot::channel(); + let mut consumer = FilesystemWriteConsumer { + chunks_tx: Some(chunks_tx), + pending_chunk: Some(PendingFilesystemWriteChunk { + result_rx, + cancellation: tokio_util::sync::CancellationToken::new(), + }), + pending_invalidation: None, + filesystem_runtime: runtime.clone(), + }; + drop(result_tx); + + let mut cx = Context::from_waker(std::task::Waker::noop()); + assert!(matches!( + consumer.poll_pending_result(&mut cx), + Poll::Pending + )); + assert!(runtime.begin_effect().await.is_err()); + assert!(!invalidated.load(std::sync::atomic::Ordering::Acquire)); + + release.add_permits(1); + let result = consumer.poll_pending_result(&mut cx); + + assert!(matches!(result, Poll::Ready(Err(_)))); + assert!(invalidated.load(std::sync::atomic::Ordering::Acquire)); + assert!(runtime.begin_effect().await.is_err()); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_fs_stream_write_returns_raw_mapping_after_retry_exhaustion() { + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); + let writer = InjectedWriter::new([(0, Some(libc::EBUSY)), (0, Some(libc::EBUSY))]); + + let (written, result) = run_classified_filesystem_write_chunk( + &writer, + &runtime, + FilesystemWriteMode::Append, + b"hello".to_vec(), + runtime.begin_append_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 0); + assert!(matches!( + result, + Err(FilesystemWriteFailure::Guest(types::ErrorCode::Busy)) + )); + assert_eq!(writer.suffixes(), [b"hello".to_vec(), b"hello".to_vec()]); + assert!(runtime.begin_effect().await.is_ok()); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_fs_stream_terminal_failure_traps_and_seals_runtime() { + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); + let writer = InjectedWriter::new([(2, Some(libc::EIO))]); + + let (written, result) = run_classified_filesystem_write_chunk( + &writer, + &runtime, + FilesystemWriteMode::At(0), + b"hello".to_vec(), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 2); + assert!(matches!(result, Err(FilesystemWriteFailure::Trap(_)))); + assert_eq!(writer.suffixes(), [b"hello".to_vec()]); + assert!(runtime.begin_effect().await.is_err()); + } + + #[cfg(target_os = "linux")] + #[test] + async fn p3_fs_stream_interruption_after_prefix_has_unknown_effect() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let writer = InjectedWriter::new([(2, Some(libc::EINTR)), (3, None)]); + + let (written, result) = run_classified_filesystem_write_chunk( + &writer, + &runtime, + FilesystemWriteMode::At(0), + b"hello".to_vec(), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 2); + assert!(matches!(result, Err(FilesystemWriteFailure::Trap(_)))); + assert_eq!(writer.suffixes(), [b"hello".to_vec()]); + assert!(runtime.begin_effect().await.is_err()); + } + + #[test] + async fn p3_fs_stream_offset_overflow_traps_after_preserving_prefix() { + let runtime = crate::services::agent_filesystem::AgentFilesystemRuntime::new_for_test(); + let writer = InjectedWriter::new([(1, None)]); + + let (written, result) = run_classified_filesystem_write_chunk( + &writer, + &runtime, + FilesystemWriteMode::At(u64::MAX), + b"hi".to_vec(), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 1); + assert!(matches!(result, Err(FilesystemWriteFailure::Trap(_)))); + assert_eq!(writer.suffixes(), [b"hi".to_vec()]); + assert!(runtime.begin_effect().await.is_err()); + } + #[cfg(unix)] #[test] async fn p3_fs_stat_at_follow_symlink_does_not_mutate_symlink_timestamps() { diff --git a/golem-worker-executor/src/grpc/mod.rs b/golem-worker-executor/src/grpc/mod.rs index eab184ca95..a0c27e1db4 100644 --- a/golem-worker-executor/src/grpc/mod.rs +++ b/golem-worker-executor/src/grpc/mod.rs @@ -384,12 +384,16 @@ impl + UsesAllDeps + Send + Sync + let fingerprint = worker.get_initial_worker_metadata().fingerprint; let mut subscription = self.events().subscribe(); - Worker::start_if_needed(worker.clone()).await?; - if worker.is_loading() { + let start_attempt = Worker::start_if_needed(worker.clone()).await?; + if let Some(start_attempt) = start_attempt { match subscription .wait_for(|event| match event { - Event::WorkerLoaded { agent_id, result } - if agent_id == &owned_agent_id.agent_id => + Event::WorkerLoaded { + agent_id, + start_attempt: event_attempt, + result, + } if agent_id == &owned_agent_id.agent_id + && event_attempt == &start_attempt => { Some(result.clone()) } diff --git a/golem-worker-executor/src/lib.rs b/golem-worker-executor/src/lib.rs index 66eb1c352c..2f0de4e461 100644 --- a/golem-worker-executor/src/lib.rs +++ b/golem-worker-executor/src/lib.rs @@ -176,13 +176,13 @@ pub trait Bootstrap { &self, golem_config: &GolemConfig, shutdown_token: tokio_util::sync::CancellationToken, - ) -> Arc> { - Arc::new(ActiveWorkers::::new( + ) -> anyhow::Result>> { + Ok(Arc::new(ActiveWorkers::::new( &golem_config.memory, &golem_config.filesystem_storage, &golem_config.agent_status_flush, shutdown_token, - )) + )?)) } fn create_shard_manager_service( @@ -798,11 +798,15 @@ pub async fn create_worker_executor_impl< } }; - let active_workers = bootstrap.create_active_workers(&golem_config, shutdown_token.clone()); + let active_workers = bootstrap.create_active_workers(&golem_config, shutdown_token.clone())?; + let initial_file_cache_root = active_workers + .agent_filesystems() + .initial_file_cache_root() + .map(|path| path.to_path_buf()); let file_loader = Arc::new(FileLoader::new( initial_files_service.clone(), - Some(active_workers.filesystem_storage_semaphore()), + initial_file_cache_root.as_deref(), )?); let running_worker_enumeration_service = Arc::new(RunningWorkerEnumerationServiceDefault::new( diff --git a/golem-worker-executor/src/metrics.rs b/golem-worker-executor/src/metrics.rs index 8c48f7d07b..58ae4efb37 100644 --- a/golem-worker-executor/src/metrics.rs +++ b/golem-worker-executor/src/metrics.rs @@ -339,7 +339,7 @@ pub mod workers { .unwrap(); pub static ref WORKER_ADMISSION_WAIT_SECONDS: HistogramVec = register_histogram_vec!( "worker_admission_wait_seconds", - "Time a starting worker spent blocked in one phase of admission before it could become resident, labelled by phase (resolve_component_charge, concurrency_slot, memory, filesystem_storage). Observed once per phase per worker start. Sum across phases for the total wait; read per phase to see which resource is the constraint. worker_waiting_for_memory_count says how many workers are waiting, this says for how long", + "Time a starting worker spent blocked in one phase of admission before it could become resident, labelled by phase (resolve_component_charge, concurrency_slot, memory). Observed once per phase per worker start. Sum across phases for the total wait; read per phase to see which resource is the constraint. worker_waiting_for_memory_count says how many workers are waiting, this says for how long", &["executor_id", "phase"], crate::metrics::ADMISSION_WAIT_BUCKETS.to_vec() ) @@ -417,6 +417,13 @@ pub mod workers { &["reason"] ) .unwrap(); + static ref AGENT_FILESYSTEM_LIFECYCLE_SECONDS: HistogramVec = register_histogram_vec!( + "golem_agent_filesystem_lifecycle_seconds", + "Time spent creating or deleting an agent runtime filesystem, labelled by operation and outcome", + &["operation", "outcome"], + golem_common::metrics::DEFAULT_TIME_BUCKETS.to_vec() + ) + .unwrap(); } pub fn record_worker_call(api_name: &'static str) { @@ -480,6 +487,16 @@ pub mod workers { .inc(); } + pub fn record_agent_filesystem_lifecycle( + operation: &'static str, + success: bool, + elapsed: Duration, + ) { + AGENT_FILESYSTEM_LIFECYCLE_SECONDS + .with_label_values(&[operation, if success { "success" } else { "failure" }]) + .observe(elapsed.as_secs_f64()); + } + pub fn set_worker_count_by_status(status: &'static str, count: f64) { WORKER_COUNT_BY_STATUS .with_label_values(&[status]) @@ -554,7 +571,6 @@ pub mod workers { ResolveComponentCharge, ConcurrencySlot, Memory, - FilesystemStorage, } impl AdmissionPhase { @@ -563,7 +579,6 @@ pub mod workers { AdmissionPhase::ResolveComponentCharge => "resolve_component_charge", AdmissionPhase::ConcurrencySlot => "concurrency_slot", AdmissionPhase::Memory => "memory", - AdmissionPhase::FilesystemStorage => "filesystem_storage", } } } diff --git a/golem-worker-executor/src/model/mod.rs b/golem-worker-executor/src/model/mod.rs index a9374bfa1b..bb2714aac5 100644 --- a/golem-worker-executor/src/model/mod.rs +++ b/golem-worker-executor/src/model/mod.rs @@ -69,7 +69,6 @@ impl ShardAssignmentCheck for ShardAssignment { pub struct AgentConfig { pub deleted_regions: DeletedRegions, pub total_linear_memory_size: u64, - pub current_filesystem_storage_usage: u64, pub component_revision_for_replay: ComponentRevision, pub created_by: AccountId, pub created_by_email: AccountEmail, @@ -82,7 +81,6 @@ impl AgentConfig { pub fn new( deleted_regions: DeletedRegions, total_linear_memory_size: u64, - current_filesystem_storage_usage: u64, component_revision_for_replay: ComponentRevision, created_by: AccountId, created_by_email: AccountEmail, @@ -93,7 +91,6 @@ impl AgentConfig { AgentConfig { deleted_regions, total_linear_memory_size, - current_filesystem_storage_usage, component_revision_for_replay, created_by, created_by_email, @@ -367,12 +364,6 @@ impl TrapType { Some(GolemSpecificWasmTrap::WorkerExceededRpcCallLimit) => { make_error(AgentError::ExceededRpcCallLimit) } - Some(GolemSpecificWasmTrap::NodeOutOfFilesystemStorage) => { - make_error(AgentError::NodeOutOfFilesystemStorage) - } - Some(GolemSpecificWasmTrap::WorkerAgentExceededFilesystemStorageLimit) => { - make_error(AgentError::AgentExceededFilesystemStorageLimit) - } Some(GolemSpecificWasmTrap::WorkerMonthlyHttpCallBudgetExhausted) => { match agent_mode { AgentMode::Durable => TrapType::Interrupt(InterruptKind::Suspend( diff --git a/golem-worker-executor/src/model/public_oplog/mod.rs b/golem-worker-executor/src/model/public_oplog/mod.rs index cf6c10cdff..82e9ef6d62 100644 --- a/golem-worker-executor/src/model/public_oplog/mod.rs +++ b/golem-worker-executor/src/model/public_oplog/mod.rs @@ -32,13 +32,12 @@ use golem_common::model::oplog::public_oplog_entry::{ CardInstalledParams, CardRevokedParams, CommittedRemoteTransactionParams, CompletionDiscardedParams, CreateParams, CreateResourceParams, DeactivatePluginParams, DropResourceParams, EndAtomicRegionParams, EndParams, ErrorParams, ExitedParams, - FailedUpdateParams, FilesystemStorageUsageUpdateParams, FinishSpanParams, GrowMemoryParams, - HostStreamFrameParams, InterruptedParams, JumpParams, LogParams, NoOpParams, - OplogProcessorCheckpointParams, PendingAgentInvocationParams, PendingUpdateParams, - PreCommitRemoteTransactionParams, PreRollbackRemoteTransactionParams, RemoveRetryPolicyParams, - RestartParams, RevertParams, RolledBackRemoteTransactionParams, SetRetryPolicyParams, - SetSpanAttributeParams, SnapshotParams, StartParams, StartSpanParams, SuccessfulUpdateParams, - SuspendParams, + FailedUpdateParams, FinishSpanParams, GrowMemoryParams, HostStreamFrameParams, + InterruptedParams, JumpParams, LogParams, NoOpParams, OplogProcessorCheckpointParams, + PendingAgentInvocationParams, PendingUpdateParams, PreCommitRemoteTransactionParams, + PreRollbackRemoteTransactionParams, RemoveRetryPolicyParams, RestartParams, RevertParams, + RolledBackRemoteTransactionParams, SetRetryPolicyParams, SetSpanAttributeParams, + SnapshotParams, StartParams, StartSpanParams, SuccessfulUpdateParams, SuspendParams, }; use golem_common::model::oplog::types::encode_span_data; use golem_common::model::oplog::{ @@ -630,11 +629,6 @@ impl PublicOplogEntryOps for PublicOplogEntry { delta, })) } - OplogEntry::FilesystemStorageUsageUpdate { timestamp, delta } => { - Ok(PublicOplogEntry::FilesystemStorageUsageUpdate( - FilesystemStorageUsageUpdateParams { timestamp, delta }, - )) - } OplogEntry::CreateResource { timestamp, id, diff --git a/golem-worker-executor/src/model/public_oplog/wit.rs b/golem-worker-executor/src/model/public_oplog/wit.rs index b2827b9640..9c7b97dc13 100644 --- a/golem-worker-executor/src/model/public_oplog/wit.rs +++ b/golem-worker-executor/src/model/public_oplog/wit.rs @@ -23,9 +23,9 @@ use golem_common::model::oplog::public_oplog_entry::{ CardInstalledParams, CardRevokedParams, CommittedRemoteTransactionParams, CompletionDiscardedParams, CreateParams, CreateResourceParams, DeactivatePluginParams, DropResourceParams, EndAtomicRegionParams, EndParams, ErrorParams, ExitedParams, - FailedUpdateParams, FilesystemStorageUsageUpdateParams, FinishSpanParams, GrowMemoryParams, - HostStreamFrameParams, InterruptedParams, JumpParams, LogParams, ManualUpdateParameters, - NoOpParams, OplogProcessorCheckpointParams, PendingAgentInvocationParams, PendingUpdateParams, + FailedUpdateParams, FinishSpanParams, GrowMemoryParams, HostStreamFrameParams, + InterruptedParams, JumpParams, LogParams, ManualUpdateParameters, NoOpParams, + OplogProcessorCheckpointParams, PendingAgentInvocationParams, PendingUpdateParams, PluginInstallationDescription, PreCommitRemoteTransactionParams, PreRollbackRemoteTransactionParams, PublicAgentInvocation, PublicAgentInvocationResult, PublicAttributeValue, PublicDurableFunctionType, PublicSpanData, RemoveRetryPolicyParams, @@ -385,14 +385,6 @@ impl TryFrom for oplog::PublicOplogEntry { delta, }) } - PublicOplogEntry::FilesystemStorageUsageUpdate( - FilesystemStorageUsageUpdateParams { timestamp, delta }, - ) => { - Self::FilesystemStorageUsageUpdate(oplog::FilesystemStorageUsageUpdateParameters { - timestamp: timestamp.into(), - delta, - }) - } PublicOplogEntry::CreateResource(CreateResourceParams { timestamp, id, @@ -921,10 +913,6 @@ impl From for golem_common::model::oplog::AgentError { oplog::WorkerError::ExceededTableLimit => Self::ExceededTableLimit, oplog::WorkerError::ExceededHttpCallLimit => Self::ExceededHttpCallLimit, oplog::WorkerError::ExceededRpcCallLimit => Self::ExceededRpcCallLimit, - oplog::WorkerError::NodeOutOfFilesystemStorage => Self::NodeOutOfFilesystemStorage, - oplog::WorkerError::AgentExceededFilesystemStorageLimit => { - Self::AgentExceededFilesystemStorageLimit - } oplog::WorkerError::AgentTerminatedByQuota(inner) => { Self::AgentTerminatedByQuota(AgentTerminatedByQuotaError { environment_id: EnvironmentId(inner.environment_id.uuid.into()), @@ -1234,10 +1222,6 @@ impl TryFrom for golem_common::model::oplog::OplogEntry { timestamp: timestamp_from_datetime(params.timestamp), delta: params.delta, }), - oplog::OplogEntry::FilesystemStorageUsageUpdate(params) => Ok(Self::FilesystemStorageUsageUpdate { - timestamp: timestamp_from_datetime(params.timestamp), - delta: params.delta, - }), oplog::OplogEntry::CreateResource(params) => Ok(Self::CreateResource { timestamp: timestamp_from_datetime(params.timestamp), id: golem_common::model::oplog::AgentResourceId(params.id), @@ -1631,10 +1615,6 @@ impl From for oplog::WorkerError { AgentError::ExceededTableLimit => Self::ExceededTableLimit, AgentError::ExceededHttpCallLimit => Self::ExceededHttpCallLimit, AgentError::ExceededRpcCallLimit => Self::ExceededRpcCallLimit, - AgentError::NodeOutOfFilesystemStorage => Self::NodeOutOfFilesystemStorage, - AgentError::AgentExceededFilesystemStorageLimit => { - Self::AgentExceededFilesystemStorageLimit - } AgentError::AgentTerminatedByQuota(inner) => { Self::AgentTerminatedByQuota(oplog::AgentTerminatedByQuotaError { environment_id: inner.environment_id.into(), @@ -1965,12 +1945,6 @@ impl TryFrom for oplog::OplogEntry { delta, })) } - M::FilesystemStorageUsageUpdate { timestamp, delta } => Ok( - Self::FilesystemStorageUsageUpdate(oplog::FilesystemStorageUsageUpdateParameters { - timestamp: timestamp.into(), - delta, - }), - ), M::CardRevoked { timestamp, queued_event_index, diff --git a/golem-worker-executor/src/services/active_workers/concurrent_agents_semaphore.rs b/golem-worker-executor/src/services/active_workers/concurrent_agents_semaphore.rs index 91f23008ff..9f8a8c70da 100644 --- a/golem-worker-executor/src/services/active_workers/concurrent_agents_semaphore.rs +++ b/golem-worker-executor/src/services/active_workers/concurrent_agents_semaphore.rs @@ -31,8 +31,7 @@ use tracing::debug; /// invocation loop) or when the agent is stopped. /// /// Extracted as a standalone struct (no `WorkerCtx` generic) so it can be -/// unit-tested in isolation, following the same pattern as -/// `FilesystemStorageSemaphore`. +/// unit-tested in isolation. /// /// ## Unlimited accounts /// diff --git a/golem-worker-executor/src/services/active_workers/fs_semaphore.rs b/golem-worker-executor/src/services/active_workers/fs_semaphore.rs deleted file mode 100644 index 9fe107df86..0000000000 --- a/golem-worker-executor/src/services/active_workers/fs_semaphore.rs +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2024-2026 Golem Cloud -// -// Licensed under the Golem Source License v1.1 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://license.golem.cloud/LICENSE -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::metrics::storage::{record_filesystem_pool_acquired, record_filesystem_pool_total}; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::{Mutex, OwnedSemaphorePermit, Semaphore, TryAcquireError}; -use tracing::debug; - -/// Executor-wide storage semaphore. One permit = `FILESYSTEM_STORAGE_PERMIT_SIZE_KB` KB. -/// -/// Extracted as a standalone struct so it can be unit-tested independently of -/// the `WorkerCtx`-generic `ActiveWorkers`. -pub struct FilesystemStorageSemaphore { - semaphore: Arc, - /// Held during non-blocking priority acquires to interrupt any in-progress - /// blocking `acquire` loops, preventing starvation of high-priority callers. - priority_lock: Arc>, - acquire_retry_delay: Duration, -} - -#[derive(Debug)] -pub struct FilesystemStoragePermit { - permit: Option, -} - -impl FilesystemStoragePermit { - fn new(permit: OwnedSemaphorePermit) -> Self { - let permits = permit.num_permits() as u32; - crate::metrics::workers::dec_filesystem_semaphore_available( - filesystem_storage_permits_to_bytes(permits), - ); - Self { - permit: Some(permit), - } - } - - pub fn num_permits(&self) -> usize { - self.permit - .as_ref() - .map_or(0, |permit| permit.num_permits()) - } - - pub fn split(&mut self, n: usize) -> Option { - self.permit - .as_mut() - .and_then(|permit| permit.split(n)) - .map(|permit| Self { - permit: Some(permit), - }) - } - - pub fn merge(&mut self, mut other: Self) { - if let Some(other_permit) = other.permit.take() { - match &mut self.permit { - Some(permit) => permit.merge(other_permit), - None => self.permit = Some(other_permit), - } - } - } -} - -impl Drop for FilesystemStoragePermit { - fn drop(&mut self) { - let permits = self.num_permits() as u32; - crate::metrics::workers::inc_filesystem_semaphore_available( - filesystem_storage_permits_to_bytes(permits), - ); - } -} - -impl FilesystemStorageSemaphore { - pub(crate) fn new(pool_bytes: usize, acquire_retry_delay: Duration) -> Self { - let permits = filesystem_storage_pool_bytes_to_permits(pool_bytes); - record_filesystem_pool_total(pool_bytes as u64); - crate::metrics::workers::set_filesystem_semaphore_available( - filesystem_storage_permits_to_bytes(permits as u32), - ); - Self { - semaphore: Arc::new(Semaphore::new(permits)), - priority_lock: Arc::new(Mutex::new(())), - acquire_retry_delay, - } - } - - /// Available bytes remaining in the pool (rounded down to KB boundary). - pub(crate) fn available_bytes(&self) -> u64 { - filesystem_storage_permits_to_bytes(self.semaphore.available_permits() as u32) - } - - /// Expose the inner semaphore for tests that need to simulate external - /// permit changes (e.g. worker eviction releasing storage). - #[cfg(test)] - pub(crate) fn inner_semaphore(&self) -> &Arc { - &self.semaphore - } - - /// Blocking acquire. Loops until `storage_bytes` are available, calling - /// `try_free_up` each time permits are exhausted. If `try_free_up` returns - /// `false` (nothing to evict), sleeps `acquire_retry_delay` before retrying. - pub(crate) async fn acquire( - &self, - storage_bytes: u64, - try_free_up: F, - ) -> FilesystemStoragePermit - where - F: Fn() -> Fut, - Fut: std::future::Future, - { - let permits = bytes_to_filesystem_storage_permits(storage_bytes); - loop { - let available = self.semaphore.available_permits(); - let lock = self.priority_lock.lock().await; - let result = self.semaphore.clone().try_acquire_many_owned(permits); - drop(lock); - match result { - Ok(permit) => { - debug!( - "Acquired {} storage permits ({} bytes) of {}, new available: {}, permit size: {}", - permits, - storage_bytes, - available, - self.semaphore.available_permits(), - permit.num_permits() - ); - let actual_bytes = filesystem_storage_permits_to_bytes(permits); - record_filesystem_pool_acquired(actual_bytes); - break FilesystemStoragePermit::new(permit); - } - Err(TryAcquireError::Closed) => panic!("worker storage semaphore has been closed"), - Err(TryAcquireError::NoPermits) => { - debug!( - "Not enough storage to allocate {} permits (available: {}), trying to free some up", - permits, - self.semaphore.available_permits() - ); - if try_free_up().await { - debug!("Freed up some storage, retrying"); - continue; - } else { - debug!("Could not free up storage, retrying after some time"); - tokio::time::sleep(self.acquire_retry_delay).await; - } - } - } - } - } - - /// Non-blocking priority acquire. Grabs the priority lock to interrupt any - /// in-progress blocking `acquire` loops, then attempts once. - /// - /// Returns `None` if `storage_bytes` are not available even after - /// interrupting waiting acquires. - pub(crate) async fn try_acquire(&self, storage_bytes: u64) -> Option { - let permits = bytes_to_filesystem_storage_permits(storage_bytes); - let mut lock = None; - loop { - match self.semaphore.clone().try_acquire_many_owned(permits) { - Ok(permit) => { - debug!( - "Acquired {} storage permits ({} bytes), available now: {}", - permits, - storage_bytes, - self.semaphore.available_permits() - ); - let actual_bytes = filesystem_storage_permits_to_bytes(permits); - record_filesystem_pool_acquired(actual_bytes); - break Some(FilesystemStoragePermit::new(permit)); - } - Err(TryAcquireError::Closed) => panic!("worker storage semaphore has been closed"), - Err(TryAcquireError::NoPermits) => { - if lock.is_none() { - debug!( - "Not enough storage to acquire {} permits (available: {}), cancelling waiting acquires and retry", - permits, - self.semaphore.available_permits() - ); - lock = Some(self.priority_lock.lock().await); - continue; - } else { - debug!( - "Not enough storage to acquire {} permits (available: {})", - permits, - self.semaphore.available_permits() - ); - break None; - } - } - } - } - } -} - -/// One storage semaphore permit represents this many kilobytes. Using KB units -/// keeps the permit count within `u32` range while supporting up to ~4 TB of -/// addressable storage space (4_294_967_295 KB ≈ 4 TB). -pub const FILESYSTEM_STORAGE_PERMIT_SIZE_KB: u64 = 1; - -/// Convert a byte count to the number of storage semaphore permits needed, -/// rounding up so that partial kilobytes always consume a full permit. -pub fn bytes_to_filesystem_storage_permits(bytes: u64) -> u32 { - let kb = bytes.div_ceil(FILESYSTEM_STORAGE_PERMIT_SIZE_KB * 1024); - kb.min(u32::MAX as u64) as u32 -} - -/// Convert a permit count back to bytes. This is the inverse of -/// `bytes_to_filesystem_storage_permits` and always returns a multiple of -/// `FILESYSTEM_STORAGE_PERMIT_SIZE_KB * 1024`. -pub fn filesystem_storage_permits_to_bytes(permits: u32) -> u64 { - permits as u64 * FILESYSTEM_STORAGE_PERMIT_SIZE_KB * 1024 -} - -/// Round a byte count up to the nearest permit boundary (1 KB). -/// Returns the actual number of bytes consumed from the pool when acquiring -/// permits for `bytes` — i.e. what will be released when those permits are dropped. -pub fn filesystem_storage_bytes_rounded_up(bytes: u64) -> u64 { - filesystem_storage_permits_to_bytes(bytes_to_filesystem_storage_permits(bytes)) -} - -/// Convert a storage semaphore pool size in bytes to the number of permits to -/// initialise the semaphore with. -pub fn filesystem_storage_pool_bytes_to_permits(bytes: usize) -> usize { - bytes.div_ceil(FILESYSTEM_STORAGE_PERMIT_SIZE_KB as usize * 1024) -} diff --git a/golem-worker-executor/src/services/active_workers/mod.rs b/golem-worker-executor/src/services/active_workers/mod.rs index 82bea2f35b..63679cbba0 100644 --- a/golem-worker-executor/src/services/active_workers/mod.rs +++ b/golem-worker-executor/src/services/active_workers/mod.rs @@ -16,39 +16,36 @@ pub mod admission; pub mod component_charge; pub mod concurrent_agents_scheduler; pub mod concurrent_agents_semaphore; -pub mod fs_semaphore; pub mod memory_probe; #[cfg(test)] mod tests; -pub use concurrent_agents_scheduler::{ConcurrentAgentPermit, ConcurrentAgentsScheduler}; -pub use concurrent_agents_semaphore::ConcurrentAgentsSemaphore; -pub use fs_semaphore::{ - FILESYSTEM_STORAGE_PERMIT_SIZE_KB, FilesystemStoragePermit, FilesystemStorageSemaphore, - bytes_to_filesystem_storage_permits, filesystem_storage_bytes_rounded_up, - filesystem_storage_permits_to_bytes, filesystem_storage_pool_bytes_to_permits, -}; - pub(crate) use admission::MemoryGrant; use admission::{AdmissionController, EvictionPriority, EvictionSource}; use async_trait::async_trait; pub use component_charge::HeldComponentCharge; use component_charge::{ChargeSource, ComponentChargeGuard, ComponentChargeRegistry}; +pub use concurrent_agents_scheduler::{ConcurrentAgentPermit, ConcurrentAgentsScheduler}; +pub use concurrent_agents_semaphore::ConcurrentAgentsSemaphore; use memory_probe::{MemoryProbe, default_probe}; +use std::future::Future; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tracing::{Instrument, debug}; use crate::services::HasAll; +use crate::services::agent_filesystem::{ + AgentFilesystems, FilesystemCapacity, FilesystemStorageError, MutationOperation, +}; use crate::services::card_interest::CardInterestIndex; use crate::services::golem_config::{ - AgentStatusFlushConfig, FilesystemStorageConfig, MemoryConfig, + AgentStatusFlushConfig, FilesystemPressureConfig, FilesystemStorageConfig, MemoryConfig, }; use crate::services::resource_limits::AtomicResourceEntry; -use crate::worker::Worker; use crate::worker::status_flusher::AgentStatusFlushQueue; +use crate::worker::{EvictionClass, EvictionStopOutcome, FilesystemPressureEligibility, Worker}; use crate::workerctx::WorkerCtx; use golem_common::cache::{BackgroundEvictionMode, Cache, FullCacheEvictionMode, SimpleCache}; use golem_common::model::account::AccountId; @@ -80,7 +77,7 @@ impl RegisteredConcurrentAccount { pub struct ActiveWorkers { workers: Cache>, WorkerExecutorError>, card_interest_index: Arc, - worker_filesystem_storage: Arc, + agent_filesystems: Arc, concurrent_agents: Arc, acquire_retry_delay: Duration, /// Authoritative measured-headroom admission gate, and the sole admission @@ -97,6 +94,7 @@ pub struct ActiveWorkers { /// module charge. component_size_coefficient: f64, status_flush_queue: Arc, + filesystem_pressure_recovery: tokio::sync::Mutex<()>, } /// Identifies a compiled component for module-charge accounting. @@ -111,7 +109,7 @@ impl ActiveWorkers { storage_config: &FilesystemStorageConfig, agent_status_flush_config: &AgentStatusFlushConfig, shutdown_token: CancellationToken, - ) -> Self { + ) -> Result { // Build the probe once and hand it to the measured-headroom gate, which // bases its decision on the pod's cgroup limit when constrained (not host // RAM). @@ -135,7 +133,8 @@ impl ActiveWorkers { storage_config: &FilesystemStorageConfig, agent_status_flush_config: &AgentStatusFlushConfig, shutdown_token: CancellationToken, - ) -> Self { + ) -> Result { + let agent_filesystems = Arc::new(AgentFilesystems::new(storage_config)?); let admission = memory_config.enable_measured_admission.then(|| { Arc::new(AdmissionController::new( probe, @@ -154,10 +153,7 @@ impl ActiveWorkers { let active_workers = Self { workers, card_interest_index: Arc::new(CardInterestIndex::new()), - worker_filesystem_storage: Arc::new(FilesystemStorageSemaphore::new( - storage_config.worker_filesystem_storage(), - storage_config.acquire_retry_delay, - )), + agent_filesystems, concurrent_agents: Arc::new(ConcurrentAgentsScheduler::new()), acquire_retry_delay: memory_config.acquire_retry_delay, admission, @@ -168,9 +164,10 @@ impl ActiveWorkers { agent_status_flush_config.max_concurrency, shutdown_token, ), + filesystem_pressure_recovery: tokio::sync::Mutex::new(()), }; active_workers.initialize_metrics(); - active_workers + Ok(active_workers) } /// The per-executor queue used to batch cached agent status blob writes in the background. @@ -178,6 +175,32 @@ impl ActiveWorkers { self.status_flush_queue.clone() } + pub(crate) fn agent_filesystems(&self) -> Arc { + Arc::clone(&self.agent_filesystems) + } + + pub(crate) async fn recover_filesystem_pressure( + &self, + operation: MutationOperation, + deadline: Instant, + ) -> bool { + let Ok(_recovery) = tokio::time::timeout_at( + tokio::time::Instant::from_std(deadline), + self.filesystem_pressure_recovery.lock(), + ) + .await + else { + return false; + }; + recover_filesystem_pressure_from( + self, + self.agent_filesystems.pressure_policy(), + operation, + deadline, + ) + .await + } + /// Acquire (or share) the per-component module charge for a worker of the /// given component. The first resident worker of the component reserves its /// compiled-module size (scaled by `component_size_coefficient`) with the @@ -410,36 +433,6 @@ impl ActiveWorkers { } } - /// Blocking acquire of storage semaphore permits. Loops until the requested - /// number of bytes is available, evicting idle workers as needed. - pub async fn acquire_filesystem_storage(&self, storage_bytes: u64) -> FilesystemStoragePermit { - let workers = self.workers.clone(); - self.worker_filesystem_storage - .acquire(storage_bytes, || { - let workers = workers.clone(); - async move { Self::try_free_up_filesystem_storage(&workers, storage_bytes).await } - }) - .await - } - - /// Non-blocking, priority storage acquire. Grabs the allocation lock to - /// interrupt any ongoing blocking `acquire_storage` loops, then attempts once. - /// - /// Returns `None` if the requested storage is not available even after - /// interrupting waiting acquires. - pub async fn try_acquire_filesystem_storage( - &self, - storage_bytes: u64, - ) -> Option { - self.worker_filesystem_storage - .try_acquire(storage_bytes) - .await - } - - pub fn filesystem_storage_semaphore(&self) -> Arc { - self.worker_filesystem_storage.clone() - } - /// Register an account with the per-account concurrent agent semaphore. /// /// Must be called (from `Worker::new`) before any concurrent-agent permit @@ -459,76 +452,216 @@ impl ActiveWorkers { } } - async fn try_free_up_filesystem_storage( - workers: &Cache>, WorkerExecutorError>, - storage_bytes: u64, - ) -> bool { - let mut idle_candidates = Vec::new(); - let mut warm_candidates = Vec::new(); + /// Initializes worker gauges. Subsequent changes are recorded inline at the mutation sites. + fn initialize_metrics(&self) { + crate::metrics::workers::initialize_worker_metrics(); + } +} - debug!("Collecting storage eviction candidates"); - for (agent_id, worker) in workers.iter().await { - if let Some(class) = worker.eviction_class().await - && let Ok(storage) = worker.filesystem_storage_requirement().await - { - let last_changed = worker.last_execution_state_change(); - let entry = (agent_id, worker, storage, last_changed); - match class { - crate::worker::EvictionClass::LoadedIdle => idle_candidates.push(entry), - crate::worker::EvictionClass::WarmRunnable => warm_candidates.push(entry), - } +struct WorkerFilesystemPressureCandidate { + agent_id: AgentId, + worker: Arc>, + eligibility: FilesystemPressureEligibility, +} + +#[async_trait] +trait FilesystemPressureRecoverySource { + type Candidate: Send; + + async fn capacity(&self) -> Result; + async fn candidates(&self) -> Vec; + fn candidate_name(&self, candidate: &Self::Candidate) -> String; + async fn evict( + &self, + candidate: Self::Candidate, + deadline: Instant, + ) -> Option; +} + +#[async_trait] +impl FilesystemPressureRecoverySource for ActiveWorkers { + type Candidate = WorkerFilesystemPressureCandidate; + + async fn capacity(&self) -> Result { + self.agent_filesystems + .observe_capacity() + .await + .map_err(|error| error.to_string()) + } + + async fn candidates(&self) -> Vec { + let mut candidates = Vec::new(); + for (agent_id, worker) in self.workers.iter().await { + if worker.eviction_class().await == Some(EvictionClass::LoadedIdle) { + let Some(eligibility) = worker.filesystem_pressure_eligibility().await else { + continue; + }; + candidates.push(( + Worker::::filesystem_pressure_eligible_since(eligibility), + agent_id.to_string(), + WorkerFilesystemPressureCandidate { + agent_id, + worker, + eligibility, + }, + )); } } + sort_filesystem_pressure_candidates(&mut candidates); + candidates + .into_iter() + .map(|(_, _, candidate)| candidate) + .collect() + } - // Sort each bucket — newest first so we pop oldest - idle_candidates.sort_by_key(|(_, _, _, ts)| ts.to_millis()); - idle_candidates.reverse(); - warm_candidates.sort_by_key(|(_, _, _, ts)| ts.to_millis()); - warm_candidates.reverse(); + fn candidate_name(&self, candidate: &Self::Candidate) -> String { + candidate.agent_id.to_string() + } - let mut freed: u64 = 0; + async fn evict( + &self, + candidate: Self::Candidate, + deadline: Instant, + ) -> Option { + let eviction = async move { + let outcome = candidate + .worker + .stop_if_evictable_with_outcome( + EvictionClass::LoadedIdle, + Some(candidate.eligibility), + ) + .await; + if outcome == EvictionStopOutcome::Unloaded { + crate::metrics::workers::record_worker_eviction("FilesystemPressureLoadedIdle"); + } + outcome + }; + match spawn_before_deadline(deadline, eviction).await { + Some(Ok(outcome)) => Some(outcome), + Some(Err(error)) => { + tracing::warn!(error = %error, "Filesystem pressure eviction task failed"); + None + } + None => None, + } + } +} - // First evict LoadedIdle workers - while freed < storage_bytes && !idle_candidates.is_empty() { - let (agent_id, worker, storage, _) = idle_candidates.pop().unwrap(); - debug!("Trying to stop idle {agent_id} to free up storage"); - if worker - .stop_if_evictable(crate::worker::EvictionClass::LoadedIdle) - .await - { - debug!("Stopped idle {agent_id}, freed {storage} bytes of storage"); - crate::metrics::workers::record_worker_eviction("LoadedIdle"); - freed += storage; +fn sort_filesystem_pressure_candidates(candidates: &mut [(u64, String, T)]) { + candidates.sort_by(|left, right| (left.0, left.1.as_str()).cmp(&(right.0, right.1.as_str()))); +} + +async fn recover_filesystem_pressure_from( + source: &S, + policy: &FilesystemPressureConfig, + operation: MutationOperation, + deadline: Instant, +) -> bool { + if Instant::now() >= deadline { + return false; + } + let initial = match before_deadline(deadline, source.capacity()).await { + Some(Ok(capacity)) => capacity, + Some(Err(error)) => { + tracing::warn!( + error, + "Failed to observe filesystem capacity before pressure recovery" + ); + return false; + } + None => return false, + }; + let Some(mut pressure) = policy.pressure(operation, initial) else { + return false; + }; + + let Some(candidates) = before_deadline(deadline, source.candidates()).await else { + return false; + }; + for candidate in candidates { + if Instant::now() >= deadline { + return false; + } + let candidate_name = source.candidate_name(&candidate); + let before = match before_deadline(deadline, source.capacity()).await { + Some(Ok(capacity)) => capacity, + Some(Err(error)) => { + tracing::warn!( + error, + "Failed to observe filesystem capacity before pressure eviction" + ); + return false; } + None => return false, + }; + pressure = pressure.include(policy.pressure(operation, before)); + if policy.target_reached(pressure, before) { + return true; } - // Then evict WarmRunnable workers if still under pressure - while freed < storage_bytes && !warm_candidates.is_empty() { - let (agent_id, worker, storage, _) = warm_candidates.pop().unwrap(); - debug!("Trying to stop warm-runnable {agent_id} to free up storage"); - if worker - .stop_if_evictable(crate::worker::EvictionClass::WarmRunnable) - .await - { - debug!("Stopped warm-runnable {agent_id}, freed {storage} bytes of storage"); - crate::metrics::workers::record_worker_eviction("WarmRunnable"); - freed += storage; + let Some(outcome) = source.evict(candidate, deadline).await else { + return false; + }; + match outcome { + EvictionStopOutcome::Ineligible => continue, + EvictionStopOutcome::CleanupFailed => { + tracing::warn!( + agent_id = candidate_name, + "Filesystem pressure victim cleanup failed" + ); + continue; } + EvictionStopOutcome::Unloaded => {} } - if freed > 0 { - debug!("Freed {freed} bytes by evicting worker(s); re-checking availability"); + if Instant::now() >= deadline { + return false; } - freed >= storage_bytes - } - /// Initializes worker gauges. Subsequent changes are recorded inline at the mutation sites. - fn initialize_metrics(&self) { - crate::metrics::workers::initialize_worker_metrics(); - crate::metrics::workers::set_filesystem_semaphore_available( - self.worker_filesystem_storage.available_bytes(), - ); + for attempt in 0..policy.reclamation_observation_attempts { + let after = match before_deadline(deadline, source.capacity()).await { + Some(Ok(capacity)) => capacity, + Some(Err(error)) => { + tracing::warn!( + error, + agent_id = candidate_name, + "Failed to observe filesystem capacity after pressure eviction" + ); + return false; + } + None => return false, + }; + pressure = pressure.include(policy.pressure(operation, after)); + if policy.target_reached(pressure, after) { + return true; + } + if attempt + 1 < policy.reclamation_observation_attempts + && before_deadline( + deadline, + tokio::time::sleep(policy.reclamation_observation_delay), + ) + .await + .is_none() + { + return false; + } + } } + + false +} + +async fn before_deadline(deadline: Instant, future: impl Future) -> Option { + tokio::time::timeout_at(tokio::time::Instant::from_std(deadline), future) + .await + .ok() +} + +async fn spawn_before_deadline( + deadline: Instant, + future: impl Future + Send + 'static, +) -> Option> { + before_deadline(deadline, tokio::spawn(future)).await } impl From for crate::worker::EvictionClass { diff --git a/golem-worker-executor/src/services/active_workers/tests.rs b/golem-worker-executor/src/services/active_workers/tests.rs index 11ed482770..cbc2f23bdc 100644 --- a/golem-worker-executor/src/services/active_workers/tests.rs +++ b/golem-worker-executor/src/services/active_workers/tests.rs @@ -1,197 +1,391 @@ use super::concurrent_agents_scheduler::ConcurrentAgentsScheduler; use super::concurrent_agents_semaphore::ConcurrentAgentsSemaphore; -use super::fs_semaphore::*; +use super::{ + FilesystemPressureRecoverySource, recover_filesystem_pressure_from, + sort_filesystem_pressure_candidates, spawn_before_deadline, +}; +use crate::services::agent_filesystem::{FilesystemCapacity, MutationOperation}; +use crate::services::golem_config::FilesystemPressureConfig; use crate::services::resource_limits::AtomicResourceEntry; +use crate::worker::EvictionStopOutcome; use golem_common::model::AgentId; use golem_common::model::account::AccountId; use golem_common::model::component::ComponentId; -use std::sync::Arc; -use std::time::Duration; +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use test_r::{non_flaky, test, timeout}; use tokio::sync::Barrier; use uuid::Uuid; test_r::enable!(); -fn concurrent_agents_semaphore() -> ConcurrentAgentsSemaphore { - ConcurrentAgentsSemaphore::new() +struct TestFilesystemPressureSource { + capacities: Mutex>>, + candidates: Mutex>>, + outcomes: Mutex>, + evicted: Mutex>, } -fn account() -> AccountId { - AccountId(Uuid::new_v4()) -} +#[async_trait::async_trait] +impl FilesystemPressureRecoverySource for TestFilesystemPressureSource { + type Candidate = u8; -fn resource_entry_with_agent_limit(limit: u64) -> Arc { - Arc::new(AtomicResourceEntry::new( - u64::MAX, - usize::MAX, - usize::MAX, - u64::MAX, - limit, - )) + async fn capacity(&self) -> Result { + self.capacities.lock().unwrap().pop_front().unwrap() + } + + async fn candidates(&self) -> Vec { + self.candidates.lock().unwrap().take().unwrap() + } + + fn candidate_name(&self, candidate: &Self::Candidate) -> String { + candidate.to_string() + } + + async fn evict( + &self, + candidate: Self::Candidate, + _deadline: Instant, + ) -> Option { + self.evicted.lock().unwrap().push(candidate); + Some(self.outcomes.lock().unwrap()[&candidate]) + } } -fn unlimited_resource_entry() -> Arc { - Arc::new(AtomicResourceEntry::new( - u64::MAX, - usize::MAX, - usize::MAX, - u64::MAX, - AtomicResourceEntry::UNLIMITED_CONCURRENT_AGENTS, - )) +fn filesystem_capacity(available_bytes: u64) -> FilesystemCapacity { + FilesystemCapacity { + total_bytes: 100, + available_bytes, + total_filesystem_objects: 100, + available_filesystem_objects: 100, + } } -#[test] -fn bytes_to_permits_exact_kb_boundary() { - assert_eq!(bytes_to_filesystem_storage_permits(1024), 1); +fn filesystem_object_capacity(available_filesystem_objects: u64) -> FilesystemCapacity { + FilesystemCapacity { + total_bytes: 100, + available_bytes: 100, + total_filesystem_objects: 100, + available_filesystem_objects, + } } -#[test] -fn bytes_to_permits_rounds_up_partial_kb() { - assert_eq!(bytes_to_filesystem_storage_permits(1), 1); - assert_eq!(bytes_to_filesystem_storage_permits(1025), 2); +fn pressure_policy() -> FilesystemPressureConfig { + FilesystemPressureConfig { + minimum_available_bytes: 10, + target_available_bytes: 20, + minimum_available_filesystem_objects: 2, + target_available_filesystem_objects: 4, + reclamation_observation_attempts: 2, + reclamation_observation_delay: Duration::ZERO, + } } -#[test] -fn bytes_to_permits_zero_bytes() { - assert_eq!(bytes_to_filesystem_storage_permits(0), 0); +fn recovery_deadline() -> Instant { + Instant::now() + Duration::from_secs(1) } #[test] -fn bytes_to_permits_1gb() { +fn filesystem_pressure_candidates_are_oldest_first_with_stable_ties() { + let mut candidates = vec![ + (2, "a".to_string(), "new"), + (1, "z".to_string(), "old-z"), + (1, "a".to_string(), "old-a"), + ]; + + sort_filesystem_pressure_candidates(&mut candidates); + assert_eq!( - bytes_to_filesystem_storage_permits(1024 * 1024 * 1024), - 1_048_576 + candidates + .into_iter() + .map(|(_, _, candidate)| candidate) + .collect::>(), + vec!["old-a", "old-z", "new"] ); } #[test] -fn bytes_to_permits_very_large_saturates_at_u32_max() { - assert_eq!(bytes_to_filesystem_storage_permits(u64::MAX), u32::MAX); +async fn filesystem_pressure_recovery_observes_after_successful_deletion() { + let source = TestFilesystemPressureSource { + capacities: Mutex::new( + [ + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(20)), + ] + .into(), + ), + candidates: Mutex::new(Some(vec![1, 2])), + outcomes: Mutex::new(HashMap::from([ + (1, EvictionStopOutcome::CleanupFailed), + (2, EvictionStopOutcome::Unloaded), + ])), + evicted: Mutex::new(Vec::new()), + }; + + assert!( + recover_filesystem_pressure_from( + &source, + &pressure_policy(), + MutationOperation::Write, + recovery_deadline(), + ) + .await + ); + assert_eq!(*source.evicted.lock().unwrap(), vec![1, 2]); + assert!(source.capacities.lock().unwrap().is_empty()); } #[test] -fn bytes_to_permits_just_under_4tb() { - let just_under: u64 = (u32::MAX as u64) * 1024; - assert_eq!(bytes_to_filesystem_storage_permits(just_under), u32::MAX); +async fn filesystem_pressure_recovery_rechecks_raced_candidate_eligibility() { + let source = TestFilesystemPressureSource { + capacities: Mutex::new( + [ + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(20)), + ] + .into(), + ), + candidates: Mutex::new(Some(vec![1, 2])), + outcomes: Mutex::new(HashMap::from([ + (1, EvictionStopOutcome::Ineligible), + (2, EvictionStopOutcome::Unloaded), + ])), + evicted: Mutex::new(Vec::new()), + }; + + assert!( + recover_filesystem_pressure_from( + &source, + &pressure_policy(), + MutationOperation::Write, + recovery_deadline(), + ) + .await + ); + assert_eq!(*source.evicted.lock().unwrap(), vec![1, 2]); } #[test] -fn storage_pool_permits_10gb() { - let ten_gb: usize = 10 * 1024 * 1024 * 1024; - assert_eq!( - filesystem_storage_pool_bytes_to_permits(ten_gb), - 10 * 1024 * 1024 +async fn filesystem_pressure_recovery_waits_for_delayed_reclamation() { + let source = TestFilesystemPressureSource { + capacities: Mutex::new( + [ + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(12)), + Ok(filesystem_capacity(20)), + ] + .into(), + ), + candidates: Mutex::new(Some(vec![1, 2])), + outcomes: Mutex::new(HashMap::from([ + (1, EvictionStopOutcome::Unloaded), + (2, EvictionStopOutcome::Unloaded), + ])), + evicted: Mutex::new(Vec::new()), + }; + + assert!( + recover_filesystem_pressure_from( + &source, + &pressure_policy(), + MutationOperation::Write, + recovery_deadline(), + ) + .await ); + assert_eq!(*source.evicted.lock().unwrap(), vec![1]); } -fn filesystem_storage_semaphore(pool_bytes: usize) -> FilesystemStorageSemaphore { - FilesystemStorageSemaphore::new(pool_bytes, Duration::from_millis(1)) +#[test] +async fn filesystem_pressure_recovery_handles_object_pressure() { + let source = TestFilesystemPressureSource { + capacities: Mutex::new( + [ + Ok(filesystem_object_capacity(1)), + Ok(filesystem_object_capacity(1)), + Ok(filesystem_object_capacity(4)), + ] + .into(), + ), + candidates: Mutex::new(Some(vec![1])), + outcomes: Mutex::new(HashMap::from([(1, EvictionStopOutcome::Unloaded)])), + evicted: Mutex::new(Vec::new()), + }; + + assert!( + recover_filesystem_pressure_from( + &source, + &pressure_policy(), + MutationOperation::Create, + recovery_deadline(), + ) + .await + ); + assert_eq!(*source.evicted.lock().unwrap(), vec![1]); } #[test] -async fn try_acquire_succeeds_when_space_available() { - let filesystem_storage_semaphore = filesystem_storage_semaphore(4 * 1024); // 4 KB pool - let permit = filesystem_storage_semaphore.try_acquire(2 * 1024).await; // ask for 2 KB - assert!(permit.is_some()); - assert_eq!(filesystem_storage_semaphore.available_bytes(), 2 * 1024); +async fn filesystem_pressure_recovery_includes_new_pressure_dimensions() { + let source = TestFilesystemPressureSource { + capacities: Mutex::new( + [ + Ok(FilesystemCapacity { + available_bytes: 5, + available_filesystem_objects: 100, + ..filesystem_capacity(5) + }), + Ok(FilesystemCapacity { + available_bytes: 20, + available_filesystem_objects: 1, + ..filesystem_capacity(20) + }), + Ok(FilesystemCapacity { + available_bytes: 20, + available_filesystem_objects: 4, + ..filesystem_capacity(20) + }), + ] + .into(), + ), + candidates: Mutex::new(Some(vec![1])), + outcomes: Mutex::new(HashMap::from([(1, EvictionStopOutcome::Unloaded)])), + evicted: Mutex::new(Vec::new()), + }; + + assert!( + recover_filesystem_pressure_from( + &source, + &pressure_policy(), + MutationOperation::Create, + recovery_deadline(), + ) + .await + ); + assert_eq!(*source.evicted.lock().unwrap(), vec![1]); } #[test] -async fn try_acquire_returns_none_when_pool_exhausted() { - let filesystem_storage_semaphore = filesystem_storage_semaphore(2 * 1024); // 2 KB pool - let _permit = filesystem_storage_semaphore - .try_acquire(2 * 1024) +async fn filesystem_pressure_observation_failure_stops_without_eviction() { + let source = TestFilesystemPressureSource { + capacities: Mutex::new([Err("statvfs failed".to_string())].into()), + candidates: Mutex::new(Some(vec![1])), + outcomes: Mutex::new(HashMap::from([(1, EvictionStopOutcome::Unloaded)])), + evicted: Mutex::new(Vec::new()), + }; + + assert!( + !recover_filesystem_pressure_from( + &source, + &pressure_policy(), + MutationOperation::Write, + recovery_deadline(), + ) .await - .unwrap(); // exhaust it - let second = filesystem_storage_semaphore.try_acquire(1024).await; // no space left - assert!(second.is_none()); + ); + assert!(source.evicted.lock().unwrap().is_empty()); } #[test] -async fn try_acquire_zero_bytes_always_succeeds() { - let filesystem_storage_semaphore = filesystem_storage_semaphore(0); // empty pool — 0 bytes → 0 permits - let permit = filesystem_storage_semaphore.try_acquire(0).await; - assert!(permit.is_some()); +async fn filesystem_pressure_recovery_does_not_start_after_deadline() { + let source = TestFilesystemPressureSource { + capacities: Mutex::new(VecDeque::new()), + candidates: Mutex::new(Some(vec![1])), + outcomes: Mutex::new(HashMap::from([(1, EvictionStopOutcome::Unloaded)])), + evicted: Mutex::new(Vec::new()), + }; + + assert!( + !recover_filesystem_pressure_from( + &source, + &pressure_policy(), + MutationOperation::Write, + Instant::now(), + ) + .await + ); + assert!(source.evicted.lock().unwrap().is_empty()); } #[test] -async fn dropping_permit_returns_space_to_pool() { - let filesystem_storage_semaphore = filesystem_storage_semaphore(4 * 1024); - { - let _permit = filesystem_storage_semaphore - .try_acquire(4 * 1024) - .await - .unwrap(); - assert_eq!(filesystem_storage_semaphore.available_bytes(), 0); - } // permit dropped here - assert_eq!(filesystem_storage_semaphore.available_bytes(), 4 * 1024); +#[timeout("1s")] +async fn filesystem_pressure_reclamation_wait_is_bounded_by_deadline() { + let source = TestFilesystemPressureSource { + capacities: Mutex::new( + [ + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(5)), + Ok(filesystem_capacity(12)), + ] + .into(), + ), + candidates: Mutex::new(Some(vec![1])), + outcomes: Mutex::new(HashMap::from([(1, EvictionStopOutcome::Unloaded)])), + evicted: Mutex::new(Vec::new()), + }; + let mut policy = pressure_policy(); + policy.reclamation_observation_delay = Duration::from_secs(5); + + assert!( + !recover_filesystem_pressure_from( + &source, + &policy, + MutationOperation::Write, + Instant::now() + Duration::from_millis(10), + ) + .await + ); + assert_eq!(*source.evicted.lock().unwrap(), vec![1]); } #[test] -async fn multiple_permits_are_independent() { - let filesystem_storage_semaphore = filesystem_storage_semaphore(6 * 1024); // 6 KB pool - let p1 = filesystem_storage_semaphore - .try_acquire(2 * 1024) - .await - .unwrap(); - let p2 = filesystem_storage_semaphore - .try_acquire(2 * 1024) +async fn timed_out_eviction_continues_to_completion() { + let completed = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let task_completed = Arc::clone(&completed); + + assert!( + spawn_before_deadline(Instant::now() + Duration::from_millis(10), async move { + tokio::time::sleep(Duration::from_millis(25)).await; + task_completed.store(true, std::sync::atomic::Ordering::Release); + }) .await - .unwrap(); - assert_eq!(filesystem_storage_semaphore.available_bytes(), 2 * 1024); - drop(p1); - assert_eq!(filesystem_storage_semaphore.available_bytes(), 4 * 1024); - drop(p2); - assert_eq!(filesystem_storage_semaphore.available_bytes(), 6 * 1024); + .is_none() + ); + tokio::time::sleep(Duration::from_millis(30)).await; + assert!(completed.load(std::sync::atomic::Ordering::Acquire)); } -#[test] -async fn try_acquire_rounds_up_to_kb_boundary() { - let filesystem_storage_semaphore = filesystem_storage_semaphore(2 * 1024); // 2 KB = 2 permits - // 1 byte rounds up to 1 KB = 1 permit; should leave 1 KB - let _p = filesystem_storage_semaphore.try_acquire(1).await.unwrap(); - assert_eq!(filesystem_storage_semaphore.available_bytes(), 1024); +fn concurrent_agents_semaphore() -> ConcurrentAgentsSemaphore { + ConcurrentAgentsSemaphore::new() } -#[test] -async fn acquire_succeeds_immediately_when_space_available() { - let filesystem_storage_semaphore = filesystem_storage_semaphore(4 * 1024); - // pool has space so it succeeds on the first try without invoking free_up - let permit = filesystem_storage_semaphore - .acquire(2 * 1024, || async { false }) - .await; - assert_eq!(permit.num_permits(), 2); // 2 KB = 2 permits - assert_eq!(filesystem_storage_semaphore.available_bytes(), 2 * 1024); +fn account() -> AccountId { + AccountId(Uuid::new_v4()) } -#[test] -async fn acquire_succeeds_after_free_up_releases_space() { - let filesystem_storage_semaphore = filesystem_storage_semaphore(4 * 1024); - let _held = filesystem_storage_semaphore - .try_acquire(4 * 1024) - .await - .unwrap(); // exhaust pool - - // Share the inner semaphore Arc with the closure so it can add permits - // back to simulate a worker releasing its storage on eviction. - let sem_arc = filesystem_storage_semaphore.inner_semaphore().clone(); - let released = std::sync::atomic::AtomicBool::new(false); - let permit = filesystem_storage_semaphore - .acquire(2 * 1024, || { - let sem = sem_arc.clone(); - let already = released.fetch_or(true, std::sync::atomic::Ordering::SeqCst); - async move { - if !already { - sem.add_permits(2); // 2 permits = 2 KB freed - true - } else { - false - } - } - }) - .await; - assert_eq!(permit.num_permits(), 2); +fn resource_entry_with_agent_limit(limit: u64) -> Arc { + Arc::new(AtomicResourceEntry::new( + u64::MAX, + usize::MAX, + usize::MAX, + u64::MAX, + limit, + )) +} + +fn unlimited_resource_entry() -> Arc { + Arc::new(AtomicResourceEntry::new( + u64::MAX, + usize::MAX, + usize::MAX, + u64::MAX, + AtomicResourceEntry::UNLIMITED_CONCURRENT_AGENTS, + )) } // --------------------------------------------------------------------------- diff --git a/golem-worker-executor/src/services/agent_filesystem/backend.rs b/golem-worker-executor/src/services/agent_filesystem/backend.rs new file mode 100644 index 0000000000..5ea654c0cd --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/backend.rs @@ -0,0 +1,233 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{ + AgentFilesystemStorageLimit, AgentFilesystemUpdateEffectLease, AgentFilesystemUsage, + FilesystemCapacity, FilesystemStorageConfig, FilesystemStorageError, OwnedAgentId, + OwnedMutexGuard, Path, PathBuf, ResolvedAgentFilesystemLimits, +}; +use crate::services::file_loader::InitialFileSource; +use async_trait::async_trait; +use std::sync::Arc; +use std::time::Instant; + +pub(super) struct ProvisionedAgentFilesystem { + backend: Arc, + cleanup: Option>, + lifecycle: Option>, +} + +impl ProvisionedAgentFilesystem { + pub fn new( + backend: Arc, + cleanup: Box, + lifecycle: OwnedMutexGuard<()>, + ) -> Self { + Self { + backend, + cleanup: Some(cleanup), + lifecycle: Some(lifecycle), + } + } + + pub fn backend(&self) -> &Arc { + &self.backend + } + + pub async fn rollback( + mut self, + creation_error: FilesystemStorageError, + ) -> FilesystemStorageError { + let started = Instant::now(); + let result = self.delete().await; + super::record_agent_filesystem_lifecycle("delete", result.is_ok(), started.elapsed()); + match result { + Ok(()) => creation_error, + Err(cleanup_error) => cleanup_error, + } + } + + pub async fn delete(&mut self) -> Result<(), FilesystemStorageError> { + let Some(cleanup) = self.cleanup.as_mut() else { + return Ok(()); + }; + let result = cleanup.delete().await; + if result.is_ok() { + self.cleanup.take(); + self.lifecycle.take(); + } + result + } +} + +impl Drop for ProvisionedAgentFilesystem { + fn drop(&mut self) { + let Some(mut cleanup) = self.cleanup.take() else { + return; + }; + let lifecycle = self.lifecycle.take(); + let background = cleanup.requires_background_drop_cleanup(); + let delete = move || { + let started = Instant::now(); + let result = cleanup.delete_blocking(); + super::record_agent_filesystem_lifecycle( + "delete_fallback", + result.is_ok(), + started.elapsed(), + ); + if let Err(error) = result { + tracing::error!(error = %error, "Failed to delete agent runtime filesystem during fallback cleanup"); + } + drop(lifecycle); + }; + if background { + std::thread::spawn(delete); + } else { + delete(); + } + } +} + +#[async_trait] +pub(super) trait FilesystemBackendProvisioner: Send + Sync { + fn initial_file_cache_root(&self) -> Option<&Path>; + + async fn provision_for( + &self, + agent_id: &OwnedAgentId, + ) -> Result; + + async fn observe_capacity(&self) -> Result; + + #[cfg(test)] + fn as_any(&self) -> &dyn std::any::Any; +} + +pub(super) struct InitialFileMaterialization { + pub materialization_root: PathBuf, + pub source: InitialFileSource, + pub target: PathBuf, + pub read_only: bool, + pub effect: AgentFilesystemUpdateEffectLease, + pub staging: Option>, +} + +#[async_trait] +pub(super) trait AgentFilesystemBackend: Send + Sync { + fn root(&self) -> &Path; + + fn create_staging_dir(&self) -> std::io::Result; + + async fn materialize_initial_file( + &self, + materialization: InitialFileMaterialization, + ) -> Result<(), FilesystemStorageError>; + + async fn observe_capacity(&self) -> Result; + + fn quota(&self) -> Option<&dyn AgentFilesystemQuota> { + None + } + + #[cfg(test)] + fn as_any(&self) -> &dyn std::any::Any; +} + +pub(super) fn agent_filesystem_owner_path(agent_id: &OwnedAgentId) -> PathBuf { + PathBuf::from(agent_id.environment_id.to_string()) + .join(agent_id.agent_id.component_id.to_string()) + .join(agent_id.agent_id.agent_name_encoded()) +} + +pub(super) struct InstalledAgentFilesystemLimit { + pub limits: ResolvedAgentFilesystemLimits, + pub usage: AgentFilesystemUsage, +} + +#[async_trait] +pub(super) trait AgentFilesystemQuota: Send + Sync { + async fn usage(&self) -> Result; + + async fn failure_observations( + &self, + installed_limits: Option, + ) -> Result<(AgentFilesystemUsage, Option), FilesystemStorageError>; + + async fn install_limit( + &self, + limit: AgentFilesystemStorageLimit, + effect: AgentFilesystemUpdateEffectLease, + ) -> Result; +} + +#[async_trait] +pub(super) trait AgentFilesystemCleanup: Send + Sync { + async fn delete(&mut self) -> Result<(), FilesystemStorageError>; + + fn delete_blocking(&mut self) -> Result<(), FilesystemStorageError>; + + fn requires_background_drop_cleanup(&self) -> bool { + false + } +} + +pub(super) fn configured_provisioner( + settings: &FilesystemStorageConfig, +) -> Result, FilesystemStorageError> { + if settings.deterministic_root_dir.is_some() && settings.managed_xfs_root_dir.is_some() { + return Err(FilesystemStorageError::verification( + "select exactly one filesystem storage backend", + Path::new(""), + )); + } + + match settings.managed_xfs_root_dir.as_deref() { + Some(root) => configured_xfs_backend(settings, root), + None => Ok(Arc::new(super::unmanaged::UnmanagedBackend::new( + settings.deterministic_root_dir.clone(), + settings.cleanup_retry.clone(), + ))), + } +} + +#[cfg(target_os = "linux")] +fn configured_xfs_backend( + settings: &FilesystemStorageConfig, + root: &Path, +) -> Result, FilesystemStorageError> { + settings.filesystem_object_limit_policy.validate()?; + settings.pressure.validate()?; + let backend = super::xfs::XfsBackend::new( + root, + &settings.cleanup_retry, + &settings.filesystem_object_limit_policy, + )?; + settings + .pressure + .validate_capacity(backend.observe_capacity().map_err(|error| { + FilesystemStorageError::io("validate managed XFS pressure capacity", root, error) + })?)?; + Ok(Arc::new(backend)) +} + +#[cfg(not(target_os = "linux"))] +fn configured_xfs_backend( + _settings: &FilesystemStorageConfig, + root: &Path, +) -> Result, FilesystemStorageError> { + Err(FilesystemStorageError::verification( + "initialize managed XFS backend on a non-Linux platform", + root, + )) +} diff --git a/golem-worker-executor/src/services/agent_filesystem/failure.rs b/golem-worker-executor/src/services/agent_filesystem/failure.rs new file mode 100644 index 0000000000..42ce00e09f --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/failure.rs @@ -0,0 +1,464 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::Ordering; + +pub(crate) type AgentFilesystemInvalidationCallback = + Arc Pin + Send>> + Send + Sync>; +pub(crate) type AgentFilesystemRetryCallback = + Arc Pin + Send>> + Send + Sync>; +pub(crate) type AgentFilesystemPressureRecoveryCallback = Arc< + dyn Fn(MutationOperation, std::time::Instant) -> Pin + Send>> + + Send + + Sync, +>; + +pub(crate) const FILESYSTEM_MUTATION_MAX_ATTEMPTS: usize = 2; +pub(crate) const FILESYSTEM_MUTATION_RETRY_TIMEOUT: std::time::Duration = + std::time::Duration::from_millis(250); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MutationEffect { + ProvenNoEffect, + KnownCompletedPrefix { bytes: u64 }, + DesiredPostconditionSatisfied, + Unknown, +} + +pub(crate) fn proven_write_progress_effect(completed: usize) -> MutationEffect { + match u64::try_from(completed) { + Ok(0) => MutationEffect::ProvenNoEffect, + Ok(bytes) => MutationEffect::KnownCompletedPrefix { bytes }, + Err(_) => MutationEffect::Unknown, + } +} + +pub(crate) fn native_write_failure_effect( + error: &std::io::Error, + completed: usize, +) -> MutationEffect { + if write_error_proves_no_effect(error) { + proven_write_progress_effect(completed) + } else { + MutationEffect::Unknown + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MutationOperation { + Write, + Resize, + Create, + Metadata, +} + +#[derive(Debug)] +pub(crate) enum MutationFailure { + Guest(G), + StorageExhaustion { guest: G, quota_hint: bool }, + TransientGuest(G), + AccessGuest(G), + UnclassifiedGuest(G), + Io(std::io::Error), + Infrastructure(std::io::Error), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MutationDecision { + PreserveGuest(G), + Quota, + InsufficientSpace, + PhysicalPressure, + BoundedRetry, + PreserveRaw, + Success, + Invalidate, +} + +impl AgentFilesystemRuntime { + pub(crate) async fn classify_mutation_failure( + &self, + failure: MutationFailure, + effect: MutationEffect, + ) -> MutationDecision { + self.classify_mutation_failure_for(MutationOperation::Metadata, failure, effect) + .await + } + + pub(crate) async fn classify_mutation_failure_for( + &self, + operation: MutationOperation, + failure: MutationFailure, + effect: MutationEffect, + ) -> MutationDecision { + if effect != MutationEffect::ProvenNoEffect + && self.observe_usage_for_billing().await.is_err() + { + return self.invalidate().await; + } + if effect == MutationEffect::Unknown { + return self.invalidate().await; + } + if matches!(&failure, MutationFailure::Infrastructure(_)) + || matches!(&failure, MutationFailure::Io(error) if is_terminal_io(error)) + { + return self.invalidate().await; + } + if effect == MutationEffect::DesiredPostconditionSatisfied { + return MutationDecision::Success; + } + if let MutationFailure::Guest(error) = failure { + return MutationDecision::PreserveGuest(error); + } + + match failure { + MutationFailure::Guest(_) => { + unreachable!("guest failures return before classification") + } + MutationFailure::StorageExhaustion { guest, quota_hint } => { + match self.fresh_failure_observations().await { + Ok((usage, limits, capacity)) => { + let quota_exhausted = usage.zip(limits).is_some_and(|(usage, limits)| { + quota_exhausted(operation, usage, limits) + }); + let physical_exhausted = + self.inner.pressure.pressure(operation, capacity).is_some(); + if quota_hint || quota_exhausted { + MutationDecision::Quota + } else if physical_exhausted { + MutationDecision::PhysicalPressure + } else { + tracing::warn!( + operation = ?operation, + quota_hint, + "Filesystem storage exhaustion was not explained by fresh quota or capacity observations" + ); + MutationDecision::PreserveGuest(guest) + } + } + Err(error) if error.is_terminal_failure() => self.invalidate().await, + Err(_) if quota_hint => MutationDecision::PreserveGuest(guest), + Err(_) => MutationDecision::PreserveGuest(guest), + } + } + MutationFailure::TransientGuest(guest) => { + match self.fresh_failure_observations().await { + Ok(_) if self.retry_permitted().await => MutationDecision::BoundedRetry, + Ok(_) => MutationDecision::PreserveGuest(guest), + Err(error) if error.is_terminal_failure() => self.invalidate().await, + Err(_) => MutationDecision::PreserveGuest(guest), + } + } + MutationFailure::AccessGuest(guest) => match self.fresh_failure_observations().await { + Ok(_) => MutationDecision::PreserveGuest(guest), + Err(_) => self.invalidate().await, + }, + MutationFailure::UnclassifiedGuest(guest) => { + match self.fresh_failure_observations().await { + Ok(_) if self.retry_permitted().await => MutationDecision::BoundedRetry, + Ok(_) => MutationDecision::PreserveGuest(guest), + Err(_) => self.invalidate().await, + } + } + MutationFailure::Infrastructure(_) => self.invalidate().await, + MutationFailure::Io(error) if is_terminal_io(&error) => self.invalidate().await, + MutationFailure::Io(error) if is_storage_exhaustion(&error) => { + match self.fresh_failure_observations().await { + Ok((usage, limits, capacity)) => { + let quota_exhausted = usage.zip(limits).is_some_and(|(usage, limits)| { + quota_exhausted(operation, usage, limits) + }); + let physical_exhausted = + self.inner.pressure.pressure(operation, capacity).is_some(); + if is_quota_error(&error) || quota_exhausted { + MutationDecision::Quota + } else if physical_exhausted { + MutationDecision::PhysicalPressure + } else { + tracing::warn!( + operation = ?operation, + raw_os_error = ?error.raw_os_error(), + "Filesystem storage exhaustion was not explained by fresh quota or capacity observations" + ); + MutationDecision::InsufficientSpace + } + } + Err(observation_error) if observation_error.is_terminal_failure() => { + self.invalidate().await + } + Err(_) if is_quota_error(&error) => MutationDecision::Quota, + Err(_) => MutationDecision::InsufficientSpace, + } + } + MutationFailure::Io(error) if is_transient_io(&error) => { + match self.fresh_failure_observations().await { + Ok(_) if self.retry_permitted().await => MutationDecision::BoundedRetry, + Ok(_) => MutationDecision::PreserveRaw, + Err(observation_error) if observation_error.is_terminal_failure() => { + self.invalidate().await + } + Err(_) => MutationDecision::PreserveRaw, + } + } + MutationFailure::Io(error) if is_guest_scoped_io(&error) => { + match self.fresh_failure_observations().await { + Ok(_) => MutationDecision::PreserveRaw, + Err(_) => self.invalidate().await, + } + } + MutationFailure::Io(_) => match self.fresh_failure_observations().await { + Ok(_) if self.retry_permitted().await => MutationDecision::BoundedRetry, + Ok(_) => MutationDecision::PreserveRaw, + Err(_) => self.invalidate().await, + }, + } + } + + pub(crate) fn set_invalidation_callback( + &self, + callback: Option, + ) { + *self + .inner + .invalidated + .lock() + .expect("agent filesystem invalidation callback lock poisoned") = callback; + } + + pub(crate) fn set_retry_callback(&self, callback: Option) { + *self + .inner + .retry_permitted + .lock() + .expect("agent filesystem retry callback lock poisoned") = callback; + } + + pub(crate) fn set_pressure_recovery_callback( + &self, + callback: Option, + ) { + *self + .inner + .pressure_recovery + .lock() + .expect("agent filesystem pressure callback lock poisoned") = callback; + } + + pub(crate) async fn recover_physical_pressure( + &self, + operation: MutationOperation, + deadline: std::time::Instant, + ) -> bool { + if std::time::Instant::now() >= deadline { + return false; + } + if !self.retry_permitted().await { + return false; + } + let callback = self + .inner + .pressure_recovery + .lock() + .expect("agent filesystem pressure callback lock poisoned") + .clone(); + match callback { + Some(callback) if std::time::Instant::now() < deadline => tokio::time::timeout_at( + tokio::time::Instant::from_std(deadline), + callback(operation, deadline), + ) + .await + .unwrap_or(false), + Some(_) | None => false, + } + } + + async fn retry_permitted(&self) -> bool { + if self.inner.state.load(Ordering::Acquire) & super::mutation::FILESYSTEM_RUNTIME_SEALED + != 0 + { + return false; + } + let callback = self + .inner + .retry_permitted + .lock() + .expect("agent filesystem retry callback lock poisoned") + .clone(); + match callback { + Some(callback) => callback().await, + None => true, + } + } + + async fn invalidate(&self) -> MutationDecision { + self.seal(); + if !self + .inner + .invalidation_notified + .swap(true, Ordering::AcqRel) + { + let callback = self + .inner + .invalidated + .lock() + .expect("agent filesystem invalidation callback lock poisoned") + .clone(); + if let Some(callback) = callback { + callback().await; + } + } + MutationDecision::Invalidate + } + + pub(crate) async fn invalidate_runtime(&self) { + let _: MutationDecision<()> = self.invalidate().await; + } +} + +fn quota_exhausted( + operation: MutationOperation, + usage: AgentFilesystemUsage, + limits: ResolvedAgentFilesystemLimits, +) -> bool { + let bytes_exhausted = usage.allocated_bytes >= limits.allocated_bytes; + let objects_exhausted = usage.filesystem_objects >= limits.filesystem_objects; + match operation { + MutationOperation::Write | MutationOperation::Resize | MutationOperation::Metadata => { + bytes_exhausted + } + MutationOperation::Create => bytes_exhausted || objects_exhausted, + } +} + +fn is_storage_exhaustion(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::StorageFull | std::io::ErrorKind::QuotaExceeded + ) || is_enospc(error) + || is_edquot(error) +} + +fn is_quota_error(error: &std::io::Error) -> bool { + error.kind() == std::io::ErrorKind::QuotaExceeded || is_edquot(error) +} + +fn is_transient_io(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted + ) || is_ebusy(error) + || is_eagain(error) +} + +fn write_error_proves_no_effect(error: &std::io::Error) -> bool { + error.kind() == std::io::ErrorKind::WouldBlock + || is_ebusy(error) + || is_storage_exhaustion(error) + || is_guest_scoped_io(error) +} + +fn is_guest_scoped_io(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::NotFound + | std::io::ErrorKind::AlreadyExists + | std::io::ErrorKind::InvalidInput + | std::io::ErrorKind::InvalidFilename + | std::io::ErrorKind::IsADirectory + | std::io::ErrorKind::NotADirectory + | std::io::ErrorKind::FileTooLarge + ) +} + +fn is_terminal_io(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::PermissionDenied | std::io::ErrorKind::ReadOnlyFilesystem + ) || is_eio(error) + || is_estale(error) + || is_enodev(error) +} + +#[cfg(target_os = "linux")] +fn is_enospc(error: &std::io::Error) -> bool { + error.raw_os_error() == Some(libc::ENOSPC) +} + +#[cfg(not(target_os = "linux"))] +fn is_enospc(_error: &std::io::Error) -> bool { + false +} + +#[cfg(target_os = "linux")] +fn is_edquot(error: &std::io::Error) -> bool { + error.raw_os_error() == Some(libc::EDQUOT) +} + +#[cfg(not(target_os = "linux"))] +fn is_edquot(_error: &std::io::Error) -> bool { + false +} + +#[cfg(target_os = "linux")] +fn is_ebusy(error: &std::io::Error) -> bool { + error.raw_os_error() == Some(libc::EBUSY) +} + +#[cfg(not(target_os = "linux"))] +fn is_ebusy(_error: &std::io::Error) -> bool { + false +} + +#[cfg(target_os = "linux")] +fn is_eagain(error: &std::io::Error) -> bool { + error.raw_os_error() == Some(libc::EAGAIN) +} + +#[cfg(not(target_os = "linux"))] +fn is_eagain(_error: &std::io::Error) -> bool { + false +} + +#[cfg(target_os = "linux")] +fn is_eio(error: &std::io::Error) -> bool { + error.raw_os_error() == Some(libc::EIO) +} + +#[cfg(not(target_os = "linux"))] +fn is_eio(_error: &std::io::Error) -> bool { + false +} + +#[cfg(target_os = "linux")] +fn is_estale(error: &std::io::Error) -> bool { + error.raw_os_error() == Some(libc::ESTALE) +} + +#[cfg(not(target_os = "linux"))] +fn is_estale(_error: &std::io::Error) -> bool { + false +} + +#[cfg(target_os = "linux")] +fn is_enodev(error: &std::io::Error) -> bool { + error.raw_os_error() == Some(libc::ENODEV) +} + +#[cfg(not(target_os = "linux"))] +fn is_enodev(_error: &std::io::Error) -> bool { + false +} diff --git a/golem-worker-executor/src/services/agent_filesystem/initial_files.rs b/golem-worker-executor/src/services/agent_filesystem/initial_files.rs new file mode 100644 index 0000000000..ba4a4e78a0 --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/initial_files.rs @@ -0,0 +1,488 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::backend::InitialFileMaterialization; +use super::*; +use crate::services::file_loader::InitialFileSource; +use golem_common::model::agent::AgentFileContentHash; +use golem_common::model::component::AgentFilePermissions; +use golem_common::model::environment::EnvironmentId; + +impl AgentFilesystemRuntime { + pub(crate) fn is_read_only(&self, path: &Path) -> bool { + self.is_read_only_path(path, true) + } + + pub(crate) fn is_read_only_path(&self, path: &Path, follow_final_symlink: bool) -> bool { + let path = resolve_policy_path(path, follow_final_symlink); + self.inner + .initial_files + .read() + .expect("initial-files policy lock poisoned") + .iter() + .any(|(initial_path, file)| { + file.permissions == AgentFilePermissions::ReadOnly + && resolve_policy_path(initial_path, true) == path + }) + } + + pub(crate) fn contains_read_only_path(&self, path: &Path, follow_final_symlink: bool) -> bool { + let path = resolve_policy_path(path, follow_final_symlink); + self.inner + .initial_files + .read() + .expect("initial-files policy lock poisoned") + .iter() + .any(|(initial_path, file)| { + let initial_path = resolve_policy_path(initial_path, true); + file.permissions == AgentFilePermissions::ReadOnly + && (initial_path == path || initial_path.starts_with(&path)) + }) + } + + pub(crate) async fn replace_initial_files( + &self, + file_loader: &FileLoader, + environment_id: EnvironmentId, + files: &[InitialAgentFile], + ) -> Result<(), FilesystemStorageError> { + let effect = self.begin_update_effect().await.map_err(|error| { + FilesystemStorageError::io( + "admit initial-file materialization", + self.inner.backend.root(), + std::io::Error::other(error), + ) + })?; + let mut materialized = HashMap::new(); + let mut sources: HashMap = HashMap::new(); + for file in files { + let target = self.inner.backend.root().join(file.path.to_rel_string()); + let source = match sources.get(&file.content_hash) { + Some(source) if source.size() == file.size => source, + Some(_) => { + return Err(FilesystemStorageError::verification( + "verify consistent initial-file source size", + &target, + )); + } + None => { + let source = file_loader + .get_source(environment_id, file.content_hash, file.size) + .await + .map_err(|error| { + FilesystemStorageError::io( + "load verified initial-file source", + &target, + std::io::Error::other(error), + ) + })?; + sources.entry(file.content_hash).or_insert(source) + } + }; + self.inner + .backend + .materialize_initial_file(InitialFileMaterialization { + materialization_root: self.inner.backend.root().to_path_buf(), + source: source.clone(), + target: target.clone(), + read_only: file.permissions == AgentFilePermissions::ReadOnly, + effect: effect.clone(), + staging: None, + }) + .await?; + if materialized.insert(target.clone(), file.clone()).is_some() { + return Err(FilesystemStorageError::verification( + "materialize unique initial-file target", + &target, + )); + } + } + *self + .inner + .initial_files + .write() + .expect("initial-files policy lock poisoned") = materialized; + Ok(()) + } + + pub(crate) async fn update_initial_files( + &self, + file_loader: &FileLoader, + environment_id: EnvironmentId, + files: &[InitialAgentFile], + ) -> Result { + let effect = self.begin_update_effect().await.map_err(|error| { + FilesystemStorageError::io( + "admit initial-file update", + self.inner.backend.root(), + std::io::Error::other(error), + ) + })?; + let current = self + .inner + .initial_files + .read() + .expect("initial-files policy lock poisoned") + .clone(); + let update_result = async { + let mut desired = HashMap::new(); + for file in files { + let target = self.inner.backend.root().join(file.path.to_rel_string()); + if desired.insert(target.clone(), file.clone()).is_some() { + return Err(FilesystemStorageError::verification( + "materialize unique initial-file update target", + &target, + )); + } + } + + for (path, file) in &desired { + if current.get(path).is_some_and(|existing| { + existing.permissions == AgentFilePermissions::ReadWrite + && file.permissions == AgentFilePermissions::ReadOnly + }) { + return Err(FilesystemStorageError::verification( + "replace read-write initial file with read-only content", + path, + )); + } + } + + let staging = Arc::new(self.inner.backend.create_staging_dir().map_err(|error| { + FilesystemStorageError::io( + "create initial-file update staging directory", + self.inner.backend.root(), + error, + ) + })?); + let mut sources: HashMap = HashMap::new(); + let mut staged = Vec::new(); + for (path, file) in &desired { + match current.get(path) { + Some(existing) + if existing.permissions == AgentFilePermissions::ReadWrite + && file.permissions == AgentFilePermissions::ReadWrite => {} + Some(existing) + if existing.permissions == AgentFilePermissions::ReadOnly + && existing.content_hash == file.content_hash + && file.permissions == AgentFilePermissions::ReadOnly => {} + None if file.permissions == AgentFilePermissions::ReadWrite + && std::fs::symlink_metadata(path) + .is_ok_and(|metadata| metadata.is_file()) => {} + _ => { + let source = match sources.get(&file.content_hash) { + Some(source) if source.size() == file.size => source, + Some(_) => { + return Err(FilesystemStorageError::verification( + "verify consistent initial-file update source size", + path, + )); + } + None => { + let source = file_loader + .get_source(environment_id, file.content_hash, file.size) + .await + .map_err(|error| { + FilesystemStorageError::io( + "load verified initial-file update source", + path, + std::io::Error::other(error), + ) + })?; + sources.entry(file.content_hash).or_insert(source) + } + }; + let staged_path = staging.path().join(staged.len().to_string()); + self.inner + .backend + .materialize_initial_file(InitialFileMaterialization { + materialization_root: staging.path().to_path_buf(), + source: source.clone(), + target: staged_path.clone(), + read_only: file.permissions == AgentFilePermissions::ReadOnly, + effect: effect.clone(), + staging: Some(Arc::clone(&staging)), + }) + .await?; + staged.push((path.clone(), staged_path)); + } + } + } + + let backup_root = staging.path().join("backups"); + std::fs::create_dir(&backup_root).map_err(|error| { + FilesystemStorageError::io( + "create initial-file update backup directory", + &backup_root, + error, + ) + })?; + let mut replacements: Vec = + staged.iter().map(|(path, _)| path.clone()).collect(); + replacements.extend( + current + .iter() + .filter(|(path, existing)| { + existing.permissions == AgentFilePermissions::ReadOnly + && !desired.contains_key(*path) + }) + .map(|(path, _)| path.clone()), + ); + replacements.sort(); + replacements.dedup(); + + let mut transaction = InitialFileUpdateTransaction::new(backup_root); + for path in &replacements { + if let Err(error) = transaction + .create_parent(self.inner.backend.root(), path) + .and_then(|_| validate_replaceable_target(path)) + { + return Err(transaction.fail( + "prepare initial-file update target", + path, + error, + )); + } + let exists = std::fs::symlink_metadata(path).is_ok(); + if exists && !current.contains_key(path) { + return Err(transaction.fail( + "preserve existing guest filesystem target", + path, + std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "initial-file update target already exists", + ), + )); + } + if exists && let Err(error) = transaction.back_up(path) { + return Err(transaction.fail( + "stage existing initial-file target", + path, + error, + )); + } + } + + for (target, staged_path) in &staged { + if let Err(error) = transaction.install(staged_path, target) { + return Err(transaction.fail( + "install initial-file update target", + target, + error, + )); + } + } + + transaction.commit(); + *self + .inner + .initial_files + .write() + .expect("initial-files policy lock poisoned") = desired; + let staging_path = staging.path().to_path_buf(); + let staging = Arc::try_unwrap(staging) + .expect("initial-file staging work must finish before commit cleanup"); + if let Err(error) = staging.close() { + let error = FilesystemStorageError::cleanup_io( + "remove initial-file update staging directory", + &staging_path, + error, + ); + self.seal(); + return Err(error); + } + Ok::<(), FilesystemStorageError>(()) + } + .await; + update_result?; + Ok(effect) + } +} + +fn validate_replaceable_target(path: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_file() || metadata.file_type().is_symlink() => Ok(()), + Ok(metadata) if metadata.is_dir() => { + if std::fs::read_dir(path)?.next().is_some() { + Err(std::io::Error::new( + std::io::ErrorKind::DirectoryNotEmpty, + "initial-file target directory is not empty", + )) + } else { + Ok(()) + } + } + Ok(_) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "initial-file target is not replaceable", + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +pub(super) struct InitialFileUpdateTransaction { + backup_root: PathBuf, + backups: Vec<(PathBuf, PathBuf)>, + installed: Vec, + created_directories: Vec, + committed: bool, +} + +impl InitialFileUpdateTransaction { + pub(super) fn new(backup_root: PathBuf) -> Self { + Self { + backup_root, + backups: Vec::new(), + installed: Vec::new(), + created_directories: Vec::new(), + committed: false, + } + } + + fn create_parent(&mut self, root: &Path, target: &Path) -> std::io::Result<()> { + let parent = target.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "initial-file target has no parent", + ) + })?; + let relative = parent.strip_prefix(root).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "initial-file target escapes the agent filesystem", + ) + })?; + let mut current = root.to_path_buf(); + for component in relative.components() { + let std::path::Component::Normal(component) = component else { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "initial-file target contains an invalid path component", + )); + }; + current.push(component); + match std::fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + Ok(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "initial-file parent is not a directory", + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir(¤t)?; + self.created_directories.push(current.clone()); + } + Err(error) => return Err(error), + } + } + Ok(()) + } + + pub(super) fn back_up(&mut self, path: &Path) -> std::io::Result<()> { + let backup = self.backup_root.join(self.backups.len().to_string()); + std::fs::rename(path, &backup)?; + self.backups.push((path.to_path_buf(), backup)); + Ok(()) + } + + pub(super) fn install(&mut self, staged: &Path, target: &Path) -> std::io::Result<()> { + std::fs::rename(staged, target)?; + self.installed.push(target.to_path_buf()); + Ok(()) + } + + fn fail( + &mut self, + operation: &'static str, + path: &Path, + error: std::io::Error, + ) -> FilesystemStorageError { + match self.rollback() { + Ok(()) => FilesystemStorageError::io(operation, path, error), + Err((rollback_path, rollback_error)) => FilesystemStorageError::cleanup_io( + "roll back failed initial-file update", + &rollback_path, + rollback_error, + ), + } + } + + fn rollback(&mut self) -> Result<(), (PathBuf, std::io::Error)> { + let mut failure = None; + for path in self.installed.drain(..).rev() { + let result = match std::fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_dir() => std::fs::remove_dir(&path), + Ok(_) => std::fs::remove_file(&path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + }; + if let Err(error) = result { + failure = Some((path, error)); + } + } + for (original, backup) in self.backups.drain(..).rev() { + if let Err(error) = std::fs::rename(&backup, &original) { + failure = Some((original, error)); + } + } + for path in self.created_directories.drain(..).rev() { + if let Err(error) = std::fs::remove_dir(&path) + && error.kind() != std::io::ErrorKind::NotFound + { + failure = Some((path, error)); + } + } + self.committed = true; + failure.map_or(Ok(()), Err) + } + + fn commit(&mut self) { + self.committed = true; + } +} + +impl Drop for InitialFileUpdateTransaction { + fn drop(&mut self) { + if !self.committed { + let _ = self.rollback(); + } + } +} + +fn resolve_policy_path(path: &Path, follow_final_symlink: bool) -> PathBuf { + if !follow_final_symlink && let (Some(parent), Some(name)) = (path.parent(), path.file_name()) { + return resolve_policy_path(parent, true).join(name); + } + let mut unresolved = Vec::new(); + let mut current = path; + loop { + match std::fs::canonicalize(current) { + Ok(mut resolved) => { + for component in unresolved.iter().rev() { + resolved.push(component); + } + return resolved; + } + Err(_) => match (current.parent(), current.file_name()) { + (Some(parent), Some(name)) => { + unresolved.push(name.to_os_string()); + current = parent; + } + _ => return path.to_path_buf(), + }, + } + } +} diff --git a/golem-worker-executor/src/services/agent_filesystem/mod.rs b/golem-worker-executor/src/services/agent_filesystem/mod.rs new file mode 100644 index 0000000000..20d860a79a --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/mod.rs @@ -0,0 +1,765 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::metrics::workers::record_agent_filesystem_lifecycle; +use crate::services::file_loader::FileLoader; +use crate::services::golem_config::{ + FilesystemObjectLimitPolicyConfig, FilesystemPressureConfig, FilesystemStorageConfig, +}; +use crate::services::resource_limits::AtomicResourceEntry; +use backend::{AgentFilesystemBackend, FilesystemBackendProvisioner}; +use golem_common::model::component::InitialAgentFile; +use golem_common::model::{OwnedAgentId, RetryConfig}; +use golem_common::retries::RetryState; +use std::collections::HashMap; +use std::fmt::{Display, Formatter}; +use std::path::{Path, PathBuf}; +#[cfg(test)] +use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::{Arc, OnceLock, Weak}; +use std::time::Instant; +use tokio::sync::{Mutex, OwnedMutexGuard}; + +mod backend; +#[allow( + dead_code, + reason = "mutation classification is exposed for filesystem host adapters" +)] +mod failure; +mod initial_files; +mod mutation; +mod postcondition; +mod quota; +mod unmanaged; + +#[cfg(target_os = "linux")] +mod xfs; + +#[allow( + unused_imports, + reason = "mutation classification is exposed for filesystem host adapters" +)] +pub(crate) use failure::{ + AgentFilesystemInvalidationCallback, AgentFilesystemPressureRecoveryCallback, + AgentFilesystemRetryCallback, FILESYSTEM_MUTATION_MAX_ATTEMPTS, + FILESYSTEM_MUTATION_RETRY_TIMEOUT, MutationDecision, MutationEffect, MutationFailure, + MutationOperation, native_write_failure_effect, proven_write_progress_effect, +}; +pub(crate) use mutation::{ + AgentFilesystemEffectAdmission, AgentFilesystemEffectLease, AgentFilesystemUpdateEffectLease, + ClassifiedFileOutputStream, FilesystemStreamMode, NativeMutationGuestError, NativeOpenOptions, + NativeOpenResult, classified_filesystem_stream_error_code, create_directory, hard_link, open, + remove_directory, rename, resize_file, run_blocking_filesystem_mutation, set_descriptor_times, + set_path_times, symlink, sync_descriptor, unlink_file, validate_descriptor_times, + validate_directory_mutation, validate_open, validate_resize, validate_two_directory_mutation, +}; +#[allow( + unused_imports, + reason = "shared probe vocabulary is consumed across host adapters and tests" +)] +pub(crate) use postcondition::{ + MutationPostcondition, ObjectIdentity, PathObjectType, PathState, RequestedTime, SymlinkState, + TimesState, create_directory_postcondition, descriptor_state, descriptor_times, + link_postcondition, open_postcondition, path_state, path_state_with_follow, path_times, + remove_postcondition, rename_postcondition, resize_postcondition, same_object, + same_optional_object, state_postcondition, symlink_postcondition, symlink_state, + times_postcondition, +}; +use quota::FilesystemLimitExceededCallback; +pub(crate) use quota::{ + AgentFilesystemStorageLimit, AgentFilesystemUsage, FilesystemCapacity, + ResolvedAgentFilesystemLimits, +}; + +#[cfg(test)] +use initial_files::InitialFileUpdateTransaction; +#[cfg(test)] +use quota::FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION; + +#[cfg(test)] +mod tests; + +static LIFECYCLE_LOCKS: OnceLock>>>> = + OnceLock::new(); + +#[derive(Debug)] +pub struct FilesystemStorageError { + operation: &'static str, + path: PathBuf, + source: Option, + cleanup_failed: bool, +} + +impl FilesystemStorageError { + fn io(operation: &'static str, path: &Path, source: std::io::Error) -> Self { + Self { + operation, + path: path.to_path_buf(), + source: Some(source), + cleanup_failed: false, + } + } + + fn verification(operation: &'static str, path: &Path) -> Self { + Self { + operation, + path: path.to_path_buf(), + source: None, + cleanup_failed: false, + } + } + + pub(crate) fn resource_billing_transition(operation: &'static str) -> Self { + Self::verification(operation, Path::new("")) + } + + fn cleanup_io(operation: &'static str, path: &Path, source: std::io::Error) -> Self { + Self { + operation, + path: path.to_path_buf(), + source: Some(source), + cleanup_failed: true, + } + } + + fn cleanup_verification(operation: &'static str, path: &Path) -> Self { + Self { + operation, + path: path.to_path_buf(), + source: None, + cleanup_failed: true, + } + } + + pub(crate) fn cleanup_failed(&self) -> bool { + self.cleanup_failed + } + + pub(crate) fn is_storage_exhaustion(&self) -> bool { + self.source.as_ref().is_some_and(|source| { + matches!( + source.kind(), + std::io::ErrorKind::StorageFull | std::io::ErrorKind::QuotaExceeded + ) + }) + } + + fn is_terminal_failure(&self) -> bool { + self.source.as_ref().is_some_and(|source| { + matches!( + source.kind(), + std::io::ErrorKind::InvalidData + | std::io::ErrorKind::PermissionDenied + | std::io::ErrorKind::ReadOnlyFilesystem + ) || is_terminal_storage_errno(source) + }) + } +} + +#[cfg(target_os = "linux")] +fn is_terminal_storage_errno(error: &std::io::Error) -> bool { + matches!(error.raw_os_error(), Some(errno) if matches!(errno, libc::EIO | libc::ESTALE | libc::ENODEV)) +} + +#[cfg(not(target_os = "linux"))] +fn is_terminal_storage_errno(_error: &std::io::Error) -> bool { + false +} + +impl Display for FilesystemStorageError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "failed to {} agent filesystem {}", + self.operation, + self.path.display() + )?; + if let Some(source) = &self.source { + write!(f, ": {source}")?; + } + Ok(()) + } +} + +impl std::error::Error for FilesystemStorageError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source + .as_ref() + .map(|source| source as &(dyn std::error::Error + 'static)) + } +} + +#[derive(Clone)] +pub(crate) struct AgentFilesystems { + provisioner: Arc, + pressure: FilesystemPressureConfig, +} + +pub(crate) struct CreateAgentFilesystem { + pub agent_id: OwnedAgentId, + pub initial_files: Vec, + pub file_loader: Arc, + pub resource_limits: Option>, + pub limit_exceeded: Option, +} + +impl AgentFilesystems { + pub(crate) fn new(settings: &FilesystemStorageConfig) -> Result { + Ok(Self { + provisioner: backend::configured_provisioner(settings)?, + pressure: settings.pressure.clone(), + }) + } + + pub(crate) fn initial_file_cache_root(&self) -> Option<&Path> { + self.provisioner.initial_file_cache_root() + } + + pub(crate) async fn create_fresh( + &self, + request: CreateAgentFilesystem, + ) -> Result { + let mut filesystem = self.create_owned_empty(&request.agent_id).await?; + filesystem + .runtime + .set_limit_exceeded_callback(request.limit_exceeded); + if let Some(resource_limits) = request.resource_limits { + if let Err(error) = resource_limits + .register_agent_filesystem(request.agent_id.clone(), filesystem.runtime()) + .await + { + return Err(rollback_owned_filesystem(filesystem, error).await); + } + filesystem.limit_registration = Some(AgentFilesystemLimitRegistration { + resource_limits, + agent_id: request.agent_id.clone(), + runtime: filesystem.runtime(), + }); + } + if let Err(error) = filesystem + .runtime + .replace_initial_files( + &request.file_loader, + request.agent_id.environment_id, + &request.initial_files, + ) + .await + { + if error.is_storage_exhaustion() { + filesystem.runtime.notify_limit_state(true).await; + } + return Err(rollback_owned_filesystem(filesystem, error).await); + } + Ok(filesystem) + } + + async fn create_owned_empty( + &self, + agent_id: &OwnedAgentId, + ) -> Result { + let started = Instant::now(); + let result = self + .provisioner + .provision_for(agent_id) + .await + .map(|provisioned| AgentFilesystem::new(provisioned, self.pressure.clone())); + record_agent_filesystem_lifecycle("create", result.is_ok(), started.elapsed()); + result + } +} + +pub(crate) struct AgentFilesystem { + runtime: AgentFilesystemRuntime, + provisioned: Option, + limit_registration: Option, +} + +struct AgentFilesystemLimitRegistration { + resource_limits: Arc, + agent_id: OwnedAgentId, + runtime: AgentFilesystemRuntime, +} + +impl Drop for AgentFilesystemLimitRegistration { + fn drop(&mut self) { + self.resource_limits + .unregister_agent_filesystem(&self.agent_id, &self.runtime); + } +} + +impl AgentFilesystem { + fn new( + provisioned: backend::ProvisionedAgentFilesystem, + pressure: FilesystemPressureConfig, + ) -> Self { + let runtime_backend = Arc::clone(provisioned.backend()); + Self { + runtime: AgentFilesystemRuntime::new(runtime_backend, pressure), + provisioned: Some(provisioned), + limit_registration: None, + } + } + + pub(crate) fn path(&self) -> &Path { + self.runtime.inner.backend.root() + } + + pub(crate) fn runtime(&self) -> AgentFilesystemRuntime { + self.runtime.clone() + } + + pub(crate) fn seal(&self) { + self.runtime.seal(); + } + + pub(crate) async fn close_and_delete(self) -> Result<(), FilesystemStorageError> { + let path = self.path().to_path_buf(); + tokio::spawn(async move { self.delete_after_drain().await }) + .await + .map_err(|error| { + FilesystemStorageError::io( + "complete agent filesystem deletion", + &path, + std::io::Error::other(error), + ) + })? + } + + async fn delete_after_drain(mut self) -> Result<(), FilesystemStorageError> { + let started = Instant::now(); + self.limit_registration.take(); + self.seal(); + self.runtime.drain().await; + let result = match self.provisioned.as_mut() { + Some(provisioned) => provisioned.delete().await, + None => Ok(()), + }; + record_agent_filesystem_lifecycle("delete", result.is_ok(), started.elapsed()); + result + } +} + +impl Drop for AgentFilesystem { + fn drop(&mut self) { + let Some(provisioned) = self.provisioned.take() else { + return; + }; + self.limit_registration.take(); + self.runtime.seal(); + if self.runtime.has_active_effects() { + let runtime = self.runtime.clone(); + std::thread::spawn(move || { + while runtime.has_active_effects() { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + drop(provisioned); + }); + } else { + drop(provisioned); + } + } +} + +#[derive(Clone)] +/// Cloneable handle to the synchronization and backend state shared by filesystem +/// adapters, completion tasks, quota enforcement, and usage sampling. +pub struct AgentFilesystemRuntime { + inner: Arc, +} + +impl std::fmt::Debug for AgentFilesystemRuntime { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("AgentFilesystemRuntime") + .finish_non_exhaustive() + } +} + +struct AgentFilesystemRuntimeInner { + state: AtomicUsize, + usage_sampling: AtomicBool, + usage_effect_epoch: std::sync::atomic::AtomicU64, + last_effect_completion_millis: std::sync::atomic::AtomicU64, + drained: tokio::sync::Notify, + admission_resumed: tokio::sync::Notify, + append: Arc>, + namespace: Arc>, + operations: Arc>, + backend: Arc, + initial_files: std::sync::RwLock>, + pressure: FilesystemPressureConfig, + applied_limits: std::sync::RwLock>, + limit_exceeded: std::sync::Mutex>, + #[allow( + dead_code, + reason = "runtime invalidation is exposed for filesystem host adapters" + )] + invalidation_notified: AtomicBool, + #[allow( + dead_code, + reason = "runtime invalidation is exposed for filesystem host adapters" + )] + invalidated: std::sync::Mutex>, + retry_permitted: std::sync::Mutex>, + pressure_recovery: std::sync::Mutex>, + usage_observer: std::sync::Mutex< + Option>, + >, + #[cfg(test)] + failure_observations: + std::sync::RwLock, FilesystemCapacity)>>, + #[cfg(test)] + usage_observation_fails: AtomicBool, +} + +impl AgentFilesystemRuntime { + fn new(backend: Arc, pressure: FilesystemPressureConfig) -> Self { + Self { + inner: Arc::new(AgentFilesystemRuntimeInner { + state: AtomicUsize::new(0), + usage_sampling: AtomicBool::new(false), + usage_effect_epoch: std::sync::atomic::AtomicU64::new(0), + last_effect_completion_millis: std::sync::atomic::AtomicU64::new(0), + drained: tokio::sync::Notify::new(), + admission_resumed: tokio::sync::Notify::new(), + append: Arc::new(Mutex::new(())), + namespace: Arc::new(Mutex::new(())), + operations: Arc::new(tokio::sync::RwLock::new(())), + backend, + initial_files: std::sync::RwLock::new(HashMap::new()), + pressure, + applied_limits: std::sync::RwLock::new(None), + limit_exceeded: std::sync::Mutex::new(None), + invalidation_notified: AtomicBool::new(false), + invalidated: std::sync::Mutex::new(None), + retry_permitted: std::sync::Mutex::new(None), + pressure_recovery: std::sync::Mutex::new(None), + usage_observer: std::sync::Mutex::new(None), + #[cfg(test)] + failure_observations: std::sync::RwLock::new(None), + #[cfg(test)] + usage_observation_fails: AtomicBool::new(false), + }), + } + } + + #[cfg(test)] + pub(crate) fn new_for_test() -> Self { + Self::new_for_test_with_observations( + None, + None, + FilesystemCapacity { + total_bytes: 1, + available_bytes: 1, + total_filesystem_objects: 1, + available_filesystem_objects: 1, + }, + ) + } + + #[cfg(test)] + pub(crate) fn new_for_test_with_observations( + usage: Option, + limits: Option, + capacity: FilesystemCapacity, + ) -> Self { + let runtime = Self::new( + Arc::new(unmanaged::UnmanagedAgentFilesystem::new(PathBuf::from( + "", + ))), + FilesystemPressureConfig { + minimum_available_bytes: 0, + target_available_bytes: 0, + minimum_available_filesystem_objects: 0, + target_available_filesystem_objects: 0, + ..FilesystemPressureConfig::default() + }, + ); + *runtime + .inner + .applied_limits + .write() + .expect("agent filesystem applied-limit lock poisoned") = limits; + *runtime + .inner + .failure_observations + .write() + .expect("agent filesystem test observation lock poisoned") = Some((usage, capacity)); + runtime + } + + #[cfg(test)] + pub(crate) fn new_for_test_with_failed_observations() -> Self { + let runtime = Self::new_for_test_with_capacity_observation_failure(); + runtime + .inner + .usage_observation_fails + .store(true, Ordering::Release); + runtime + } + + #[cfg(test)] + pub(crate) fn new_for_test_with_capacity_observation_failure() -> Self { + Self::new( + Arc::new(unmanaged::UnmanagedAgentFilesystem::new(PathBuf::from( + "", + ))), + FilesystemPressureConfig { + minimum_available_bytes: 0, + target_available_bytes: 0, + minimum_available_filesystem_objects: 0, + target_available_filesystem_objects: 0, + ..FilesystemPressureConfig::default() + }, + ) + } +} + +pub(super) fn create_materialization_parent<'a>( + root: &Path, + target: &'a Path, +) -> std::io::Result<&'a Path> { + let parent = target.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "initial-file target has no parent", + ) + })?; + let relative = parent.strip_prefix(root).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "initial-file target escapes the agent filesystem", + ) + })?; + let mut current = root.to_path_buf(); + for component in relative.components() { + let std::path::Component::Normal(component) = component else { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "initial-file target contains an invalid path component", + )); + }; + current.push(component); + match std::fs::symlink_metadata(¤t) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + Ok(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "initial-file parent is not a directory", + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir(¤t)?; + } + Err(error) => return Err(error), + } + } + Ok(parent) +} + +pub(super) fn set_initial_file_permissions( + file: &std::fs::File, + read_only: bool, +) -> std::io::Result<()> { + let mut permissions = file.metadata()?.permissions(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + permissions.set_mode(if read_only { 0o444 } else { 0o644 }); + } + #[cfg(not(unix))] + permissions.set_readonly(read_only); + file.set_permissions(permissions) +} + +pub(super) async fn acquire_lifecycle_lock(path: &Path) -> OwnedMutexGuard<()> { + let lock = { + let mut locks = LIFECYCLE_LOCKS + .get_or_init(|| std::sync::Mutex::new(HashMap::new())) + .lock() + .expect("agent filesystem lifecycle lock registry poisoned"); + locks.retain(|_, lock| lock.strong_count() > 0); + match locks.get(path).and_then(Weak::upgrade) { + Some(lock) => lock, + None => { + let lock = Arc::new(Mutex::new(())); + locks.insert(path.to_path_buf(), Arc::downgrade(&lock)); + lock + } + } + }; + lock.lock_owned().await +} + +pub(super) async fn verify_fresh_directory(path: &Path) -> Result<(), FilesystemStorageError> { + let metadata = tokio::fs::symlink_metadata(path) + .await + .map_err(|error| FilesystemStorageError::io("verify runtime directory", path, error))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(FilesystemStorageError::verification( + "verify fresh runtime directory", + path, + )); + } + + let mut entries = tokio::fs::read_dir(path).await.map_err(|error| { + FilesystemStorageError::io("verify empty runtime directory", path, error) + })?; + let empty = entries + .next_entry() + .await + .map_err(|error| FilesystemStorageError::io("verify empty runtime directory", path, error))? + .is_none(); + if !empty { + return Err(FilesystemStorageError::verification( + "verify empty runtime directory", + path, + )); + } + + Ok(()) +} + +pub(super) async fn verify_fresh_open_directory(path: &Path) -> Result<(), FilesystemStorageError> { + let metadata = tokio::fs::metadata(path) + .await + .map_err(|error| FilesystemStorageError::io("verify runtime directory", path, error))?; + if !metadata.is_dir() { + return Err(FilesystemStorageError::verification( + "verify fresh runtime directory", + path, + )); + } + + let mut entries = tokio::fs::read_dir(path).await.map_err(|error| { + FilesystemStorageError::io("verify empty runtime directory", path, error) + })?; + if entries + .next_entry() + .await + .map_err(|error| FilesystemStorageError::io("verify empty runtime directory", path, error))? + .is_some() + { + return Err(FilesystemStorageError::verification( + "verify empty runtime directory", + path, + )); + } + Ok(()) +} + +pub(super) async fn rollback_creation( + path: &Path, + creation_error: FilesystemStorageError, + cleanup_retry: &RetryConfig, +) -> FilesystemStorageError { + match remove_and_verify(path, "roll back runtime directory", cleanup_retry).await { + Ok(()) => creation_error, + Err(cleanup_error) => cleanup_error, + } +} + +async fn rollback_owned_filesystem( + filesystem: AgentFilesystem, + creation_error: FilesystemStorageError, +) -> FilesystemStorageError { + match filesystem.close_and_delete().await { + Ok(()) => creation_error, + Err(cleanup_error) => cleanup_error, + } +} + +pub(super) async fn remove_and_verify( + path: &Path, + operation: &'static str, + cleanup_retry: &RetryConfig, +) -> Result<(), FilesystemStorageError> { + let mut retry = RetryState::new(cleanup_retry); + loop { + retry.start_attempt(); + match remove_and_verify_once(path, operation).await { + Ok(()) => return Ok(()), + Err(error) => { + if !retry.failed_attempt().await { + return Err(error); + } + } + } + } +} + +async fn remove_and_verify_once( + path: &Path, + operation: &'static str, +) -> Result<(), FilesystemStorageError> { + match tokio::fs::symlink_metadata(path).await { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => { + tokio::fs::remove_dir_all(path) + .await + .map_err(|error| FilesystemStorageError::cleanup_io(operation, path, error))?; + } + Ok(_) => { + tokio::fs::remove_file(path) + .await + .map_err(|error| FilesystemStorageError::cleanup_io(operation, path, error))?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(FilesystemStorageError::cleanup_io(operation, path, error)), + } + + match tokio::fs::symlink_metadata(path).await { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Ok(_) => Err(FilesystemStorageError::cleanup_verification( + operation, path, + )), + Err(error) => Err(FilesystemStorageError::cleanup_io(operation, path, error)), + } +} + +pub(super) fn remove_and_verify_blocking(path: &Path) -> Result<(), FilesystemStorageError> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => { + std::fs::remove_dir_all(path).map_err(|error| { + FilesystemStorageError::cleanup_io("delete runtime directory", path, error) + })?; + } + Ok(_) => { + std::fs::remove_file(path).map_err(|error| { + FilesystemStorageError::cleanup_io("delete runtime directory", path, error) + })?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => { + return Err(FilesystemStorageError::cleanup_io( + "delete runtime directory", + path, + error, + )); + } + } + + match std::fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Ok(_) => Err(FilesystemStorageError::cleanup_verification( + "delete runtime directory", + path, + )), + Err(error) => Err(FilesystemStorageError::cleanup_io( + "delete runtime directory", + path, + error, + )), + } +} diff --git a/golem-worker-executor/src/services/agent_filesystem/mutation.rs b/golem-worker-executor/src/services/agent_filesystem/mutation.rs new file mode 100644 index 0000000000..e6d763ef14 --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/mutation.rs @@ -0,0 +1,1674 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use async_trait::async_trait; +use bytes::Bytes; +use cap_fs_ext::{DirExt as _, FollowSymlinks, OpenOptionsFollowExt, OpenOptionsMaybeDirExt}; +use cap_std::fs::FileExt; +use fs_set_times::{SetTimes as _, SystemTimeSpec}; +use std::io::{Seek, SeekFrom, Write}; +use std::sync::atomic::Ordering; +#[cfg(test)] +use std::time::Duration; +use std::time::Instant; +use tokio::sync::OwnedRwLockReadGuard; +use wasmtime_wasi::filesystem::{Descriptor, Dir, File, OpenMode}; +use wasmtime_wasi::p2::bindings::filesystem::types::ErrorCode; +use wasmtime_wasi::p2::{DynOutputStream, OutputStream, Pollable}; +use wasmtime_wasi::runtime::spawn_blocking; +use wasmtime_wasi::{DirPerms, FilePerms}; +use wasmtime_wasi::{StreamError, StreamResult}; + +// Terminal flag: once set, this runtime never admits another filesystem effect. +pub(super) const FILESYSTEM_RUNTIME_SEALED: usize = 1 << (usize::BITS - 1); +// Temporary flag: reject new effects while an existing set drains for a consistent observation. +const FILESYSTEM_RUNTIME_ADMISSION_PAUSED: usize = 1 << (usize::BITS - 2); +// All remaining low bits form the active-effect reference count. +const FILESYSTEM_RUNTIME_ACTIVE_EFFECTS: usize = + !(FILESYSTEM_RUNTIME_SEALED | FILESYSTEM_RUNTIME_ADMISSION_PAUSED); +// Bound short-effect observation lag while avoiding continuous quota probes at the executor epoch cadence. +const FILESYSTEM_USAGE_COMPLETION_DELAY: std::time::Duration = std::time::Duration::from_millis(10); +const FILESYSTEM_USAGE_SUSTAINED_INTERVAL: std::time::Duration = + std::time::Duration::from_millis(100); + +pub(crate) async fn run_blocking_filesystem_mutation(lease: Arc, operation: F) -> R +where + L: Send + Sync + 'static, + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + spawn_blocking(move || { + let _lease = lease; + operation() + }) + .await +} + +pub(crate) fn sync_descriptor(descriptor: &Descriptor, data_only: bool) -> std::io::Result<()> { + let result = match descriptor { + Descriptor::File(file) => { + if data_only { + file.file.sync_data() + } else { + file.file.sync_all() + } + } + Descriptor::Dir(directory) => { + let directory = directory.dir.open(std::path::Component::CurDir)?; + if data_only { + directory.sync_data() + } else { + directory.sync_all() + } + } + }; + #[cfg(windows)] + if matches!(descriptor, Descriptor::File(_)) + && result.as_ref().is_err_and(|error| { + error.raw_os_error() == Some(windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED as i32) + }) + { + return Ok(()); + } + result +} + +pub(crate) fn resize_file(file: &File, size: u64) -> std::io::Result<()> { + file.file.set_len(size) +} + +pub(crate) fn set_descriptor_times( + descriptor: &Descriptor, + accessed: Option, + modified: Option, +) -> std::io::Result<()> { + let accessed = accessed.map(SystemTimeSpec::Absolute); + let modified = modified.map(SystemTimeSpec::Absolute); + match descriptor { + Descriptor::File(file) => file.file.set_times(accessed, modified), + Descriptor::Dir(directory) => directory.dir.set_times(accessed, modified), + } +} + +pub(crate) fn set_path_times( + directory: &Dir, + path: &str, + follow: bool, + accessed: Option, + modified: Option, +) -> std::io::Result<()> { + let accessed = accessed.map(|time| { + cap_fs_ext::SystemTimeSpec::Absolute(cap_std::time::SystemTime::from_std(time)) + }); + let modified = modified.map(|time| { + cap_fs_ext::SystemTimeSpec::Absolute(cap_std::time::SystemTime::from_std(time)) + }); + if follow { + cap_fs_ext::DirExt::set_times(directory.dir.as_ref(), path, accessed, modified) + } else { + directory.dir.set_symlink_times(path, accessed, modified) + } +} + +pub(crate) fn create_directory(directory: &Dir, path: &str) -> std::io::Result<()> { + directory.dir.create_dir(path) +} + +pub(crate) fn hard_link( + source: &Dir, + source_path: &str, + destination: &Dir, + destination_path: &str, +) -> std::io::Result<()> { + source + .dir + .hard_link(source_path, &destination.dir, destination_path) +} + +pub(crate) fn rename( + source: &Dir, + source_path: &str, + destination: &Dir, + destination_path: &str, +) -> std::io::Result<()> { + source + .dir + .rename(source_path, &destination.dir, destination_path) +} + +pub(crate) fn remove_directory(directory: &Dir, path: &str) -> std::io::Result<()> { + directory.dir.remove_dir(path) +} + +pub(crate) fn unlink_file(directory: &Dir, path: &str) -> std::io::Result<()> { + directory.dir.remove_file_or_symlink(path) +} + +pub(crate) fn symlink(directory: &Dir, source: &str, destination: &str) -> std::io::Result<()> { + directory.dir.symlink(source, destination) +} + +#[derive(Clone, Copy)] +pub(crate) enum NativeMutationGuestError { + Invalid, + NotDirectory, + NotPermitted, + Unsupported, +} + +pub(crate) fn validate_resize(file: &File) -> Result<(), NativeMutationGuestError> { + if file.perms.contains(FilePerms::WRITE) { + Ok(()) + } else { + Err(NativeMutationGuestError::NotPermitted) + } +} + +pub(crate) fn validate_descriptor_times( + descriptor: &Descriptor, +) -> Result<(), NativeMutationGuestError> { + let permitted = match descriptor { + Descriptor::File(file) => file.perms.contains(FilePerms::WRITE), + Descriptor::Dir(directory) => directory.perms.contains(DirPerms::MUTATE), + }; + if permitted { + Ok(()) + } else { + Err(NativeMutationGuestError::NotPermitted) + } +} + +pub(crate) fn validate_directory_mutation(directory: &Dir) -> Result<(), NativeMutationGuestError> { + if directory.perms.contains(DirPerms::MUTATE) { + Ok(()) + } else { + Err(NativeMutationGuestError::NotPermitted) + } +} + +pub(crate) fn validate_two_directory_mutation( + source: &Dir, + destination: &Dir, +) -> Result<(), NativeMutationGuestError> { + validate_directory_mutation(source)?; + validate_directory_mutation(destination) +} + +#[derive(Clone, Copy)] +pub(crate) struct NativeOpenOptions { + pub create: bool, + pub directory: bool, + pub exclusive: bool, + pub truncate: bool, + pub follow: bool, + pub read: bool, + pub write: bool, +} + +pub(crate) fn validate_open( + directory: &Dir, + options: NativeOpenOptions, + unsupported_sync_flags: bool, +) -> Result<(), NativeMutationGuestError> { + if !directory.perms.contains(DirPerms::READ) { + return Err(NativeMutationGuestError::NotPermitted); + } + if !directory.perms.contains(DirPerms::MUTATE) + && (options.create || options.truncate || options.write) + { + return Err(NativeMutationGuestError::NotPermitted); + } + if unsupported_sync_flags { + return Err(NativeMutationGuestError::Unsupported); + } + if options.directory && (options.create || options.exclusive || options.truncate) { + return Err(NativeMutationGuestError::Invalid); + } + let opens_for_write = options.create || options.truncate || options.write; + if opens_for_write && !directory.file_perms.contains(FilePerms::WRITE) { + return Err(NativeMutationGuestError::NotPermitted); + } + Ok(()) +} + +pub(crate) enum NativeOpenResult { + Descriptor(Descriptor), + #[cfg(windows)] + IsDirectory, + NotDirectory, +} + +pub(crate) fn open( + directory: &Dir, + path: &str, + options: NativeOpenOptions, +) -> std::io::Result { + let mut native = cap_std::fs::OpenOptions::new(); + native.maybe_dir(true); + let mut mode = OpenMode::empty(); + + if options.create { + if options.exclusive { + native.create_new(true); + } else { + native.create(true); + } + native.write(true); + mode |= OpenMode::WRITE; + } + if options.truncate { + native.truncate(true).write(true); + mode |= OpenMode::WRITE; + } + if options.read { + native.read(true); + mode |= OpenMode::READ; + } + if options.write { + native.write(true); + mode |= OpenMode::WRITE; + } else { + native.read(true); + mode |= OpenMode::READ; + } + native.follow(if options.follow { + FollowSymlinks::Yes + } else { + FollowSymlinks::No + }); + + let opened = directory.dir.open_with(path, &native)?; + let child_path = directory.path.join(path); + if opened.metadata()?.is_dir() { + #[cfg(windows)] + if options.write { + return Ok(NativeOpenResult::IsDirectory); + } + Ok(NativeOpenResult::Descriptor(Descriptor::Dir(Dir::new( + cap_std::fs::Dir::from_std_file(opened.into_std()), + directory.perms, + directory.file_perms, + mode, + false, + child_path.clone(), + )))) + } else if options.directory { + Ok(NativeOpenResult::NotDirectory) + } else { + Ok(NativeOpenResult::Descriptor(Descriptor::File(File::new( + opened, + directory.file_perms, + mode, + false, + child_path, + )))) + } +} + +impl AgentFilesystemRuntime { + pub(crate) async fn begin_effect(&self) -> Result { + self.admit_effect()?.begin().await + } + + pub(crate) async fn begin_append_effect( + &self, + ) -> Result { + self.admit_effect()?.begin_append().await + } + + pub(crate) async fn begin_path_effect( + &self, + ) -> Result { + self.admit_effect()?.begin_path().await + } + + pub(crate) async fn begin_update_effect( + &self, + ) -> Result { + let admission = loop { + let mut admission_resumed = Box::pin(self.inner.admission_resumed.notified()); + admission_resumed.as_mut().enable(); + match self.admit_effect() { + Ok(admission) => break admission, + Err(error) => { + let state = self.inner.state.load(Ordering::Acquire); + if state & FILESYSTEM_RUNTIME_SEALED != 0 { + return Err(error); + } + if state & FILESYSTEM_RUNTIME_ADMISSION_PAUSED != 0 { + admission_resumed.await; + } + } + } + }; + let operation_guard = Arc::clone(&self.inner.operations).write_owned().await; + Ok(AgentFilesystemUpdateEffectLease { + _inner: Arc::new(AgentFilesystemUpdateEffectLeaseInner { + _admission: admission, + _operation_guard: operation_guard, + }), + }) + } + + pub(crate) fn admit_effect(&self) -> Result { + let mut state = self.inner.state.load(Ordering::Acquire); + loop { + if state & FILESYSTEM_RUNTIME_SEALED != 0 { + return Err(wasmtime::Error::msg("agent filesystem is closing")); + } + if state & FILESYSTEM_RUNTIME_ADMISSION_PAUSED != 0 { + return Err(wasmtime::Error::msg( + "agent filesystem resource window is transitioning", + )); + } + let active_effects = state & FILESYSTEM_RUNTIME_ACTIVE_EFFECTS; + let next = active_effects + .checked_add(1) + .filter(|next| next & !FILESYSTEM_RUNTIME_ACTIVE_EFFECTS == 0) + .expect("agent filesystem effect count overflowed"); + match self.inner.state.compare_exchange_weak( + state, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break, + Err(observed) => state = observed, + } + } + self.inner + .usage_effect_epoch + .fetch_add(1, Ordering::Release); + self.inner.schedule_usage_sampling(); + Ok(AgentFilesystemEffectAdmission { + inner: Arc::clone(&self.inner), + }) + } + + pub(crate) fn seal(&self) { + self.inner + .state + .fetch_or(FILESYSTEM_RUNTIME_SEALED, Ordering::AcqRel); + } + + pub(crate) fn pause_effect_admission(&self) -> AgentFilesystemEffectAdmissionPause { + let previous = self + .inner + .state + .fetch_or(FILESYSTEM_RUNTIME_ADMISSION_PAUSED, Ordering::AcqRel); + assert_eq!( + previous & FILESYSTEM_RUNTIME_ADMISSION_PAUSED, + 0, + "agent filesystem effect admission is already paused" + ); + AgentFilesystemEffectAdmissionPause { + inner: Arc::clone(&self.inner), + } + } + + pub(crate) fn seal_if_no_active_effects(&self) -> bool { + self.inner + .state + .compare_exchange( + 0, + FILESYSTEM_RUNTIME_SEALED, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + } + + pub(crate) async fn drain(&self) { + while self.has_active_effects() { + let mut drained = Box::pin(self.inner.drained.notified()); + drained.as_mut().enable(); + if !self.has_active_effects() { + break; + } + drained.await; + } + } + + pub(crate) async fn wait_for_usage_completion_debounce(&self) { + debug_assert!(!self.has_active_effects()); + tokio::time::sleep(FILESYSTEM_USAGE_COMPLETION_DELAY).await; + } + + pub(crate) fn has_active_effects(&self) -> bool { + self.inner.state.load(Ordering::Acquire) & FILESYSTEM_RUNTIME_ACTIVE_EFFECTS != 0 + } + + #[cfg(test)] + pub(crate) fn effect_admission_is_paused(&self) -> bool { + self.inner.state.load(Ordering::Acquire) & FILESYSTEM_RUNTIME_ADMISSION_PAUSED != 0 + } + + pub(crate) fn last_effect_completion_millis(&self) -> u64 { + self.inner + .last_effect_completion_millis + .load(Ordering::Acquire) + } +} + +impl AgentFilesystemRuntimeInner { + fn finish_effect(self: &Arc) { + self.usage_effect_epoch.fetch_add(1, Ordering::Release); + let previous = self.state.fetch_sub(1, Ordering::AcqRel); + let previous_active = previous & FILESYSTEM_RUNTIME_ACTIVE_EFFECTS; + debug_assert!(previous_active > 0); + if previous_active == 1 { + let completed_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let completed_at = u64::try_from(completed_at).unwrap_or(u64::MAX); + let _ = self.last_effect_completion_millis.fetch_update( + Ordering::Release, + Ordering::Acquire, + |previous| Some(completed_at.max(previous.saturating_add(1))), + ); + self.drained.notify_waiters(); + } + } + + pub(super) fn schedule_usage_sampling(self: &Arc) { + // Observer replacement is rare; this lock is held only for the presence check. + // The atomic flag below coalesces all active effects onto one background sampler. + if self + .usage_observer + .lock() + .expect("agent filesystem usage-observer lock poisoned") + .is_none() + || self.usage_sampling.swap(true, Ordering::AcqRel) + { + return; + } + + let runtime = AgentFilesystemRuntime { + inner: Arc::clone(self), + }; + let Ok(handle) = tokio::runtime::Handle::try_current() else { + self.usage_sampling.store(false, Ordering::Release); + return; + }; + handle.spawn(async move { + let mut sampled_effect_epoch = 0; + let mut drained = Box::pin(runtime.inner.drained.notified()); + drained.as_mut().enable(); + if runtime.has_active_effects() { + tokio::select! { + _ = tokio::time::sleep(FILESYSTEM_USAGE_COMPLETION_DELAY) => {} + _ = &mut drained => { + tokio::time::sleep(FILESYSTEM_USAGE_COMPLETION_DELAY).await; + } + } + } else { + tokio::time::sleep(FILESYSTEM_USAGE_COMPLETION_DELAY).await; + } + loop { + if !runtime.usage_observer_is_active() { + break; + } + let effect_epoch = runtime.inner.usage_effect_epoch.load(Ordering::Acquire); + if let Err(error) = runtime.observe_usage_for_billing().await { + tracing::error!(error = %error, "Failed to observe filesystem usage during an active resource window"); + runtime.invalidate_runtime().await; + break; + } + sampled_effect_epoch = effect_epoch; + if !runtime.has_active_effects() + && runtime.inner.usage_effect_epoch.load(Ordering::Acquire) + == sampled_effect_epoch + { + break; + } + + let mut drained = Box::pin(runtime.inner.drained.notified()); + drained.as_mut().enable(); + if runtime.has_active_effects() { + tokio::select! { + _ = tokio::time::sleep(FILESYSTEM_USAGE_SUSTAINED_INTERVAL) => {} + _ = &mut drained => { + tokio::time::sleep(FILESYSTEM_USAGE_COMPLETION_DELAY).await; + } + } + } else { + tokio::time::sleep(FILESYSTEM_USAGE_COMPLETION_DELAY).await; + } + } + + runtime + .inner + .finish_usage_sampling(sampled_effect_epoch); + }); + } + + pub(super) fn finish_usage_sampling(self: &Arc, sampled_effect_epoch: u64) { + self.usage_sampling.store(false, Ordering::Release); + let observer_active = self + .usage_observer + .lock() + .expect("agent filesystem usage-observer lock poisoned") + .as_ref() + .is_some_and(|observer| observer.is_active()); + if observer_active + && (self.state.load(Ordering::Acquire) & FILESYSTEM_RUNTIME_ACTIVE_EFFECTS != 0 + || self.usage_effect_epoch.load(Ordering::Acquire) != sampled_effect_epoch) + { + self.schedule_usage_sampling(); + } + } +} + +#[derive(Debug)] +pub(crate) struct AgentFilesystemEffectLease { + _admission: AgentFilesystemEffectAdmission, + _operation_guard: OwnedRwLockReadGuard<()>, + _append_guard: Option>, + _namespace_guard: Option>, +} + +#[derive(Clone)] +pub(crate) struct AgentFilesystemUpdateEffectLease { + _inner: Arc, +} + +struct AgentFilesystemUpdateEffectLeaseInner { + _admission: AgentFilesystemEffectAdmission, + _operation_guard: tokio::sync::OwnedRwLockWriteGuard<()>, +} + +pub(crate) struct AgentFilesystemEffectAdmission { + inner: Arc, +} + +#[must_use = "effect admission resumes when the pause is dropped"] +pub(crate) struct AgentFilesystemEffectAdmissionPause { + inner: Arc, +} + +impl Drop for AgentFilesystemEffectAdmissionPause { + fn drop(&mut self) { + let previous = self + .inner + .state + .fetch_and(!FILESYSTEM_RUNTIME_ADMISSION_PAUSED, Ordering::AcqRel); + debug_assert_ne!(previous & FILESYSTEM_RUNTIME_ADMISSION_PAUSED, 0); + self.inner.admission_resumed.notify_waiters(); + } +} + +impl Drop for AgentFilesystemEffectAdmission { + fn drop(&mut self) { + self.inner.finish_effect(); + } +} + +impl AgentFilesystemEffectAdmission { + pub(crate) async fn begin(self) -> Result { + let operation_guard = Arc::clone(&self.inner.operations).read_owned().await; + self.ensure_open()?; + Ok(AgentFilesystemEffectLease { + _admission: self, + _operation_guard: operation_guard, + _append_guard: None, + _namespace_guard: None, + }) + } + + pub(crate) async fn begin_append(self) -> Result { + let guard = Arc::clone(&self.inner.append).lock_owned().await; + let operation_guard = Arc::clone(&self.inner.operations).read_owned().await; + self.ensure_open()?; + Ok(AgentFilesystemEffectLease { + _admission: self, + _operation_guard: operation_guard, + _append_guard: Some(guard), + _namespace_guard: None, + }) + } + + pub(crate) async fn begin_path(self) -> Result { + let operation_guard = Arc::clone(&self.inner.operations).read_owned().await; + let namespace_guard = Arc::clone(&self.inner.namespace).lock_owned().await; + self.ensure_open()?; + Ok(AgentFilesystemEffectLease { + _admission: self, + _operation_guard: operation_guard, + _append_guard: None, + _namespace_guard: Some(namespace_guard), + }) + } + + fn ensure_open(&self) -> Result<(), wasmtime::Error> { + if self.inner.state.load(Ordering::Acquire) & FILESYSTEM_RUNTIME_SEALED != 0 { + Err(wasmtime::Error::msg("agent filesystem is closing")) + } else { + Ok(()) + } + } +} + +impl std::fmt::Debug for AgentFilesystemEffectAdmission { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("AgentFilesystemEffectAdmission") + .finish_non_exhaustive() + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) enum FilesystemStreamMode { + Position(u64), + Append, +} + +pub(crate) struct ClassifiedFileOutputStream { + writer: Arc, + filesystem_runtime: AgentFilesystemRuntime, + mode: FilesystemStreamMode, + state: FilesystemOutputState, + prepared_effect: std::sync::Mutex>, +} + +enum FilesystemOutputState { + Ready, + Waiting { + task: tokio::task::JoinHandle<(usize, ClassifiedFilesystemStreamResult)>, + cancellation: tokio_util::sync::CancellationToken, + }, + Error(ClassifiedFilesystemStreamFailure), + Closed, +} + +struct FilesystemWriteAttempt { + written: usize, + result: std::io::Result<()>, +} + +#[derive(Debug)] +enum ClassifiedFilesystemStreamFailure { + Guest(ErrorCode), + Raw(std::io::Error), + Trap(String), +} + +type ClassifiedFilesystemStreamResult = Result<(), ClassifiedFilesystemStreamFailure>; + +#[derive(Debug)] +struct ClassifiedFilesystemErrorCode(ErrorCode); + +impl std::fmt::Display for ClassifiedFilesystemErrorCode { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + write!(formatter, "filesystem error: {:?}", self.0) + } +} + +impl std::error::Error for ClassifiedFilesystemErrorCode {} + +#[async_trait] +trait FilesystemStreamWriter: Send + Sync { + async fn write( + &self, + mode: FilesystemStreamMode, + contents: Bytes, + start: usize, + effect: Arc, + ) -> FilesystemWriteAttempt; +} + +struct NativeFilesystemStreamWriter { + file: File, +} + +#[async_trait] +impl FilesystemStreamWriter for NativeFilesystemStreamWriter { + async fn write( + &self, + mode: FilesystemStreamMode, + contents: Bytes, + start: usize, + effect: Arc, + ) -> FilesystemWriteAttempt { + let file = Arc::clone(&self.file.file); + spawn_blocking(move || { + let _effect = effect; + let suffix = &contents[start..]; + let result = match mode { + FilesystemStreamMode::Position(position) => file.write_at(suffix, position), + FilesystemStreamMode::Append => { + let mut file = file.as_ref(); + file.seek(SeekFrom::End(0)).and_then(|_| file.write(suffix)) + } + }; + match result { + Ok(written) => FilesystemWriteAttempt { + written, + result: Ok(()), + }, + Err(error) => FilesystemWriteAttempt { + written: 0, + result: Err(error), + }, + } + }) + .await + } +} + +impl ClassifiedFileOutputStream { + pub(crate) fn new( + file: File, + filesystem_runtime: AgentFilesystemRuntime, + mode: FilesystemStreamMode, + ) -> Self { + Self::new_with_writer( + Arc::new(NativeFilesystemStreamWriter { file }), + filesystem_runtime, + mode, + ) + } + + fn new_with_writer( + writer: Arc, + filesystem_runtime: AgentFilesystemRuntime, + mode: FilesystemStreamMode, + ) -> Self { + Self { + writer, + filesystem_runtime, + mode, + state: FilesystemOutputState::Ready, + prepared_effect: std::sync::Mutex::new(None), + } + } + + #[cfg(test)] + fn new_for_test( + writer: Arc, + filesystem_runtime: AgentFilesystemRuntime, + mode: FilesystemStreamMode, + ) -> Self + where + W: FilesystemStreamWriter + 'static, + { + Self::new_with_writer(writer, filesystem_runtime, mode) + } + + pub(crate) fn into_dyn(self) -> DynOutputStream { + Box::new(self) + } + + pub(crate) fn is_active(&self) -> bool { + matches!(self.state, FilesystemOutputState::Waiting { .. }) + } + + pub(crate) fn prepare_effect(&self, effect: AgentFilesystemEffectLease) { + let previous = self + .prepared_effect + .lock() + .expect("filesystem output stream effect lock poisoned") + .replace(effect); + debug_assert!(previous.is_none()); + } + + pub(crate) fn clear_unused_effect(&self) { + self.prepared_effect + .lock() + .expect("filesystem output stream effect lock poisoned") + .take(); + } + + async fn wait_until_ready(&mut self) { + let state = std::mem::replace(&mut self.state, FilesystemOutputState::Closed); + let task = match state { + FilesystemOutputState::Waiting { task, .. } => task, + state => { + self.state = state; + return; + } + }; + + self.state = match task.await { + Ok((written, result)) => { + if let FilesystemStreamMode::Position(position) = &mut self.mode { + let Some(next_position) = u64::try_from(written) + .ok() + .and_then(|written| position.checked_add(written)) + else { + let _ = self + .filesystem_runtime + .classify_mutation_failure::( + MutationFailure::Infrastructure(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "filesystem stream position overflowed", + )), + MutationEffect::Unknown, + ) + .await; + return self.set_trap("filesystem stream position overflowed"); + }; + *position = next_position; + } + match result { + Ok(()) => FilesystemOutputState::Ready, + Err(error) => FilesystemOutputState::Error(error), + } + } + Err(error) => { + let message = format!("filesystem stream write task failed: {error}"); + let _ = self + .filesystem_runtime + .classify_mutation_failure::( + MutationFailure::Infrastructure(std::io::Error::other(message.clone())), + MutationEffect::Unknown, + ) + .await; + FilesystemOutputState::Error(ClassifiedFilesystemStreamFailure::Trap(message)) + } + }; + } + + fn set_trap(&mut self, message: &'static str) { + self.state = FilesystemOutputState::Error(ClassifiedFilesystemStreamFailure::Trap( + message.to_string(), + )); + } + + fn take_error(&mut self) -> StreamResult { + match std::mem::replace(&mut self.state, FilesystemOutputState::Closed) { + FilesystemOutputState::Error(ClassifiedFilesystemStreamFailure::Guest(error)) => Err( + StreamError::LastOperationFailed(ClassifiedFilesystemErrorCode(error).into()), + ), + FilesystemOutputState::Error(ClassifiedFilesystemStreamFailure::Raw(error)) => { + Err(StreamError::LastOperationFailed(error.into())) + } + FilesystemOutputState::Error(ClassifiedFilesystemStreamFailure::Trap(message)) => { + Err(StreamError::Trap(wasmtime::Error::msg(message))) + } + _ => unreachable!("filesystem stream error state changed unexpectedly"), + } + } + + fn cancel_active_write(&self) { + if let FilesystemOutputState::Waiting { cancellation, .. } = &self.state { + cancellation.cancel(); + } + } +} + +#[async_trait] +impl OutputStream for ClassifiedFileOutputStream { + fn write(&mut self, bytes: Bytes) -> StreamResult<()> { + match self.state { + FilesystemOutputState::Ready => {} + FilesystemOutputState::Closed => return Err(StreamError::Closed), + FilesystemOutputState::Waiting { .. } | FilesystemOutputState::Error(_) => { + return Err(StreamError::Trap(wasmtime::Error::msg( + "write not permitted: check_write not called first", + ))); + } + } + + let effect = self + .prepared_effect + .lock() + .expect("filesystem output stream effect lock poisoned") + .take() + .ok_or_else(|| { + StreamError::Trap(wasmtime::Error::msg( + "filesystem output stream write has no effect lease", + )) + })?; + let writer = Arc::clone(&self.writer); + let filesystem_runtime = self.filesystem_runtime.clone(); + let mode = self.mode; + let cancellation = tokio_util::sync::CancellationToken::new(); + let write_cancellation = cancellation.clone(); + self.state = FilesystemOutputState::Waiting { + task: tokio::spawn(async move { + run_classified_filesystem_stream_write( + writer.as_ref(), + &filesystem_runtime, + mode, + bytes, + effect, + write_cancellation, + ) + .await + }), + cancellation, + }; + Ok(()) + } + + fn flush(&mut self) -> StreamResult<()> { + match self.state { + FilesystemOutputState::Ready | FilesystemOutputState::Waiting { .. } => Ok(()), + FilesystemOutputState::Closed => Err(StreamError::Closed), + FilesystemOutputState::Error(_) => self.take_error().map(|_| ()), + } + } + + fn check_write(&mut self) -> StreamResult { + match self.state { + FilesystemOutputState::Ready => Ok(1024 * 1024), + FilesystemOutputState::Waiting { .. } => Ok(0), + FilesystemOutputState::Closed => Err(StreamError::Closed), + FilesystemOutputState::Error(_) => self.take_error(), + } + } + + async fn cancel(&mut self) { + self.cancel_active_write(); + if self.is_active() { + self.wait_until_ready().await; + } + self.state = FilesystemOutputState::Closed; + self.clear_unused_effect(); + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[async_trait] +impl Pollable for ClassifiedFileOutputStream { + async fn ready(&mut self) { + self.wait_until_ready().await; + } +} + +impl Drop for ClassifiedFileOutputStream { + fn drop(&mut self) { + self.cancel_active_write(); + self.clear_unused_effect(); + } +} + +async fn run_classified_filesystem_stream_write( + writer: &W, + filesystem_runtime: &AgentFilesystemRuntime, + mode: FilesystemStreamMode, + contents: Bytes, + effect: AgentFilesystemEffectLease, + cancellation: tokio_util::sync::CancellationToken, +) -> (usize, ClassifiedFilesystemStreamResult) { + let effect = Arc::new(effect); + let started = Instant::now(); + let mut completed = 0usize; + let mut failed_attempts = 0usize; + + while completed < contents.len() { + if cancellation.is_cancelled() { + return (completed, Ok(())); + } + let attempt_mode = match mode { + FilesystemStreamMode::Position(position) => { + let Some(position) = u64::try_from(completed) + .ok() + .and_then(|completed| position.checked_add(completed)) + else { + return invalidate_filesystem_stream( + filesystem_runtime, + completed, + "filesystem stream write offset overflowed", + ) + .await; + }; + FilesystemStreamMode::Position(position) + } + FilesystemStreamMode::Append => FilesystemStreamMode::Append, + }; + let attempt = writer + .write( + attempt_mode, + contents.clone(), + completed, + Arc::clone(&effect), + ) + .await; + let remaining = contents.len() - completed; + if attempt.written > remaining { + return invalidate_filesystem_stream( + filesystem_runtime, + completed, + "filesystem stream writer reported more bytes than requested", + ) + .await; + } + completed += attempt.written; + + let (error, effect) = match attempt.result { + Ok(()) if attempt.written != 0 => { + if cancellation.is_cancelled() { + return (completed, Ok(())); + } + continue; + } + Ok(()) => ( + std::io::Error::from(std::io::ErrorKind::WriteZero), + proven_write_progress_effect(completed), + ), + Err(error) => { + let effect = native_write_failure_effect(&error, completed); + (error, effect) + } + }; + failed_attempts += 1; + let raw_os_error = error.raw_os_error(); + let error_kind = error.kind(); + let error_message = error.to_string(); + let decision = filesystem_runtime + .classify_mutation_failure_for::( + MutationOperation::Write, + MutationFailure::Io(error), + effect, + ) + .await; + + match decision { + MutationDecision::BoundedRetry if cancellation.is_cancelled() => { + return (completed, Ok(())); + } + MutationDecision::BoundedRetry + if failed_attempts < super::failure::FILESYSTEM_MUTATION_MAX_ATTEMPTS + && started.elapsed() <= super::failure::FILESYSTEM_MUTATION_RETRY_TIMEOUT => {} + MutationDecision::BoundedRetry => { + let error = raw_os_error.map_or_else( + || std::io::Error::new(error_kind, error_message), + std::io::Error::from_raw_os_error, + ); + return ( + completed, + Err(ClassifiedFilesystemStreamFailure::Raw(error)), + ); + } + MutationDecision::PreserveRaw => { + let error = raw_os_error.map_or_else( + || std::io::Error::new(error_kind, error_message), + std::io::Error::from_raw_os_error, + ); + return ( + completed, + Err(ClassifiedFilesystemStreamFailure::Raw(error)), + ); + } + MutationDecision::PreserveGuest(error) => { + return ( + completed, + Err(ClassifiedFilesystemStreamFailure::Guest(error)), + ); + } + MutationDecision::Quota => { + return ( + completed, + Err(ClassifiedFilesystemStreamFailure::Guest(ErrorCode::Quota)), + ); + } + MutationDecision::InsufficientSpace => { + return ( + completed, + Err(ClassifiedFilesystemStreamFailure::Guest( + ErrorCode::InsufficientSpace, + )), + ); + } + MutationDecision::PhysicalPressure if cancellation.is_cancelled() => { + return (completed, Ok(())); + } + MutationDecision::PhysicalPressure + if failed_attempts < super::failure::FILESYSTEM_MUTATION_MAX_ATTEMPTS + && started.elapsed() <= super::failure::FILESYSTEM_MUTATION_RETRY_TIMEOUT + && filesystem_runtime + .recover_physical_pressure( + MutationOperation::Write, + started + super::failure::FILESYSTEM_MUTATION_RETRY_TIMEOUT, + ) + .await + && started.elapsed() <= super::failure::FILESYSTEM_MUTATION_RETRY_TIMEOUT => {} + MutationDecision::PhysicalPressure => { + return ( + completed, + Err(ClassifiedFilesystemStreamFailure::Guest( + ErrorCode::InsufficientSpace, + )), + ); + } + MutationDecision::Success => return (completed, Ok(())), + MutationDecision::Invalidate => { + return ( + completed, + Err(ClassifiedFilesystemStreamFailure::Trap( + "agent filesystem mutation invalidated the runtime".to_string(), + )), + ); + } + } + } + + (completed, Ok(())) +} + +async fn invalidate_filesystem_stream( + filesystem_runtime: &AgentFilesystemRuntime, + completed: usize, + message: &'static str, +) -> (usize, ClassifiedFilesystemStreamResult) { + let effect = u64::try_from(completed).map_or(MutationEffect::Unknown, |bytes| { + if bytes == 0 { + MutationEffect::ProvenNoEffect + } else { + MutationEffect::KnownCompletedPrefix { bytes } + } + }); + let _ = filesystem_runtime + .classify_mutation_failure::( + MutationFailure::Infrastructure(std::io::Error::new( + std::io::ErrorKind::InvalidData, + message, + )), + effect, + ) + .await; + ( + completed, + Err(ClassifiedFilesystemStreamFailure::Trap(message.to_string())), + ) +} + +pub(crate) fn classified_filesystem_stream_error_code( + error: &wasmtime::Error, +) -> Option { + error + .downcast_ref::() + .map(|error| error.0) +} + +#[cfg(test)] +mod classified_stream_tests { + use super::*; + use std::collections::VecDeque; + use std::sync::atomic::{AtomicBool, Ordering}; + + struct InjectedFilesystemWriter { + attempts: std::sync::Mutex>, + suffixes: std::sync::Mutex>>, + started: Option>, + release: Option>, + } + + impl InjectedFilesystemWriter { + fn new(attempts: impl IntoIterator) -> Self { + Self { + attempts: std::sync::Mutex::new(attempts.into_iter().collect()), + suffixes: std::sync::Mutex::new(Vec::new()), + started: None, + release: None, + } + } + + fn delayed_first( + attempts: impl IntoIterator, + started: Arc, + release: Arc, + ) -> Self { + Self { + attempts: std::sync::Mutex::new(attempts.into_iter().collect()), + suffixes: std::sync::Mutex::new(Vec::new()), + started: Some(started), + release: Some(release), + } + } + + fn suffixes(&self) -> Vec> { + self.suffixes.lock().unwrap().clone() + } + } + + #[async_trait] + impl FilesystemStreamWriter for InjectedFilesystemWriter { + async fn write( + &self, + _mode: FilesystemStreamMode, + contents: Bytes, + start: usize, + _effect: Arc, + ) -> FilesystemWriteAttempt { + let attempt_index = { + let mut suffixes = self.suffixes.lock().unwrap(); + let attempt_index = suffixes.len(); + suffixes.push(contents[start..].to_vec()); + attempt_index + }; + if attempt_index == 0 { + if let Some(started) = &self.started { + started.notify_one(); + } + if let Some(release) = &self.release { + release.acquire().await.unwrap().forget(); + } + } + self.attempts.lock().unwrap().pop_front().unwrap() + } + } + + fn success(written: usize) -> FilesystemWriteAttempt { + FilesystemWriteAttempt { + written, + result: Ok(()), + } + } + + fn failure(written: usize, errno: i32) -> FilesystemWriteAttempt { + FilesystemWriteAttempt { + written, + result: Err(std::io::Error::from_raw_os_error(errno)), + } + } + + #[test_r::test] + fn two_directory_mutation_allows_different_authority_sets() { + let source_root = tempfile::TempDir::new().unwrap(); + let destination_root = tempfile::TempDir::new().unwrap(); + let source = Dir::new( + cap_std::fs::Dir::open_ambient_dir(source_root.path(), cap_std::ambient_authority()) + .unwrap(), + DirPerms::all(), + FilePerms::all(), + OpenMode::READ | OpenMode::WRITE, + false, + source_root.path().to_path_buf(), + ); + let destination = Dir::new( + cap_std::fs::Dir::open_ambient_dir( + destination_root.path(), + cap_std::ambient_authority(), + ) + .unwrap(), + DirPerms::MUTATE, + FilePerms::READ, + OpenMode::READ | OpenMode::WRITE, + false, + destination_root.path().to_path_buf(), + ); + + assert!(validate_two_directory_mutation(&source, &destination).is_ok()); + } + + #[test_r::test] + async fn cancelled_blocking_mutation_keeps_effect_lease_until_native_completion() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let started = Arc::new(AtomicBool::new(false)); + let release = Arc::new(AtomicBool::new(false)); + let mutation = tokio::spawn({ + let started = Arc::clone(&started); + let release = Arc::clone(&release); + let lease = Arc::new(runtime.begin_effect().await.unwrap()); + async move { + run_blocking_filesystem_mutation(lease, move || { + started.store(true, Ordering::Release); + while !release.load(Ordering::Acquire) { + std::thread::sleep(Duration::from_millis(1)); + } + }) + .await + } + }); + while !started.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + + mutation.abort(); + let _ = mutation.await; + let update = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.begin_update_effect().await } + }); + tokio::task::yield_now().await; + let update_finished_while_native_operation_was_running = update.is_finished(); + release.store(true, Ordering::Release); + assert!(update.await.unwrap().is_ok()); + assert!(!update_finished_while_native_operation_was_running); + } + + #[cfg(target_os = "linux")] + #[test_r::test] + async fn p2_filesystem_stream_retries_transient_before_effect() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let writer = InjectedFilesystemWriter::new([failure(0, libc::EAGAIN), success(5)]); + + let (written, result) = run_classified_filesystem_stream_write( + &writer, + &runtime, + FilesystemStreamMode::Position(7), + Bytes::from_static(b"hello"), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 5); + assert!(result.is_ok()); + assert_eq!(writer.suffixes(), [b"hello".to_vec(), b"hello".to_vec()]); + } + + #[cfg(target_os = "linux")] + #[test_r::test] + async fn p2_filesystem_stream_retries_only_unwritten_suffix() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let writer = InjectedFilesystemWriter::new([failure(2, libc::EBUSY), success(3)]); + + let (written, result) = run_classified_filesystem_stream_write( + &writer, + &runtime, + FilesystemStreamMode::Position(11), + Bytes::from_static(b"hello"), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 5); + assert!(result.is_ok()); + assert_eq!(writer.suffixes(), [b"hello".to_vec(), b"llo".to_vec()]); + } + + #[cfg(target_os = "linux")] + #[test_r::test] + async fn p2_filesystem_stream_preserves_raw_error_after_retry_exhaustion() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let writer = + InjectedFilesystemWriter::new([failure(0, libc::EBUSY), failure(0, libc::EBUSY)]); + + let (written, result) = run_classified_filesystem_stream_write( + &writer, + &runtime, + FilesystemStreamMode::Append, + Bytes::from_static(b"hello"), + runtime.begin_append_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 0); + assert!(matches!( + result, + Err(ClassifiedFilesystemStreamFailure::Raw(error)) + if error.raw_os_error() == Some(libc::EBUSY) + )); + assert_eq!(writer.suffixes(), [b"hello".to_vec(), b"hello".to_vec()]); + assert!(runtime.begin_effect().await.is_ok()); + } + + #[cfg(target_os = "linux")] + #[test_r::test] + async fn p2_filesystem_stream_terminal_failure_traps_and_seals_runtime() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let writer = InjectedFilesystemWriter::new([failure(2, libc::EIO)]); + + let (written, result) = run_classified_filesystem_stream_write( + &writer, + &runtime, + FilesystemStreamMode::Position(0), + Bytes::from_static(b"hello"), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 2); + assert!(matches!( + result, + Err(ClassifiedFilesystemStreamFailure::Trap(_)) + )); + assert_eq!(writer.suffixes(), [b"hello".to_vec()]); + assert!(runtime.begin_effect().await.is_err()); + } + + #[cfg(target_os = "linux")] + #[test_r::test] + async fn p2_filesystem_stream_interruption_has_unknown_effect() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let writer = InjectedFilesystemWriter::new([failure(0, libc::EINTR)]); + + let (written, result) = run_classified_filesystem_stream_write( + &writer, + &runtime, + FilesystemStreamMode::Position(0), + Bytes::from_static(b"hello"), + runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + + assert_eq!(written, 0); + assert!(matches!( + result, + Err(ClassifiedFilesystemStreamFailure::Trap(_)) + )); + assert_eq!(writer.suffixes(), [b"hello".to_vec()]); + assert!(runtime.begin_effect().await.is_err()); + } + + #[cfg(target_os = "linux")] + #[test_r::test] + async fn p2_filesystem_stream_maps_classified_storage_exhaustion() { + let capacity = FilesystemCapacity { + total_bytes: 100, + available_bytes: 0, + total_filesystem_objects: 100, + available_filesystem_objects: 0, + }; + let quota_runtime = AgentFilesystemRuntime::new_for_test_with_observations( + Some(AgentFilesystemUsage { + allocated_bytes: 50, + filesystem_objects: 10, + }), + Some(ResolvedAgentFilesystemLimits { + allocated_bytes: 50, + filesystem_objects: 10, + filesystem_object_limit_policy_version: FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION, + }), + capacity, + ); + let quota_writer = InjectedFilesystemWriter::new([failure(0, libc::ENOSPC)]); + let (_, quota_result) = run_classified_filesystem_stream_write( + "a_writer, + "a_runtime, + FilesystemStreamMode::Position(0), + Bytes::from_static(b"hello"), + quota_runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + assert!(matches!( + quota_result, + Err(ClassifiedFilesystemStreamFailure::Guest(ErrorCode::Quota)) + )); + + let physical_runtime = + AgentFilesystemRuntime::new_for_test_with_observations(None, None, capacity); + let physical_writer = InjectedFilesystemWriter::new([failure(0, libc::ENOSPC)]); + let (_, physical_result) = run_classified_filesystem_stream_write( + &physical_writer, + &physical_runtime, + FilesystemStreamMode::Position(0), + Bytes::from_static(b"hello"), + physical_runtime.begin_effect().await.unwrap(), + tokio_util::sync::CancellationToken::new(), + ) + .await; + assert!(matches!( + physical_result, + Err(ClassifiedFilesystemStreamFailure::Guest( + ErrorCode::InsufficientSpace + )) + )); + } + + #[test_r::test] + async fn p2_filesystem_stream_idle_readiness_preserves_write_capacity() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let writer = Arc::new(InjectedFilesystemWriter::new([])); + let mut stream = ClassifiedFileOutputStream::new_for_test( + writer, + runtime, + FilesystemStreamMode::Position(0), + ); + + stream.ready().await; + + assert_eq!(stream.check_write().unwrap(), 1024 * 1024); + } + + #[test_r::test] + async fn p2_filesystem_stream_pending_flush_and_check_preserve_effect() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let writer = Arc::new(InjectedFilesystemWriter::delayed_first( + [success(5)], + Arc::clone(&started), + Arc::clone(&release), + )); + let mut stream = ClassifiedFileOutputStream::new_for_test( + writer, + runtime.clone(), + FilesystemStreamMode::Append, + ); + stream.prepare_effect(runtime.begin_append_effect().await.unwrap()); + stream.write(Bytes::from_static(b"hello")).unwrap(); + started.notified().await; + + assert!(stream.flush().is_ok()); + assert_eq!(stream.check_write().unwrap(), 0); + let next_append = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.begin_append_effect().await } + }); + tokio::task::yield_now().await; + assert!(!next_append.is_finished()); + + release.add_permits(1); + stream.ready().await; + stream.ready().await; + assert_eq!(stream.check_write().unwrap(), 1024 * 1024); + assert!(next_append.await.unwrap().is_ok()); + } + + #[cfg(target_os = "linux")] + #[test_r::test] + async fn p2_filesystem_stream_readiness_preserves_error_until_flush() { + let capacity = FilesystemCapacity { + total_bytes: 100, + available_bytes: 0, + total_filesystem_objects: 100, + available_filesystem_objects: 0, + }; + let runtime = AgentFilesystemRuntime::new_for_test_with_observations(None, None, capacity); + let writer = Arc::new(InjectedFilesystemWriter::new([failure(0, libc::ENOSPC)])); + let mut stream = ClassifiedFileOutputStream::new_for_test( + writer, + runtime.clone(), + FilesystemStreamMode::Position(0), + ); + stream.prepare_effect(runtime.begin_effect().await.unwrap()); + stream.write(Bytes::from_static(b"hello")).unwrap(); + stream.ready().await; + + stream.ready().await; + let error = stream.flush().unwrap_err(); + assert!(matches!( + error, + StreamError::LastOperationFailed(error) + if classified_filesystem_stream_error_code(&error) + == Some(ErrorCode::InsufficientSpace) + )); + assert!(matches!(stream.check_write(), Err(StreamError::Closed))); + } + + #[cfg(target_os = "linux")] + #[test_r::test] + async fn cancelling_p2_filesystem_stream_keeps_lease_until_native_completion() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let writer = Arc::new(InjectedFilesystemWriter::delayed_first( + [failure(0, libc::EAGAIN), success(5)], + Arc::clone(&started), + Arc::clone(&release), + )); + let mut stream = ClassifiedFileOutputStream::new_for_test( + Arc::clone(&writer), + runtime.clone(), + FilesystemStreamMode::Append, + ); + stream.prepare_effect(runtime.begin_append_effect().await.unwrap()); + stream.write(Bytes::from_static(b"hello")).unwrap(); + started.notified().await; + + stream.cancel_active_write(); + let cancellation = tokio::spawn(async move { + stream.cancel().await; + stream + }); + let next_append = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.begin_append_effect().await } + }); + tokio::task::yield_now().await; + assert!(!cancellation.is_finished()); + assert!(!next_append.is_finished()); + + release.add_permits(1); + let _stream = cancellation.await.unwrap(); + assert!(next_append.await.unwrap().is_ok()); + assert_eq!(writer.suffixes(), [b"hello".to_vec()]); + } + + #[test_r::test] + async fn dropping_p2_filesystem_stream_keeps_lease_until_native_completion() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let writer = Arc::new(InjectedFilesystemWriter::delayed_first( + [success(2), success(3)], + Arc::clone(&started), + Arc::clone(&release), + )); + let mut stream = ClassifiedFileOutputStream::new_for_test( + Arc::clone(&writer), + runtime.clone(), + FilesystemStreamMode::Append, + ); + stream.prepare_effect(runtime.begin_append_effect().await.unwrap()); + stream.write(Bytes::from_static(b"hello")).unwrap(); + started.notified().await; + drop(stream); + + let next_append = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.begin_append_effect().await } + }); + tokio::task::yield_now().await; + assert!(!next_append.is_finished()); + + release.add_permits(1); + assert!(next_append.await.unwrap().is_ok()); + assert_eq!(writer.suffixes(), [b"hello".to_vec()]); + } +} diff --git a/golem-worker-executor/src/services/agent_filesystem/postcondition.rs b/golem-worker-executor/src/services/agent_filesystem/postcondition.rs new file mode 100644 index 0000000000..90c94f6f8a --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/postcondition.rs @@ -0,0 +1,441 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; +use std::time::SystemTime; + +#[cfg(target_os = "linux")] +use cap_std::fs::MetadataExt as _; +use wasmtime_wasi::filesystem::{Descriptor, Dir}; +use wasmtime_wasi::runtime::spawn_blocking; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MutationPostcondition { + Satisfied, + NoEffect, + Unknown, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ObjectIdentity { + pub(crate) device: u64, + pub(crate) inode: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum PathObjectType { + Directory, + RegularFile, + SymbolicLink, + Other, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct PathState { + pub(crate) identity: Option, + pub(crate) type_: PathObjectType, + pub(crate) size: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SymlinkState { + pub(crate) object: Option, + pub(crate) target: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct TimesState { + pub(crate) identity: Option, + pub(crate) accessed: Option, + pub(crate) modified: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RequestedTime { + NoChange, + Now, + Timestamp { seconds: i128, nanoseconds: u32 }, +} + +#[cfg(target_os = "linux")] +fn object_identity(metadata: &cap_std::fs::Metadata) -> Option { + Some(ObjectIdentity { + device: metadata.dev(), + inode: metadata.ino(), + }) +} + +#[cfg(not(target_os = "linux"))] +fn object_identity(_metadata: &cap_std::fs::Metadata) -> Option { + None +} + +pub(crate) fn same_object(left: PathState, right: PathState) -> bool { + left.identity + .zip(right.identity) + .is_some_and(|(left, right)| left == right) +} + +pub(crate) fn same_optional_object(left: Option, right: Option) -> bool { + match (left, right) { + (None, None) => true, + (Some(left), Some(right)) => same_object(left, right), + _ => false, + } +} + +fn path_state_from_metadata(metadata: cap_std::fs::Metadata) -> PathState { + let file_type = metadata.file_type(); + let type_ = if file_type.is_dir() { + PathObjectType::Directory + } else if file_type.is_file() { + PathObjectType::RegularFile + } else if file_type.is_symlink() { + PathObjectType::SymbolicLink + } else { + PathObjectType::Other + }; + PathState { + identity: object_identity(&metadata), + type_, + size: metadata.len(), + } +} + +pub(crate) async fn descriptor_state(descriptor: &Descriptor) -> Result { + match descriptor { + Descriptor::File(file) => { + let file = Arc::clone(&file.file); + spawn_blocking(move || file.metadata().map(path_state_from_metadata)).await + } + Descriptor::Dir(dir) => { + let dir = Arc::clone(&dir.dir); + spawn_blocking(move || dir.dir_metadata().map(path_state_from_metadata)).await + } + } +} + +pub(crate) async fn path_state( + directory: &Dir, + path: &str, +) -> Result, std::io::Error> { + path_state_with_follow(directory, path, false).await +} + +pub(crate) async fn path_state_with_follow( + directory: &Dir, + path: &str, + follow_symlink: bool, +) -> Result, std::io::Error> { + let directory = Arc::clone(&directory.dir); + let path = path.to_string(); + spawn_blocking(move || { + match if follow_symlink { + directory.metadata(path) + } else { + directory.symlink_metadata(path) + } { + Ok(metadata) => Ok(Some(path_state_from_metadata(metadata))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } + }) + .await +} + +pub(crate) async fn read_link( + directory: &Dir, + path: &str, +) -> Result, std::io::Error> { + let directory = Arc::clone(&directory.dir); + let path = path.to_string(); + spawn_blocking(move || match directory.read_link(path) { + Ok(path) => path + .into_os_string() + .into_string() + .map(Some) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidData)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + }) + .await +} + +pub(crate) async fn symlink_state( + directory: &Dir, + path: &str, +) -> Result { + let object = path_state(directory, path).await?; + let target = if object.is_some_and(|state| state.type_ == PathObjectType::SymbolicLink) { + read_link(directory, path) + .await? + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "symlink disappeared while probing mutation postcondition", + ) + })? + .into() + } else { + None + }; + Ok(SymlinkState { object, target }) +} + +pub(crate) fn state_postcondition( + current: Result, std::io::Error>, + desired: impl FnOnce(Option) -> bool, + unchanged: impl FnOnce(Option) -> bool, +) -> MutationPostcondition { + match current { + Ok(current) if desired(current) => MutationPostcondition::Satisfied, + Ok(current) if unchanged(current) => MutationPostcondition::NoEffect, + Ok(_) | Err(_) => MutationPostcondition::Unknown, + } +} + +pub(crate) fn create_directory_postcondition( + before: Option, + current: Result, std::io::Error>, +) -> MutationPostcondition { + state_postcondition( + current, + |state| { + before.is_none() && state.is_some_and(|state| state.type_ == PathObjectType::Directory) + }, + |state| same_optional_object(before, state), + ) +} + +pub(crate) fn remove_postcondition( + before: Option, + current: Result, std::io::Error>, +) -> MutationPostcondition { + state_postcondition( + current, + |state| before.is_some() && state.is_none(), + |state| same_optional_object(before, state), + ) +} + +pub(crate) fn link_postcondition( + source_before: Option, + destination_before: Option, + source_after: Result, std::io::Error>, + destination_after: Result, std::io::Error>, +) -> MutationPostcondition { + match (source_before, source_after, destination_after) { + (Some(source), Ok(Some(current_source)), Ok(Some(destination))) + if same_object(destination, source) + && same_object(current_source, source) + && match destination_before { + None => true, + Some(before) => before + .identity + .zip(source.identity) + .is_some_and(|(before, source)| before != source), + } => + { + MutationPostcondition::Satisfied + } + (Some(source), Ok(Some(current_source)), Ok(None)) + if destination_before.is_none() && same_object(current_source, source) => + { + MutationPostcondition::NoEffect + } + (_, Ok(current_source), Ok(current_destination)) + if same_optional_object(source_before, current_source) + && same_optional_object(destination_before, current_destination) => + { + MutationPostcondition::NoEffect + } + _ => MutationPostcondition::Unknown, + } +} + +pub(crate) fn rename_postcondition( + source_before: Option, + destination_before: Option, + source_after: Result, std::io::Error>, + destination_after: Result, std::io::Error>, +) -> MutationPostcondition { + match (source_before, source_after, destination_after) { + (Some(source), Ok(None), Ok(Some(destination))) if same_object(destination, source) => { + MutationPostcondition::Satisfied + } + (Some(source), Ok(Some(current_source)), Ok(destination)) + if same_object(current_source, source) + && same_optional_object(destination_before, destination) => + { + MutationPostcondition::NoEffect + } + (_, Ok(current_source), Ok(current_destination)) + if same_optional_object(source_before, current_source) + && same_optional_object(destination_before, current_destination) => + { + MutationPostcondition::NoEffect + } + _ => MutationPostcondition::Unknown, + } +} + +pub(crate) fn symlink_postcondition( + before: &SymlinkState, + current: Result, + target: &str, +) -> MutationPostcondition { + match current { + Ok(current) + if before.target.as_deref() != Some(target) + && current.target.as_deref() == Some(target) => + { + MutationPostcondition::Satisfied + } + Ok(current) + if same_optional_object(before.object, current.object) + && before.target == current.target => + { + MutationPostcondition::NoEffect + } + Ok(_) | Err(_) => MutationPostcondition::Unknown, + } +} + +pub(crate) fn open_postcondition( + before: Option, + current: Result, std::io::Error>, + requested_type: PathObjectType, + truncate: bool, + exclusive: bool, +) -> MutationPostcondition { + match current { + Ok(Some(current)) + if current.type_ == requested_type + && (!truncate || current.size == 0) + && (!exclusive || before.is_none()) => + { + MutationPostcondition::Satisfied + } + Ok(current) + if match (before, current) { + (None, None) => true, + (Some(before), Some(current)) => { + same_object(before, current) && before.size == current.size + } + _ => false, + } => + { + MutationPostcondition::NoEffect + } + Ok(_) | Err(_) => MutationPostcondition::Unknown, + } +} + +pub(crate) fn resize_postcondition( + before: PathState, + current: Result, + size: u64, +) -> MutationPostcondition { + match current { + Ok(current) if current.size == size => MutationPostcondition::Satisfied, + Ok(current) if current.size == before.size => MutationPostcondition::NoEffect, + Ok(_) | Err(_) => MutationPostcondition::Unknown, + } +} + +fn times_state(metadata: cap_std::fs::Metadata) -> TimesState { + TimesState { + identity: object_identity(&metadata), + accessed: metadata.accessed().ok().map(|time| time.into_std()), + modified: metadata.modified().ok().map(|time| time.into_std()), + } +} + +pub(crate) async fn descriptor_times( + descriptor: &Descriptor, +) -> Result { + match descriptor { + Descriptor::File(file) => { + let file = Arc::clone(&file.file); + spawn_blocking(move || file.metadata().map(times_state)).await + } + Descriptor::Dir(dir) => { + let dir = Arc::clone(&dir.dir); + spawn_blocking(move || dir.dir_metadata().map(times_state)).await + } + } +} + +pub(crate) async fn path_times( + directory: &Dir, + path: &str, + follow_symlink: bool, +) -> Result { + let directory = Arc::clone(&directory.dir); + let path = path.to_string(); + spawn_blocking(move || { + if follow_symlink { + directory.metadata(path).map(times_state) + } else { + directory.symlink_metadata(path).map(times_state) + } + }) + .await +} + +fn requested_time_matches(requested: RequestedTime, actual: Option) -> bool { + match requested { + RequestedTime::NoChange => true, + RequestedTime::Now => false, + RequestedTime::Timestamp { + seconds, + nanoseconds, + } => actual.is_some_and(|actual| { + actual + .duration_since(SystemTime::UNIX_EPOCH) + .is_ok_and(|actual| { + i128::from(actual.as_secs()) == seconds && actual.subsec_nanos() == nanoseconds + }) + }), + } +} + +pub(crate) fn times_postcondition( + current: Result, + before: TimesState, + accessed: RequestedTime, + modified: RequestedTime, + identity_required: bool, +) -> MutationPostcondition { + match current { + Ok(current) + if requested_time_matches(accessed, current.accessed) + && requested_time_matches(modified, current.modified) => + { + MutationPostcondition::Satisfied + } + Ok(current) + if current.accessed == before.accessed + && current.modified == before.modified + && (!identity_required + || current + .identity + .zip(before.identity) + .is_some_and(|(current, before)| current == before)) => + { + MutationPostcondition::NoEffect + } + Ok(_) | Err(_) => MutationPostcondition::Unknown, + } +} diff --git a/golem-worker-executor/src/services/agent_filesystem/quota.rs b/golem-worker-executor/src/services/agent_filesystem/quota.rs new file mode 100644 index 0000000000..7932c91b03 --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/quota.rs @@ -0,0 +1,526 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use std::future::Future; +use std::pin::Pin; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct AgentFilesystemUsage { + pub allocated_bytes: u64, + pub filesystem_objects: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct AgentFilesystemStorageLimit { + pub allocated_bytes: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ResolvedAgentFilesystemLimits { + pub allocated_bytes: u64, + pub filesystem_objects: u64, + pub filesystem_object_limit_policy_version: u32, +} + +pub(super) type FilesystemLimitExceededCallback = + Arc Pin + Send>> + Send + Sync>; + +pub(super) const FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION: u32 = 2; +const BYTES_PER_GIB: u128 = 1024 * 1024 * 1024; + +impl FilesystemObjectLimitPolicyConfig { + pub(super) fn validate(&self) -> Result<(), FilesystemStorageError> { + let valid = self.objects_per_gib != 0 + && self.minimum_objects != 0 + && self.maximum_objects != 0 + && self.minimum_objects <= self.maximum_objects; + if valid { + Ok(()) + } else { + Err(FilesystemStorageError::verification( + "validate filesystem object limit policy", + Path::new(""), + )) + } + } + + pub(super) fn resolve( + &self, + limit: AgentFilesystemStorageLimit, + ) -> Result { + self.validate()?; + if limit.allocated_bytes == 0 { + return Err(FilesystemStorageError::verification( + "resolve nonzero agent filesystem storage limit", + Path::new(""), + )); + } + + let proportional = (u128::from(limit.allocated_bytes) * u128::from(self.objects_per_gib)) + .div_ceil(BYTES_PER_GIB); + let proportional = u64::try_from(proportional).map_err(|_| { + FilesystemStorageError::verification( + "derive agent filesystem object limit", + Path::new(""), + ) + })?; + + Ok(ResolvedAgentFilesystemLimits { + allocated_bytes: limit.allocated_bytes, + filesystem_objects: proportional.clamp(self.minimum_objects, self.maximum_objects), + filesystem_object_limit_policy_version: FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION, + }) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow( + dead_code, + reason = "authoritative capacity observation is part of the filesystem interface" +)] +pub(crate) struct FilesystemCapacity { + pub total_bytes: u64, + pub available_bytes: u64, + pub total_filesystem_objects: u64, + pub available_filesystem_objects: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct FilesystemPressure { + bytes: bool, + filesystem_objects: bool, +} + +impl FilesystemPressure { + pub(crate) fn include(self, pressure: Option) -> Self { + match pressure { + Some(pressure) => Self { + bytes: self.bytes || pressure.bytes, + filesystem_objects: self.filesystem_objects || pressure.filesystem_objects, + }, + None => self, + } + } +} + +impl FilesystemPressureConfig { + pub(super) fn validate(&self) -> Result<(), FilesystemStorageError> { + if self.minimum_available_bytes < self.target_available_bytes + && self.minimum_available_filesystem_objects < self.target_available_filesystem_objects + && self.reclamation_observation_attempts != 0 + { + Ok(()) + } else { + Err(FilesystemStorageError::verification( + "validate filesystem pressure watermarks", + Path::new(""), + )) + } + } + + pub(super) fn validate_capacity( + &self, + capacity: FilesystemCapacity, + ) -> Result<(), FilesystemStorageError> { + if self.target_available_bytes <= capacity.total_bytes { + Ok(()) + } else { + Err(FilesystemStorageError::verification( + "fit filesystem pressure byte target within managed capacity", + Path::new(""), + )) + } + } + + pub(crate) fn pressure( + &self, + operation: MutationOperation, + capacity: FilesystemCapacity, + ) -> Option { + let bytes = capacity.available_bytes <= self.minimum_available_bytes; + let filesystem_objects = operation == MutationOperation::Create + && capacity.available_filesystem_objects <= self.minimum_available_filesystem_objects; + (bytes || filesystem_objects).then_some(FilesystemPressure { + bytes, + filesystem_objects, + }) + } + + pub(crate) fn target_reached( + &self, + pressure: FilesystemPressure, + capacity: FilesystemCapacity, + ) -> bool { + (!pressure.bytes || capacity.available_bytes >= self.target_available_bytes) + && (!pressure.filesystem_objects + || capacity.available_filesystem_objects + >= self.target_available_filesystem_objects) + } +} + +impl AgentFilesystems { + #[allow( + dead_code, + reason = "authoritative capacity observation is part of the filesystem interface" + )] + pub(crate) async fn observe_capacity( + &self, + ) -> Result { + self.provisioner.observe_capacity().await + } + + pub(crate) fn pressure_policy(&self) -> &FilesystemPressureConfig { + &self.pressure + } +} + +impl AgentFilesystem { + #[allow( + dead_code, + reason = "authoritative usage observation is part of the filesystem interface" + )] + pub(crate) async fn usage( + &self, + ) -> Result, FilesystemStorageError> { + self.runtime.usage().await + } + + pub(crate) async fn settle_reconstruction(&self) -> Result<(), FilesystemStorageError> { + let _effect = self.runtime.begin_update_effect().await.map_err(|error| { + FilesystemStorageError::io( + "settle reconstructed agent filesystem", + self.runtime.inner.backend.root(), + std::io::Error::other(error), + ) + })?; + Ok(()) + } +} + +impl AgentFilesystemRuntime { + pub(crate) async fn usage( + &self, + ) -> Result, FilesystemStorageError> { + #[cfg(test)] + if self.inner.usage_observation_fails.load(Ordering::Acquire) { + return Err(FilesystemStorageError::verification( + "observe test agent filesystem usage", + self.inner.backend.root(), + )); + } + match self.inner.backend.quota() { + Some(quota) => quota.usage().await.map(Some), + None => Ok(None), + } + } + + pub(crate) fn set_usage_observer( + &self, + observer: Option>, + ) { + *self + .inner + .usage_observer + .lock() + .expect("agent filesystem usage-observer lock poisoned") = observer; + if self.has_active_effects() { + self.inner.schedule_usage_sampling(); + } + } + + pub(super) fn usage_observer_is_active(&self) -> bool { + self.inner + .usage_observer + .lock() + .expect("agent filesystem usage-observer lock poisoned") + .as_ref() + .is_some_and(|observer| observer.is_active()) + } + + pub(super) async fn observe_usage_for_billing(&self) -> Result<(), FilesystemStorageError> { + let observer = self + .inner + .usage_observer + .lock() + .expect("agent filesystem usage-observer lock poisoned") + .clone(); + let Some(observer) = observer else { + return Ok(()); + }; + let observation = observer.begin_observation(); + match self.usage().await { + Ok(usage) => { + observer.complete_observation(observation, usage, Instant::now()); + Ok(()) + } + Err(error) => { + if observer.fail_observation(observation) { + Err(error) + } else { + Ok(()) + } + } + } + } + + #[allow( + dead_code, + reason = "fresh physical capacity is part of the runtime filesystem interface" + )] + pub(crate) async fn observe_capacity( + &self, + ) -> Result { + self.inner.backend.observe_capacity().await + } + + #[allow( + dead_code, + reason = "fresh failure observations support runtime mutation classification" + )] + pub(super) async fn fresh_failure_observations( + &self, + ) -> Result< + ( + Option, + Option, + FilesystemCapacity, + ), + FilesystemStorageError, + > { + let observer = self + .inner + .usage_observer + .lock() + .expect("agent filesystem usage-observer lock poisoned") + .clone(); + let observation = observer + .as_ref() + .map(|observer| observer.begin_observation()); + + #[cfg(test)] + if let Some((usage, capacity)) = *self + .inner + .failure_observations + .read() + .expect("agent filesystem test observation lock poisoned") + { + let limits = *self + .inner + .applied_limits + .read() + .expect("agent filesystem applied-limit lock poisoned"); + if let (Some(observer), Some(observation)) = (observer, observation) { + observer.complete_observation(observation, usage, Instant::now()); + } + return Ok((usage, limits, capacity)); + } + + let installed_limits = *self + .inner + .applied_limits + .read() + .expect("agent filesystem applied-limit lock poisoned"); + let observations = async { + match self.inner.backend.quota() { + Some(quota) => quota + .failure_observations(installed_limits) + .await + .map(|(usage, limits)| (Some(usage), limits)), + None => Ok((None, None)), + } + }; + let (observations, capacity) = tokio::join!(observations, self.observe_capacity()); + let (usage, limits) = match observations { + Ok(observations) => observations, + Err(error) => { + if let (Some(observer), Some(observation)) = (&observer, observation) { + let _ = observer.fail_observation(observation); + } + return Err(error); + } + }; + if let (Some(observer), Some(observation)) = (&observer, observation) { + observer.complete_observation(observation, usage, Instant::now()); + } + let capacity = capacity?; + Ok((usage, limits, capacity)) + } + + pub(super) fn set_limit_exceeded_callback( + &self, + callback: Option, + ) { + *self + .inner + .limit_exceeded + .lock() + .expect("agent filesystem limit callback lock poisoned") = callback; + } + + pub(crate) fn is_same_runtime(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.inner, &other.inner) + } + + pub(crate) async fn set_allocated_byte_limit( + &self, + limit: AgentFilesystemStorageLimit, + ) -> Result<(), FilesystemStorageError> { + let effect = self.begin_update_effect().await.map_err(|error| { + FilesystemStorageError::io( + "admit agent filesystem limit update", + self.inner.backend.root(), + std::io::Error::other(error), + ) + })?; + let Some(quota) = self.inner.backend.quota() else { + return Ok(()); + }; + let installed = quota.install_limit(limit, effect.clone()).await?; + *self + .inner + .applied_limits + .write() + .expect("agent filesystem applied-limit lock poisoned") = Some(installed.limits); + let exceeded = installed.usage.allocated_bytes > installed.limits.allocated_bytes + || installed.usage.filesystem_objects > installed.limits.filesystem_objects; + self.notify_limit_state(exceeded).await; + drop(effect); + Ok(()) + } + + pub(super) async fn notify_limit_state(&self, exceeded: bool) { + let callback = self + .inner + .limit_exceeded + .lock() + .expect("agent filesystem limit callback lock poisoned") + .clone(); + if let Some(callback) = callback { + callback(exceeded).await; + } + } +} + +#[cfg(target_os = "linux")] +pub(super) fn validate_observed_limits( + root: &Path, + installed: Option, + observed: Option, +) -> Result<(), FilesystemStorageError> { + if installed == observed { + Ok(()) + } else { + Err(FilesystemStorageError::io( + "validate managed XFS project quota limits", + root, + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("installed limits {installed:?} differ from observed limits {observed:?}"), + ), + )) + } +} + +#[cfg(target_os = "linux")] +#[allow( + dead_code, + reason = "fresh physical capacity is part of the runtime filesystem interface" +)] +pub(super) async fn observe_path_capacity( + root: &Path, +) -> Result { + let root = root.to_path_buf(); + tokio::task::spawn_blocking({ + let root = root.clone(); + move || { + let capacity = rustix::fs::statvfs(&root) + .map_err(|error| std::io::Error::from_raw_os_error(error.raw_os_error()))?; + if capacity + .f_flag + .contains(rustix::fs::StatVfsMountFlags::RDONLY) + { + return Err(std::io::Error::new( + std::io::ErrorKind::ReadOnlyFilesystem, + "agent filesystem mount is read-only", + )); + } + #[cfg(target_os = "linux")] + let available_filesystem_objects = capacity.f_ffree; + #[cfg(not(target_os = "linux"))] + let available_filesystem_objects = capacity.f_favail; + capacity_from_values( + capacity.f_blocks, + capacity.f_bavail, + capacity.f_frsize, + capacity.f_files, + available_filesystem_objects, + ) + } + }) + .await + .map_err(|error| { + FilesystemStorageError::io( + "observe agent filesystem capacity", + &root, + std::io::Error::other(error), + ) + })? + .map_err(|error| FilesystemStorageError::io("observe agent filesystem capacity", &root, error)) +} + +#[cfg(not(target_os = "linux"))] +#[allow( + dead_code, + reason = "fresh physical capacity is part of the runtime filesystem interface" +)] +pub(super) async fn observe_path_capacity( + root: &Path, +) -> Result { + Err(FilesystemStorageError::io( + "observe agent filesystem capacity", + root, + std::io::Error::new( + std::io::ErrorKind::Unsupported, + "filesystem capacity observation requires statvfs", + ), + )) +} + +pub(super) fn capacity_from_values( + blocks: u64, + available_blocks: u64, + fragment_size: u64, + filesystem_objects: u64, + available_filesystem_objects: u64, +) -> std::io::Result { + let total_bytes = blocks.checked_mul(fragment_size).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "filesystem total capacity exceeds u64", + ) + })?; + let available_bytes = available_blocks.checked_mul(fragment_size).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "filesystem available capacity exceeds u64", + ) + })?; + Ok(FilesystemCapacity { + total_bytes, + available_bytes, + total_filesystem_objects: filesystem_objects, + available_filesystem_objects, + }) +} diff --git a/golem-worker-executor/src/services/agent_filesystem/tests.rs b/golem-worker-executor/src/services/agent_filesystem/tests.rs new file mode 100644 index 0000000000..d940175126 --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/tests.rs @@ -0,0 +1,2301 @@ +use super::*; +use crate::services::active_workers::MemoryGrant; +use crate::services::agent_resource_billing::{AgentResourceBilling, FilesystemUsageObserver}; +use crate::services::agent_storage_meter::FilesystemUsageObservation; +use crate::services::linear_memory::LinearMemoryTracker; +use async_trait::async_trait; +use golem_common::model::agent::AgentMode; +use golem_common::model::component::{AgentFilePath, AgentFilePermissions, ComponentId}; +use golem_common::model::environment::EnvironmentId; +use golem_common::model::{AgentId, OwnedAgentId}; +use golem_common::widen_infallible; +use golem_service_base::replayable_stream::ReplayableStream as _; +use golem_service_base::service::initial_agent_files::InitialAgentFilesService; +use golem_service_base::storage::blob::memory::InMemoryBlobStorage; +#[cfg(target_os = "linux")] +use std::fs::File; +#[cfg(target_os = "linux")] +use std::os::fd::AsRawFd; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use test_r::test; + +struct CountingUsageObserver { + active: AtomicBool, + begun: AtomicUsize, + completed: AtomicUsize, + failed: AtomicUsize, + reject_failures: AtomicBool, + completed_at: std::sync::Mutex>, +} + +impl Default for CountingUsageObserver { + fn default() -> Self { + Self { + active: AtomicBool::new(true), + begun: AtomicUsize::new(0), + completed: AtomicUsize::new(0), + failed: AtomicUsize::new(0), + reject_failures: AtomicBool::new(false), + completed_at: std::sync::Mutex::new(None), + } + } +} + +impl FilesystemUsageObserver for CountingUsageObserver { + fn is_active(&self) -> bool { + self.active.load(Ordering::Acquire) + } + + fn begin_observation(&self) -> FilesystemUsageObservation { + let sequence = self.begun.fetch_add(1, Ordering::AcqRel) as u64 + 1; + FilesystemUsageObservation { + generation: 1, + sequence, + } + } + + fn complete_observation( + &self, + _observation: FilesystemUsageObservation, + _usage: Option, + now: Instant, + ) { + *self.completed_at.lock().unwrap() = Some(now); + self.completed.fetch_add(1, Ordering::AcqRel); + } + + fn fail_observation(&self, _observation: FilesystemUsageObservation) -> bool { + self.failed.fetch_add(1, Ordering::AcqRel); + !self.reject_failures.load(Ordering::Acquire) + } +} + +struct PausedCleanup { + started: Arc, + release: Arc, + deleted: Arc, + used_fallback: Arc, +} + +#[async_trait] +impl backend::AgentFilesystemCleanup for PausedCleanup { + async fn delete(&mut self) -> Result<(), FilesystemStorageError> { + self.started.notify_one(); + self.release.notified().await; + self.deleted.store(true, Ordering::Release); + Ok(()) + } + + fn delete_blocking(&mut self) -> Result<(), FilesystemStorageError> { + self.used_fallback.store(true, Ordering::Release); + self.deleted.store(true, Ordering::Release); + Ok(()) + } +} + +fn agent_id() -> OwnedAgentId { + OwnedAgentId::new( + EnvironmentId::new(), + &AgentId::from_agent_name_string(ComponentId::new(), "agent").unwrap(), + ) +} + +async fn file_loader_with_content( + environment_id: EnvironmentId, + cache_parent: Option<&Path>, + content: &[u8], +) -> ( + Arc, + golem_common::model::agent::AgentFileContentHash, +) { + let service = Arc::new(InitialAgentFilesService::new(Arc::new( + InMemoryBlobStorage::new(), + ))); + let hash = service + .put_if_not_exists( + environment_id, + content + .to_vec() + .map_error(widen_infallible::) + .map_item(|item| item.map_err(widen_infallible::)), + ) + .await + .unwrap(); + ( + Arc::new(FileLoader::new(service, cache_parent).unwrap()), + hash, + ) +} + +fn initial_file( + content_hash: golem_common::model::agent::AgentFileContentHash, + path: &str, + permissions: AgentFilePermissions, + size: u64, +) -> InitialAgentFile { + InitialAgentFile { + content_hash, + path: AgentFilePath::from_abs_str(path).unwrap(), + permissions, + size, + } +} + +#[cfg(target_os = "linux")] +#[test] +async fn mutation_failure_preserves_guest_results_and_effect_evidence() { + let runtime = AgentFilesystemRuntime::new_for_test(); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::Guest("not-found"), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::PreserveGuest("not-found") + ); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<&str>::Io(std::io::Error::from_raw_os_error(libc::EBUSY)), + MutationEffect::KnownCompletedPrefix { bytes: 7 }, + ) + .await, + MutationDecision::BoundedRetry + ); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<&str>::Io(std::io::Error::other("unclassified")), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::BoundedRetry + ); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<&str>::Io(std::io::Error::from_raw_os_error(libc::EINTR)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::BoundedRetry + ); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<&str>::Io(std::io::Error::from_raw_os_error(libc::EAGAIN)), + MutationEffect::DesiredPostconditionSatisfied, + ) + .await, + MutationDecision::Success + ); + + let unknown_guest_runtime = AgentFilesystemRuntime::new_for_test(); + assert_eq!( + unknown_guest_runtime + .classify_mutation_failure(MutationFailure::Guest("access"), MutationEffect::Unknown) + .await, + MutationDecision::Invalidate + ); + assert!(unknown_guest_runtime.begin_effect().await.is_err()); +} + +#[cfg(target_os = "linux")] +#[test] +fn native_write_errors_only_claim_no_effect_for_explicitly_safe_causes() { + assert_eq!( + native_write_failure_effect(&std::io::Error::from_raw_os_error(libc::EAGAIN), 0), + MutationEffect::ProvenNoEffect + ); + assert_eq!( + native_write_failure_effect(&std::io::Error::from_raw_os_error(libc::EBUSY), 7), + MutationEffect::KnownCompletedPrefix { bytes: 7 } + ); + assert_eq!( + native_write_failure_effect(&std::io::Error::from_raw_os_error(libc::ENOSPC), 0), + MutationEffect::ProvenNoEffect + ); + assert_eq!( + native_write_failure_effect(&std::io::Error::from_raw_os_error(libc::EINTR), 0), + MutationEffect::Unknown + ); + assert_eq!( + native_write_failure_effect(&std::io::Error::from(std::io::ErrorKind::TimedOut), 3), + MutationEffect::Unknown + ); + assert_eq!( + native_write_failure_effect(&std::io::Error::other("unclassified"), 0), + MutationEffect::Unknown + ); +} + +#[test] +async fn unexplained_raw_permission_failure_invalidates_runtime() { + let runtime = AgentFilesystemRuntime::new_for_test(); + + assert_eq!( + runtime + .classify_mutation_failure::<()>( + MutationFailure::Io(std::io::Error::from(std::io::ErrorKind::PermissionDenied,)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::Invalidate + ); + assert!(runtime.begin_effect().await.is_err()); +} + +#[cfg(target_os = "linux")] +#[test] +async fn stale_or_disappeared_backing_device_invalidates_runtime() { + for errno in [libc::ESTALE, libc::ENODEV] { + let runtime = AgentFilesystemRuntime::new_for_test(); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(errno)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::Invalidate + ); + assert!(runtime.begin_effect().await.is_err()); + } +} + +#[cfg(target_os = "linux")] +#[test] +fn wrapped_terminal_probe_errors_are_terminal() { + for errno in [libc::EIO, libc::ESTALE, libc::ENODEV] { + let error = FilesystemStorageError::io( + "probe runtime filesystem", + Path::new(""), + std::io::Error::from_raw_os_error(errno), + ); + assert!(error.is_terminal_failure()); + } +} + +#[cfg(target_os = "linux")] +#[test] +fn changed_or_missing_live_xfs_limits_are_terminal() { + let installed = ResolvedAgentFilesystemLimits { + allocated_bytes: 1024 * 1024, + filesystem_objects: 8192, + filesystem_object_limit_policy_version: FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION, + }; + for observed in [ + None, + Some(ResolvedAgentFilesystemLimits { + allocated_bytes: 2 * 1024 * 1024, + ..installed + }), + ] { + let error = quota::validate_observed_limits( + Path::new(""), + Some(installed), + observed, + ) + .unwrap_err(); + assert!(error.is_terminal_failure()); + } +} + +#[cfg(target_os = "linux")] +#[test] +async fn terminal_cause_invalidates_even_when_postcondition_is_satisfied() { + let runtime = AgentFilesystemRuntime::new_for_test(); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::EIO)), + MutationEffect::DesiredPostconditionSatisfied, + ) + .await, + MutationDecision::Invalidate + ); + assert!(runtime.begin_effect().await.is_err()); +} + +#[cfg(target_os = "linux")] +#[test] +async fn byte_mutation_ignores_exhausted_physical_inode_dimension() { + let runtime = AgentFilesystemRuntime::new_for_test_with_observations( + None, + None, + FilesystemCapacity { + total_bytes: 100, + available_bytes: 50, + total_filesystem_objects: 100, + available_filesystem_objects: 0, + }, + ); + assert_eq!( + runtime + .classify_mutation_failure_for( + MutationOperation::Write, + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::EDQUOT)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::Quota + ); +} + +#[cfg(target_os = "linux")] +#[test] +async fn storage_exhaustion_uses_fresh_quota_and_capacity_observations() { + let exhausted = FilesystemCapacity { + total_bytes: 100, + available_bytes: 0, + total_filesystem_objects: 100, + available_filesystem_objects: 0, + }; + let runtime = AgentFilesystemRuntime::new_for_test_with_observations( + Some(AgentFilesystemUsage { + allocated_bytes: 50, + filesystem_objects: 10, + }), + Some(ResolvedAgentFilesystemLimits { + allocated_bytes: 50, + filesystem_objects: 10, + filesystem_object_limit_policy_version: FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION, + }), + exhausted, + ); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::ENOSPC)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::Quota + ); + + let unmanaged = AgentFilesystemRuntime::new_for_test_with_observations(None, None, exhausted); + assert_eq!( + unmanaged + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::EDQUOT)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::Quota + ); + assert_eq!( + unmanaged + .classify_mutation_failure( + MutationFailure::StorageExhaustion { + guest: (), + quota_hint: true, + }, + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::Quota + ); +} + +#[cfg(target_os = "linux")] +#[test] +async fn unexplained_storage_exhaustion_preserves_errno_mapping() { + let healthy = FilesystemCapacity { + total_bytes: 100, + available_bytes: 50, + total_filesystem_objects: 100, + available_filesystem_objects: 50, + }; + let runtime = AgentFilesystemRuntime::new_for_test_with_observations(None, None, healthy); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::EDQUOT)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::Quota + ); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::ENOSPC)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::InsufficientSpace + ); +} + +#[cfg(target_os = "linux")] +#[test] +async fn quota_classification_uses_the_operation_relevant_limit() { + let capacity = FilesystemCapacity { + total_bytes: 100, + available_bytes: 50, + total_filesystem_objects: 100, + available_filesystem_objects: 50, + }; + let runtime = AgentFilesystemRuntime::new_for_test_with_observations( + Some(AgentFilesystemUsage { + allocated_bytes: 50, + filesystem_objects: 10, + }), + Some(ResolvedAgentFilesystemLimits { + allocated_bytes: 100, + filesystem_objects: 10, + filesystem_object_limit_policy_version: FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION, + }), + capacity, + ); + + assert_eq!( + runtime + .classify_mutation_failure_for( + MutationOperation::Write, + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::ENOSPC)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::InsufficientSpace + ); + assert_eq!( + runtime + .classify_mutation_failure_for( + MutationOperation::Create, + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::ENOSPC)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::Quota + ); +} + +#[cfg(target_os = "linux")] +#[test] +async fn storage_probe_failure_preserves_errno_when_effect_is_known() { + let runtime = AgentFilesystemRuntime::new_for_test_with_capacity_observation_failure(); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::ENOSPC)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::InsufficientSpace + ); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::EDQUOT)), + MutationEffect::KnownCompletedPrefix { bytes: 3 }, + ) + .await, + MutationDecision::Quota + ); + assert!(runtime.begin_effect().await.is_ok()); +} + +#[test] +async fn proven_no_effect_guest_failure_does_not_observe_usage() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let observer = Arc::new(CountingUsageObserver::default()); + runtime.set_usage_observer(Some(observer.clone())); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::Guest("not-found"), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::PreserveGuest("not-found") + ); + assert_eq!(observer.begun.load(Ordering::Acquire), 0); + assert_eq!(observer.completed.load(Ordering::Acquire), 0); + assert_eq!(observer.failed.load(Ordering::Acquire), 0); +} + +#[test] +async fn completed_prefix_is_observed_before_early_invalidation() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let observer = Arc::new(CountingUsageObserver::default()); + let observed_at_invalidation = Arc::new(AtomicUsize::new(0)); + runtime.set_usage_observer(Some(observer.clone())); + runtime.set_invalidation_callback(Some({ + let observer = Arc::clone(&observer); + let observed_at_invalidation = Arc::clone(&observed_at_invalidation); + Arc::new(move || { + let completed = observer.completed.load(Ordering::Acquire); + let observed_at_invalidation = Arc::clone(&observed_at_invalidation); + Box::pin(async move { + observed_at_invalidation.store(completed, Ordering::Release); + }) + }) + })); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Infrastructure(std::io::Error::other( + "terminal backend failure", + )), + MutationEffect::KnownCompletedPrefix { bytes: 3 }, + ) + .await, + MutationDecision::Invalidate + ); + assert_eq!(observer.completed.load(Ordering::Acquire), 1); + assert_eq!(observed_at_invalidation.load(Ordering::Acquire), 1); + assert!(runtime.begin_effect().await.is_err()); +} + +#[test] +async fn known_effect_preserves_classifier_behavior_without_billing_observer() { + let runtime = AgentFilesystemRuntime::new_for_test_with_failed_observations(); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::Guest("not-found"), + MutationEffect::KnownCompletedPrefix { bytes: 3 }, + ) + .await, + MutationDecision::PreserveGuest("not-found") + ); + assert!(runtime.begin_effect().await.is_ok()); +} + +#[test] +async fn billing_usage_failure_invalidates_before_classification() { + let runtime = AgentFilesystemRuntime::new_for_test_with_failed_observations(); + let observer = Arc::new(CountingUsageObserver::default()); + runtime.set_usage_observer(Some(observer.clone())); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::Guest("not-found"), + MutationEffect::KnownCompletedPrefix { bytes: 3 }, + ) + .await, + MutationDecision::Invalidate + ); + assert_eq!(observer.completed.load(Ordering::Acquire), 0); + assert_eq!(observer.failed.load(Ordering::Acquire), 1); + assert!(runtime.begin_effect().await.is_err()); +} + +#[test] +async fn successful_usage_is_installed_before_capacity_observation_failure() { + let runtime = AgentFilesystemRuntime::new_for_test_with_capacity_observation_failure(); + let observer = Arc::new(CountingUsageObserver::default()); + runtime.set_usage_observer(Some(observer.clone())); + + let decision = runtime + .classify_mutation_failure( + MutationFailure::TransientGuest("busy"), + MutationEffect::ProvenNoEffect, + ) + .await; + + assert_eq!(decision, MutationDecision::PreserveGuest("busy")); + assert_eq!(observer.completed.load(Ordering::Acquire), 1); + assert_eq!(observer.failed.load(Ordering::Acquire), 0); + assert!(runtime.begin_effect().await.is_ok()); +} + +#[cfg(target_os = "linux")] +#[test] +async fn invalidating_mutation_failure_seals_runtime_and_notifies_once() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let notifications = Arc::new(AtomicUsize::new(0)); + runtime.set_invalidation_callback(Some({ + let notifications = Arc::clone(¬ifications); + Arc::new(move || { + let notifications = Arc::clone(¬ifications); + Box::pin(async move { + notifications.fetch_add(1, Ordering::AcqRel); + }) + }) + })); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::EINTR)), + MutationEffect::Unknown, + ) + .await, + MutationDecision::Invalidate + ); + assert!(runtime.begin_effect().await.is_err()); + assert_eq!(notifications.load(Ordering::Acquire), 1); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::EIO)), + MutationEffect::DesiredPostconditionSatisfied, + ) + .await, + MutationDecision::Invalidate + ); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Infrastructure(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "backend policy rejected access", + )), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::Invalidate + ); + assert_eq!(notifications.load(Ordering::Acquire), 1); +} + +#[cfg(target_os = "linux")] +#[test] +async fn pending_worker_interrupt_suppresses_mutation_retry() { + let runtime = AgentFilesystemRuntime::new_for_test(); + runtime.set_retry_callback(Some(Arc::new(|| Box::pin(async { false })))); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::TransientGuest("busy"), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::PreserveGuest("busy") + ); + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::EAGAIN)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::PreserveRaw + ); +} + +#[cfg(target_os = "linux")] +#[test] +async fn failed_health_probe_suppresses_transient_retry() { + let runtime = AgentFilesystemRuntime::new_for_test_with_capacity_observation_failure(); + + assert_eq!( + runtime + .classify_mutation_failure( + MutationFailure::<()>::Io(std::io::Error::from_raw_os_error(libc::EAGAIN)), + MutationEffect::ProvenNoEffect, + ) + .await, + MutationDecision::PreserveRaw + ); + assert!(runtime.begin_effect().await.is_ok()); +} + +#[cfg(target_os = "linux")] +#[test] +async fn unmanaged_runtime_observes_fresh_physical_capacity() { + let filesystems = AgentFilesystems::new(&FilesystemStorageConfig::default()).unwrap(); + let filesystem = filesystems.create_owned_empty(&agent_id()).await.unwrap(); + + let capacity = filesystem.runtime().observe_capacity().await.unwrap(); + + assert!(capacity.total_bytes > 0); + assert!(capacity.available_bytes <= capacity.total_bytes); + assert!(capacity.total_filesystem_objects > 0); + assert!(capacity.available_filesystem_objects <= capacity.total_filesystem_objects); + filesystem.close_and_delete().await.unwrap(); +} + +#[test] +async fn unpublished_backend_filesystem_cleans_up_on_drop() { + let storage_root = tempfile::tempdir().unwrap(); + let backend = unmanaged::UnmanagedBackend::new( + Some(storage_root.path().to_path_buf()), + RetryConfig::default(), + ); + let created = backend.provision_for(&agent_id()).await.unwrap(); + let path = created.backend().root().to_path_buf(); + + drop(created); + + assert!(!path.exists()); +} + +#[test] +async fn short_effect_batch_is_observed_on_the_bounded_cadence() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let observer = Arc::new(CountingUsageObserver::default()); + runtime.set_usage_observer(Some(observer.clone())); + let first = runtime.begin_effect().await.unwrap(); + let second = runtime.begin_effect().await.unwrap(); + + drop(first); + drop(second); + runtime.drain().await; + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while observer.completed.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + assert_eq!(observer.completed.load(Ordering::Acquire), 1); + assert_eq!(observer.begun.load(Ordering::Acquire), 1); +} + +#[test] +async fn short_effect_final_sample_is_debounced_from_drain() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let observer = Arc::new(CountingUsageObserver::default()); + runtime.set_usage_observer(Some(observer.clone())); + // Start the sampler after the drain so scheduler load cannot race the debounce assertion. + runtime.inner.usage_sampling.store(true, Ordering::Release); + let effect = runtime.begin_effect().await.unwrap(); + + let drained_at = Instant::now(); + drop(effect); + runtime.drain().await; + runtime.inner.usage_sampling.store(false, Ordering::Release); + runtime.inner.schedule_usage_sampling(); + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while observer.completed.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(observer.completed.load(Ordering::Acquire), 1); + assert!( + observer.completed_at.lock().unwrap().unwrap() - drained_at + >= std::time::Duration::from_millis(10) + ); +} + +#[test] +async fn paused_effect_admission_drains_existing_effects_and_rejects_new_ones() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let effect = runtime.begin_effect().await.unwrap(); + let admission_pause = runtime.pause_effect_admission(); + + assert!(runtime.begin_effect().await.is_err()); + drop(effect); + runtime.drain().await; + drop(admission_pause); + + assert!(runtime.begin_effect().await.is_ok()); +} + +#[test] +async fn update_effect_waits_for_paused_admission_to_resume() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let admission_pause = runtime.pause_effect_admission(); + let update = tokio::spawn({ + let runtime = runtime.clone(); + async move { runtime.begin_update_effect().await } + }); + + tokio::task::yield_now().await; + assert!(!update.is_finished()); + drop(admission_pause); + + assert!(update.await.unwrap().is_ok()); +} + +#[test] +async fn sampler_exit_hands_off_to_an_effect_admitted_during_teardown() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let observer = Arc::new(CountingUsageObserver::default()); + runtime.set_usage_observer(Some(observer.clone())); + runtime.inner.usage_sampling.store(true, Ordering::Release); + let effect = runtime.begin_effect().await.unwrap(); + + runtime.inner.finish_usage_sampling(0); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while observer.completed.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + drop(effect); +} + +#[test] +async fn sampler_exit_does_not_restart_an_inactive_window() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let observer = Arc::new(CountingUsageObserver::default()); + observer.active.store(false, Ordering::Release); + runtime.set_usage_observer(Some(observer.clone())); + runtime.inner.usage_sampling.store(true, Ordering::Release); + let effect = runtime.begin_effect().await.unwrap(); + + runtime.inner.finish_usage_sampling(0); + + assert!(!runtime.inner.usage_sampling.load(Ordering::Acquire)); + assert_eq!(observer.begun.load(Ordering::Acquire), 0); + drop(effect); +} + +#[test] +fn sampler_exit_does_not_restart_without_pending_effects() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let observer = Arc::new(CountingUsageObserver::default()); + runtime.set_usage_observer(Some(observer.clone())); + runtime.inner.usage_sampling.store(true, Ordering::Release); + + runtime.inner.finish_usage_sampling(0); + + assert!(!runtime.inner.usage_sampling.load(Ordering::Acquire)); + assert_eq!(observer.begun.load(Ordering::Acquire), 0); +} + +#[test] +async fn sustained_effects_use_a_slower_cadence_until_completion() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let observer = Arc::new(CountingUsageObserver::default()); + runtime.set_usage_observer(Some(observer.clone())); + let effect = runtime.begin_effect().await.unwrap(); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while observer.completed.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let first_sample_at = observer.completed_at.lock().unwrap().unwrap(); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while observer.completed.load(Ordering::Acquire) < 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let second_sample_at = observer.completed_at.lock().unwrap().unwrap(); + assert!(second_sample_at - first_sample_at >= std::time::Duration::from_millis(100)); + assert!(runtime.has_active_effects()); + + let samples_before_completion = observer.completed.load(Ordering::Acquire); + drop(effect); + runtime.drain().await; + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while observer.completed.load(Ordering::Acquire) <= samples_before_completion { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +#[test] +async fn failed_scheduled_usage_observation_invalidates_runtime() { + let runtime = AgentFilesystemRuntime::new_for_test_with_failed_observations(); + let observer = Arc::new(CountingUsageObserver::default()); + let invalidated = Arc::new(AtomicBool::new(false)); + runtime.set_usage_observer(Some(observer.clone())); + runtime.set_invalidation_callback(Some({ + let invalidated = Arc::clone(&invalidated); + Arc::new(move || { + let invalidated = Arc::clone(&invalidated); + Box::pin(async move { + invalidated.store(true, Ordering::Release); + }) + }) + })); + let effect = runtime.begin_effect().await.unwrap(); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while !invalidated.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + assert_eq!(observer.failed.load(Ordering::Acquire), 1); + assert!(runtime.begin_effect().await.is_err()); + drop(effect); +} + +#[test] +async fn rejected_scheduled_usage_failure_does_not_invalidate_runtime() { + let runtime = AgentFilesystemRuntime::new_for_test_with_failed_observations(); + let observer = Arc::new(CountingUsageObserver::default()); + observer.reject_failures.store(true, Ordering::Release); + let invalidated = Arc::new(AtomicBool::new(false)); + runtime.set_usage_observer(Some(observer.clone())); + runtime.set_invalidation_callback(Some({ + let invalidated = Arc::clone(&invalidated); + Arc::new(move || { + let invalidated = Arc::clone(&invalidated); + Box::pin(async move { + invalidated.store(true, Ordering::Release); + }) + }) + })); + let effect = runtime.begin_effect().await.unwrap(); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while observer.failed.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + drop(effect); + runtime.drain().await; + + assert!(!invalidated.load(Ordering::Acquire)); + assert!(runtime.begin_effect().await.is_ok()); +} + +#[test] +async fn rejected_forced_usage_failure_is_ignored() { + let runtime = AgentFilesystemRuntime::new_for_test_with_failed_observations(); + let observer = Arc::new(CountingUsageObserver::default()); + observer.reject_failures.store(true, Ordering::Release); + runtime.set_usage_observer(Some(observer.clone())); + + runtime.observe_usage_for_billing().await.unwrap(); + + assert_eq!(observer.failed.load(Ordering::Acquire), 1); + assert!(runtime.begin_effect().await.is_ok()); +} + +#[test] +fn pressure_policy_uses_independent_minimum_and_target_watermarks() { + let policy = FilesystemPressureConfig { + minimum_available_bytes: 10, + target_available_bytes: 20, + minimum_available_filesystem_objects: 2, + target_available_filesystem_objects: 4, + ..FilesystemPressureConfig::default() + }; + let byte_pressure = policy + .pressure( + MutationOperation::Write, + FilesystemCapacity { + total_bytes: 100, + available_bytes: 10, + total_filesystem_objects: 100, + available_filesystem_objects: 100, + }, + ) + .unwrap(); + assert!(!policy.target_reached( + byte_pressure, + FilesystemCapacity { + total_bytes: 100, + available_bytes: 19, + total_filesystem_objects: 100, + available_filesystem_objects: 100, + } + )); + assert!(policy.target_reached( + byte_pressure, + FilesystemCapacity { + total_bytes: 100, + available_bytes: 20, + total_filesystem_objects: 100, + available_filesystem_objects: 0, + } + )); + + let object_capacity = FilesystemCapacity { + total_bytes: 100, + available_bytes: 100, + total_filesystem_objects: 100, + available_filesystem_objects: 2, + }; + assert!( + policy + .pressure(MutationOperation::Write, object_capacity) + .is_none() + ); + let object_pressure = policy + .pressure(MutationOperation::Create, object_capacity) + .unwrap(); + assert!(policy.target_reached( + object_pressure, + FilesystemCapacity { + available_filesystem_objects: 4, + ..object_capacity + } + )); +} + +#[test] +fn pressure_policy_rejects_targets_below_minimums() { + assert!( + FilesystemPressureConfig { + minimum_available_bytes: 2, + target_available_bytes: 1, + minimum_available_filesystem_objects: 1, + target_available_filesystem_objects: 1, + ..FilesystemPressureConfig::default() + } + .validate() + .is_err() + ); + assert!( + FilesystemPressureConfig { + minimum_available_bytes: 1, + target_available_bytes: 1, + minimum_available_filesystem_objects: 1, + target_available_filesystem_objects: 1, + ..FilesystemPressureConfig::default() + } + .validate() + .is_err() + ); + assert!( + FilesystemPressureConfig { + minimum_available_bytes: 1, + target_available_bytes: 101, + minimum_available_filesystem_objects: 1, + target_available_filesystem_objects: 1, + ..FilesystemPressureConfig::default() + } + .validate_capacity(FilesystemCapacity { + total_bytes: 100, + available_bytes: 100, + total_filesystem_objects: 1, + available_filesystem_objects: 1, + }) + .is_err() + ); +} + +#[test] +fn default_object_limit_policy_resolves_storage_levels() { + let policy = FilesystemObjectLimitPolicyConfig::default(); + + assert_eq!( + policy + .resolve(AgentFilesystemStorageLimit { + allocated_bytes: 128 * 1024 * 1024, + },) + .unwrap(), + ResolvedAgentFilesystemLimits { + allocated_bytes: 128 * 1024 * 1024, + filesystem_objects: 8_192, + filesystem_object_limit_policy_version: 2, + } + ); + assert_eq!( + policy + .resolve(AgentFilesystemStorageLimit { + allocated_bytes: 384 * 1024 * 1024, + },) + .unwrap() + .filesystem_objects, + 12_288 + ); + assert_eq!( + policy + .resolve(AgentFilesystemStorageLimit { + allocated_bytes: 1024 * 1024 * 1024, + },) + .unwrap() + .filesystem_objects, + 32_768 + ); +} + +#[test] +fn object_limit_policy_rejects_unrepresentable_inputs() { + let policy = FilesystemObjectLimitPolicyConfig::default(); + + assert!( + policy + .resolve(AgentFilesystemStorageLimit { allocated_bytes: 0 }) + .is_err() + ); + let overflowing = FilesystemObjectLimitPolicyConfig { + objects_per_gib: u64::MAX, + maximum_objects: u64::MAX, + ..policy.clone() + }; + assert!( + overflowing + .resolve(AgentFilesystemStorageLimit { + allocated_bytes: u64::MAX, + }) + .is_err() + ); + + let invalid = FilesystemObjectLimitPolicyConfig { + objects_per_gib: 0, + ..policy + }; + assert!(invalid.validate().is_err()); +} + +#[cfg(target_os = "linux")] +#[test] +fn managed_backend_fails_closed_on_non_xfs() { + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + managed_xfs_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + + let error = match AgentFilesystems::new(&settings) { + Ok(_) => panic!("managed backend unexpectedly accepted a non-XFS root"), + Err(error) => error, + }; + + assert!(error.to_string().contains("validate managed XFS root")); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +async fn managed_xfs_owns_observes_and_cleans_project_filesystem() { + let root = std::env::var_os("GOLEM_MANAGED_XFS_TEST_ROOT") + .map(PathBuf::from) + .expect("GOLEM_MANAGED_XFS_TEST_ROOT must name the mounted XFS test root"); + let settings = FilesystemStorageConfig { + managed_xfs_root_dir: Some(root.clone()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + + let second_owner = AgentFilesystems::new(&settings); + assert!(second_owner.is_err()); + + let escaped_id = agent_id(); + let outside = tempfile::tempdir().unwrap(); + let environment_link = root.join(escaped_id.environment_id.to_string()); + std::os::unix::fs::symlink(outside.path(), &environment_link).unwrap(); + assert!(filesystems.create_owned_empty(&escaped_id).await.is_err()); + assert!(std::fs::read_dir(outside.path()).unwrap().next().is_none()); + std::fs::remove_file(environment_link).unwrap(); + + let stale_file_id = agent_id(); + let backend = filesystems + .provisioner + .as_any() + .downcast_ref::() + .expect("configured backend must be XFS") + .clone(); + let environment = stale_file_id.environment_id.to_string(); + let component = stale_file_id.agent_id.component_id.to_string(); + let agent = stale_file_id.agent_id.agent_name_encoded(); + let owner = PathBuf::from(&environment).join(&component).join(&agent); + let parent = backend.open_agent_parent(&environment, &component).unwrap(); + let parent_path = PathBuf::from(format!("/proc/self/fd/{}", parent.as_raw_fd())); + let stale_file = parent_path.join(&agent); + let staging = parent_path.join(format!("{agent}.staging")); + std::fs::create_dir(&staging).unwrap(); + let stale_project = backend.reserve_project(&owner).unwrap(); + let staging_directory = File::open(&staging).unwrap(); + backend + .assign_project(&staging_directory, stale_project) + .unwrap(); + std::fs::write(staging.join("file"), b"stale").unwrap(); + std::fs::rename(staging.join("file"), &stale_file).unwrap(); + drop(staging_directory); + std::fs::remove_dir(staging).unwrap(); + drop(parent); + + let stale_file_replacement = filesystems + .create_owned_empty(&stale_file_id) + .await + .unwrap(); + assert!(stale_file_replacement.path().is_dir()); + stale_file_replacement.close_and_delete().await.unwrap(); + assert_eq!( + backend.usage(stale_project).unwrap(), + AgentFilesystemUsage { + allocated_bytes: 0, + filesystem_objects: 0, + } + ); + + let capacity = filesystems.observe_capacity().await.unwrap(); + assert!(capacity.total_bytes > 0); + assert!(capacity.available_bytes <= capacity.total_bytes); + assert!(capacity.total_filesystem_objects > 0); + assert!(capacity.available_filesystem_objects <= capacity.total_filesystem_objects); + + let materialized_id = agent_id(); + let content = vec![0x5a; 8192]; + let (file_loader, content_hash) = file_loader_with_content( + materialized_id.environment_id, + filesystems.initial_file_cache_root(), + &content, + ) + .await; + let cached_source = file_loader + .get_source( + materialized_id.environment_id, + content_hash, + content.len() as u64, + ) + .await + .unwrap(); + let managed_backend = filesystems + .provisioner + .as_any() + .downcast_ref::() + .expect("configured backend must be XFS"); + assert_eq!( + managed_backend + .project_id(&File::open(cached_source.path()).unwrap()) + .unwrap(), + None, + "the shared cache source must not inherit an agent project" + ); + let filesystem = filesystems + .create_fresh(CreateAgentFilesystem { + agent_id: materialized_id.clone(), + initial_files: vec![ + initial_file( + content_hash, + "/immutable-a", + AgentFilePermissions::ReadOnly, + content.len() as u64, + ), + initial_file( + content_hash, + "/immutable-b", + AgentFilePermissions::ReadOnly, + content.len() as u64, + ), + initial_file( + content_hash, + "/writable", + AgentFilePermissions::ReadWrite, + content.len() as u64, + ), + ], + file_loader: Arc::clone(&file_loader), + resource_limits: None, + limit_exceeded: None, + }) + .await + .unwrap(); + let path = filesystem.path().to_path_buf(); + let project_id = xfs::project_id_for_test(filesystem.runtime.inner.backend.as_ref()); + let materialized_usage = filesystem.usage().await.unwrap().unwrap(); + assert!(materialized_usage.allocated_bytes >= 3 * 8192); + assert!(materialized_usage.filesystem_objects >= 4); + assert_eq!(std::fs::read(path.join("immutable-a")).unwrap(), content); + assert_eq!(std::fs::read(path.join("immutable-b")).unwrap(), content); + assert_eq!(std::fs::read(path.join("writable")).unwrap(), content); + let immutable_a = File::open(path.join("immutable-a")).unwrap(); + let immutable_b = File::open(path.join("immutable-b")).unwrap(); + let writable = File::open(path.join("writable")).unwrap(); + assert_eq!(backend.project_id(&immutable_a).unwrap(), Some(project_id)); + assert_eq!(backend.project_id(&immutable_b).unwrap(), Some(project_id)); + assert_eq!(backend.project_id(&writable).unwrap(), Some(project_id)); + drop((immutable_a, immutable_b, writable)); + + filesystem + .runtime() + .set_allocated_byte_limit(AgentFilesystemStorageLimit { + allocated_bytes: 128 * 1024 * 1024, + }) + .await + .unwrap(); + + filesystem + .runtime() + .update_initial_files( + &file_loader, + materialized_id.environment_id, + &[ + initial_file( + content_hash, + "/immutable-a", + AgentFilePermissions::ReadOnly, + content.len() as u64, + ), + initial_file( + content_hash, + "/immutable-c", + AgentFilePermissions::ReadOnly, + content.len() as u64, + ), + initial_file( + content_hash, + "/writable", + AgentFilePermissions::ReadWrite, + content.len() as u64, + ), + ], + ) + .await + .unwrap(); + assert!(!path.join("immutable-b").exists()); + assert_eq!(std::fs::read(path.join("immutable-c")).unwrap(), content); + let immutable_c = File::open(path.join("immutable-c")).unwrap(); + assert_eq!(backend.project_id(&immutable_c).unwrap(), Some(project_id)); + drop(immutable_c); + + let usage_before_cow = filesystem.usage().await.unwrap().unwrap(); + std::fs::write(path.join("writable"), vec![0x6c; content.len()]).unwrap(); + assert_eq!(std::fs::read(path.join("immutable-a")).unwrap(), content); + assert_eq!(std::fs::read(path.join("immutable-c")).unwrap(), content); + let usage_after_cow = filesystem.usage().await.unwrap().unwrap(); + assert!(usage_after_cow.allocated_bytes >= usage_before_cow.allocated_bytes); + + use std::io::{Seek, SeekFrom, Write}; + let usage_before_sparse = filesystem.usage().await.unwrap().unwrap(); + let sparse_path = path.join("sparse"); + let mut sparse = std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&sparse_path) + .unwrap(); + sparse.seek(SeekFrom::Start(4 * 1024 * 1024)).unwrap(); + sparse.write_all(&[0x7d]).unwrap(); + sparse.sync_all().unwrap(); + rustix::fs::syncfs(File::open(&path).unwrap()).unwrap(); + let usage_after_sparse = filesystem.usage().await.unwrap().unwrap(); + assert_eq!( + std::fs::metadata(&sparse_path).unwrap().len(), + 4 * 1024 * 1024 + 1 + ); + assert!(usage_after_sparse.allocated_bytes > usage_before_sparse.allocated_bytes); + assert!( + usage_after_sparse.allocated_bytes - usage_before_sparse.allocated_bytes < 1024 * 1024, + "sparse logical extension must be charged by physical allocation" + ); + + let dense_path = path.join("dense"); + let mut dense = std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(&dense_path) + .unwrap(); + dense.write_all(&vec![0x4e; 4096]).unwrap(); + dense.sync_all().unwrap(); + let capacity_during_allocation = filesystems.observe_capacity().await.unwrap(); + assert!(capacity_during_allocation.available_bytes < capacity.available_bytes); + assert!( + capacity_during_allocation.available_filesystem_objects + < capacity.available_filesystem_objects + ); + let usage = filesystem.usage().await.unwrap().unwrap(); + assert!(usage.allocated_bytes > 4096); + let limit_exceeded = Arc::new(AtomicBool::new(false)); + filesystem.runtime().set_limit_exceeded_callback(Some({ + let limit_exceeded = Arc::clone(&limit_exceeded); + Arc::new(move |exceeded| { + let limit_exceeded = Arc::clone(&limit_exceeded); + Box::pin(async move { + if exceeded { + limit_exceeded.store(true, Ordering::Release); + } + }) + }) + })); + filesystem + .runtime() + .set_allocated_byte_limit(AgentFilesystemStorageLimit { + allocated_bytes: usage.allocated_bytes, + }) + .await + .unwrap(); + assert!(!limit_exceeded.load(Ordering::Acquire)); + dense.seek(SeekFrom::Start(0)).unwrap(); + dense.write_all(&vec![0x5f; 4096]).unwrap(); + dense.sync_all().unwrap(); + assert_eq!( + filesystem.usage().await.unwrap().unwrap().allocated_bytes, + usage.allocated_bytes, + "overwriting allocated blocks at quota equality must not consume capacity" + ); + let allocation_error = backend + .materialize_initial_file( + &path, + project_id, + cached_source.path(), + &path.join("exact-limit-allocation"), + false, + ) + .expect_err("allocating at the exact byte limit must be denied"); + assert!( + matches!( + allocation_error.kind(), + std::io::ErrorKind::StorageFull | std::io::ErrorKind::QuotaExceeded + ), + "unexpected exact-limit allocation error: {allocation_error:?}" + ); + drop((sparse, dense)); + filesystem + .runtime() + .set_allocated_byte_limit(AgentFilesystemStorageLimit { + allocated_bytes: usage.allocated_bytes - 4096, + }) + .await + .unwrap(); + assert!(limit_exceeded.load(Ordering::Acquire)); + + let retained_runtime = filesystem.runtime(); + filesystem.close_and_delete().await.unwrap(); + assert!(!path.exists()); + assert!( + !retained_runtime.inner.backend.root().exists(), + "a retained runtime must not keep the per-agent root descriptor open" + ); + assert_eq!( + backend.usage(project_id).unwrap(), + AgentFilesystemUsage { + allocated_bytes: 0, + filesystem_objects: 0, + } + ); + let mut capacity_after_deletion = filesystems.observe_capacity().await.unwrap(); + for _ in 0..50 { + if capacity_after_deletion.available_bytes > capacity_during_allocation.available_bytes + && capacity_after_deletion.available_filesystem_objects + > capacity_during_allocation.available_filesystem_objects + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + capacity_after_deletion = filesystems.observe_capacity().await.unwrap(); + } + assert!(capacity_after_deletion.available_bytes > capacity_during_allocation.available_bytes); + assert!( + capacity_after_deletion.available_filesystem_objects + > capacity_during_allocation.available_filesystem_objects + ); + + let over_limit_id = agent_id(); + let over_limit_content = vec![0x7b; 8192]; + let (over_limit_loader, over_limit_hash) = file_loader_with_content( + over_limit_id.environment_id, + filesystems.initial_file_cache_root(), + &over_limit_content, + ) + .await; + let error = filesystems + .create_fresh(CreateAgentFilesystem { + agent_id: over_limit_id, + initial_files: vec![initial_file( + over_limit_hash, + "/over-limit-initial-file", + AgentFilePermissions::ReadOnly, + over_limit_content.len() as u64, + )], + file_loader: over_limit_loader, + resource_limits: Some(Arc::new(AtomicResourceEntry::new( + u64::MAX, + usize::MAX, + usize::MAX, + 4096, + u64::MAX, + ))), + limit_exceeded: None, + }) + .await; + assert!( + error.is_err(), + "initial files above the installed byte limit must prevent startup" + ); + + let object_limited = filesystems.create_owned_empty(&agent_id()).await.unwrap(); + let object_backend = backend.clone(); + let object_project = xfs::project_id_for_test(object_limited.runtime.inner.backend.as_ref()); + object_backend + .install_project_limits( + object_project, + ResolvedAgentFilesystemLimits { + allocated_bytes: 128 * 1024 * 1024, + filesystem_objects: 2, + filesystem_object_limit_policy_version: FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION, + }, + ) + .unwrap(); + let object_path = object_limited.path().join("object"); + std::fs::write(&object_path, []).unwrap(); + std::fs::hard_link(&object_path, object_limited.path().join("alias")).unwrap(); + assert_eq!( + object_limited + .usage() + .await + .unwrap() + .unwrap() + .filesystem_objects, + 2 + ); + let object_error = std::fs::write(object_limited.path().join("exhausted"), []) + .expect_err("a new inode must exceed the project object limit"); + assert_eq!( + object_error.raw_os_error(), + Some(rustix::io::Errno::NOSPC.raw_os_error()) + ); + + let open_unlinked = File::open(&object_path).unwrap(); + std::fs::remove_file(&object_path).unwrap(); + std::fs::remove_file(object_limited.path().join("alias")).unwrap(); + assert_eq!( + object_limited + .usage() + .await + .unwrap() + .unwrap() + .filesystem_objects, + 2 + ); + drop(open_unlinked); + object_limited.close_and_delete().await.unwrap(); + assert_eq!( + object_backend.usage(object_project).unwrap(), + AgentFilesystemUsage { + allocated_bytes: 0, + filesystem_objects: 0, + } + ); + + let deferred = filesystems.create_owned_empty(&agent_id()).await.unwrap(); + let deferred_project = xfs::project_id_for_test(deferred.runtime.inner.backend.as_ref()); + let retained_root = File::open(deferred.path()).unwrap(); + drop(deferred); + drop(retained_root); + let mut released = false; + for _ in 0..500 { + if backend.usage(deferred_project).unwrap() + == (AgentFilesystemUsage { + allocated_bytes: 0, + filesystem_objects: 0, + }) + { + released = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert!(released, "deferred managed project cleanup did not finish"); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +async fn managed_xfs_allocated_bytes_flow_through_resource_billing() { + use std::io::{Seek, SeekFrom, Write}; + + let root = std::env::var_os("GOLEM_MANAGED_XFS_TEST_ROOT") + .map(PathBuf::from) + .expect("GOLEM_MANAGED_XFS_TEST_ROOT must name the mounted XFS test root"); + let filesystems = AgentFilesystems::new(&FilesystemStorageConfig { + managed_xfs_root_dir: Some(root), + ..FilesystemStorageConfig::default() + }) + .unwrap(); + let filesystem = filesystems.create_owned_empty(&agent_id()).await.unwrap(); + let runtime = filesystem.runtime(); + let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); + let now = Instant::now(); + let memory = LinearMemoryTracker::new( + 0, + 0, + AgentMode::Durable, + false, + entry.clone(), + Arc::new(std::sync::Mutex::new(MemoryGrant::inert(0))), + now, + ); + let meter = AgentResourceBilling::new(AgentMode::Durable, memory, entry.clone(), now); + runtime.set_usage_observer(Some(Arc::new(meter.clone()))); + let opening_usage = runtime.usage().await.unwrap().unwrap(); + let window_started = Instant::now(); + meter.open(&runtime).await.unwrap(); + + let effect = runtime.begin_effect().await.unwrap(); + let sparse_path = filesystem.path().join("billed-sparse-file"); + let mut sparse = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&sparse_path) + .unwrap(); + sparse.seek(SeekFrom::Start(64 * 1024 * 1024)).unwrap(); + sparse.write_all(&[0x7d]).unwrap(); + sparse.sync_all().unwrap(); + drop((sparse, effect)); + runtime.drain().await; + + let logical_bytes = std::fs::metadata(&sparse_path).unwrap().len(); + let usage = runtime.usage().await.unwrap().unwrap(); + assert!(usage.allocated_bytes > 0); + assert!(usage.allocated_bytes < logical_bytes); + runtime.observe_usage_for_billing().await.unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + meter.close(&runtime).await.unwrap(); + let elapsed = window_started.elapsed().as_secs_f64(); + meter.flush(Instant::now()); + let billed = entry.durable_byte_seconds_delta(); + let minimum = ((usage.allocated_bytes as f64 * 0.1).floor() as i64).max(1); + let maximum_level = opening_usage.allocated_bytes.max(usage.allocated_bytes); + let maximum = (maximum_level as f64 * (elapsed + 0.25)).ceil() as i64; + assert!( + billed >= minimum, + "authoritative allocation produced too little billing: usage={usage:?}, billed={billed}" + ); + assert!( + billed <= maximum, + "authoritative allocation produced too much billing: usage={usage:?}, elapsed={elapsed}, billed={billed}" + ); + assert!( + billed < logical_bytes as i64, + "sparse logical length was billed instead of authoritative allocation" + ); + + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + meter.flush(Instant::now()); + assert_eq!(entry.durable_byte_seconds_delta(), billed); + + runtime.set_usage_observer(None); + filesystem.close_and_delete().await.unwrap(); +} + +#[cfg(unix)] +#[test] +async fn unmanaged_materialization_creates_distinct_owned_files() { + use std::os::unix::fs::MetadataExt; + + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + deterministic_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + let id = agent_id(); + let content = b"shared initial content"; + let (file_loader, content_hash) = + file_loader_with_content(id.environment_id, None, content).await; + let filesystem = filesystems + .create_fresh(CreateAgentFilesystem { + agent_id: id, + initial_files: vec![ + initial_file( + content_hash, + "/first/immutable", + AgentFilePermissions::ReadOnly, + content.len() as u64, + ), + initial_file( + content_hash, + "/second/immutable", + AgentFilePermissions::ReadOnly, + content.len() as u64, + ), + initial_file( + content_hash, + "/writable", + AgentFilePermissions::ReadWrite, + content.len() as u64, + ), + ], + file_loader, + resource_limits: None, + limit_exceeded: None, + }) + .await + .unwrap(); + + let first = filesystem.path().join("first/immutable"); + let second = filesystem.path().join("second/immutable"); + let writable = filesystem.path().join("writable"); + assert_eq!(std::fs::read(&first).unwrap(), content); + assert_eq!(std::fs::read(&second).unwrap(), content); + assert_eq!(std::fs::read(&writable).unwrap(), content); + assert_ne!( + first.metadata().unwrap().ino(), + second.metadata().unwrap().ino() + ); + assert_ne!( + first.metadata().unwrap().ino(), + writable.metadata().unwrap().ino() + ); + assert!(filesystem.runtime().is_read_only(&first)); + assert!(filesystem.runtime().is_read_only(&second)); + assert!(!filesystem.runtime().is_read_only(&writable)); + assert!( + filesystem + .runtime() + .is_read_only(&filesystem.path().join("first/../first/immutable")) + ); + std::os::unix::fs::symlink(&first, filesystem.path().join("immutable-link")).unwrap(); + assert!( + filesystem + .runtime() + .is_read_only(&filesystem.path().join("immutable-link")) + ); + assert!( + !filesystem + .runtime() + .is_read_only_path(&filesystem.path().join("immutable-link"), false,) + ); + tokio::fs::write(&writable, b"changed").await.unwrap(); + + let path = filesystem.path().to_path_buf(); + filesystem.close_and_delete().await.unwrap(); + assert!(!path.exists()); +} + +#[test] +async fn failed_initial_file_update_preserves_current_files() { + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + deterministic_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + let id = agent_id(); + let content = b"initial content"; + let (file_loader, content_hash) = + file_loader_with_content(id.environment_id, None, content).await; + let filesystem = filesystems + .create_fresh(CreateAgentFilesystem { + agent_id: id.clone(), + initial_files: vec![initial_file( + content_hash, + "/current", + AgentFilePermissions::ReadOnly, + content.len() as u64, + )], + file_loader: Arc::clone(&file_loader), + resource_limits: None, + limit_exceeded: None, + }) + .await + .unwrap(); + + let result = filesystem + .runtime() + .update_initial_files( + &file_loader, + id.environment_id, + &[ + initial_file( + content_hash, + "/new", + AgentFilePermissions::ReadOnly, + content.len() as u64, + ), + initial_file( + content_hash, + "/invalid", + AgentFilePermissions::ReadOnly, + content.len() as u64 + 1, + ), + ], + ) + .await; + + assert!(result.is_err()); + let current = filesystem.path().join("current"); + assert_eq!(std::fs::read(¤t).unwrap(), content); + assert!(filesystem.runtime().is_read_only(¤t)); + assert!(!filesystem.path().join("new").exists()); + assert!(!filesystem.path().join("invalid").exists()); + filesystem.close_and_delete().await.unwrap(); +} + +#[test] +async fn initial_file_update_commits_staged_files_and_policy_together() { + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + deterministic_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + let id = agent_id(); + let content = b"initial content"; + let (file_loader, content_hash) = + file_loader_with_content(id.environment_id, None, content).await; + let filesystem = filesystems + .create_fresh(CreateAgentFilesystem { + agent_id: id.clone(), + initial_files: vec![initial_file( + content_hash, + "/old", + AgentFilePermissions::ReadOnly, + content.len() as u64, + )], + file_loader: Arc::clone(&file_loader), + resource_limits: None, + limit_exceeded: None, + }) + .await + .unwrap(); + + filesystem + .runtime() + .update_initial_files( + &file_loader, + id.environment_id, + &[ + initial_file( + content_hash, + "/new", + AgentFilePermissions::ReadOnly, + content.len() as u64, + ), + initial_file( + content_hash, + "/writable", + AgentFilePermissions::ReadWrite, + content.len() as u64, + ), + ], + ) + .await + .unwrap(); + + let new = filesystem.path().join("new"); + let writable = filesystem.path().join("writable"); + assert!(!filesystem.path().join("old").exists()); + assert_eq!(std::fs::read(&new).unwrap(), content); + assert_eq!(std::fs::read(&writable).unwrap(), content); + assert!(filesystem.runtime().is_read_only(&new)); + assert!(!filesystem.runtime().is_read_only(&writable)); + filesystem.close_and_delete().await.unwrap(); +} + +#[test] +async fn initial_file_updates_are_exclusive_with_filesystem_effects() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let effect = runtime.begin_effect().await.unwrap(); + let update_runtime = runtime.clone(); + let update = tokio::spawn(async move { update_runtime.begin_update_effect().await.unwrap() }); + tokio::task::yield_now().await; + assert!(!update.is_finished()); + + drop(effect); + let update = update.await.unwrap(); + let effect_runtime = runtime.clone(); + let next_effect = tokio::spawn(async move { effect_runtime.begin_effect().await.unwrap() }); + tokio::task::yield_now().await; + assert!(!next_effect.is_finished()); + + drop(update); + drop(next_effect.await.unwrap()); +} + +#[test] +fn dropped_initial_file_transaction_restores_backups() { + let root = tempfile::tempdir().unwrap(); + let live = root.path().join("live"); + let staged = root.path().join("staged"); + let backup = root.path().join("backups"); + std::fs::write(&live, b"old").unwrap(); + std::fs::write(&staged, b"new").unwrap(); + std::fs::create_dir(&backup).unwrap(); + + { + let mut transaction = InitialFileUpdateTransaction::new(backup); + transaction.back_up(&live).unwrap(); + transaction.install(&staged, &live).unwrap(); + } + + assert_eq!(std::fs::read(&live).unwrap(), b"old"); +} + +#[test] +async fn initial_file_update_rejects_guest_file_collision() { + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + deterministic_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + let id = agent_id(); + let content = b"initial content"; + let (file_loader, content_hash) = + file_loader_with_content(id.environment_id, None, content).await; + let filesystem = filesystems + .create_fresh(CreateAgentFilesystem { + agent_id: id.clone(), + initial_files: Vec::new(), + file_loader: Arc::clone(&file_loader), + resource_limits: None, + limit_exceeded: None, + }) + .await + .unwrap(); + let collision = filesystem.path().join("collision"); + std::fs::write(&collision, b"guest data").unwrap(); + + let result = filesystem + .runtime() + .update_initial_files( + &file_loader, + id.environment_id, + &[initial_file( + content_hash, + "/collision", + AgentFilePermissions::ReadOnly, + content.len() as u64, + )], + ) + .await; + + assert!(result.is_err()); + assert_eq!(std::fs::read(collision).unwrap(), b"guest data"); + filesystem.close_and_delete().await.unwrap(); +} + +#[test] +async fn initial_file_update_preserves_guest_file_for_read_write_target() { + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + deterministic_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + let id = agent_id(); + let content = b"initial content"; + let (file_loader, content_hash) = + file_loader_with_content(id.environment_id, None, content).await; + let filesystem = filesystems + .create_fresh(CreateAgentFilesystem { + agent_id: id.clone(), + initial_files: Vec::new(), + file_loader: Arc::clone(&file_loader), + resource_limits: None, + limit_exceeded: None, + }) + .await + .unwrap(); + let collision = filesystem.path().join("collision"); + std::fs::write(&collision, b"guest data").unwrap(); + + let update = filesystem + .runtime() + .update_initial_files( + &file_loader, + id.environment_id, + &[initial_file( + content_hash, + "/collision", + AgentFilePermissions::ReadWrite, + content.len() as u64, + )], + ) + .await + .unwrap(); + + assert_eq!(std::fs::read(collision).unwrap(), b"guest data"); + drop(update); + filesystem.close_and_delete().await.unwrap(); +} + +#[test] +async fn deterministic_creation_removes_existing_garbage() { + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + deterministic_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + let id = agent_id(); + + let filesystem = filesystems.create_owned_empty(&id).await.unwrap(); + assert_eq!(filesystem.usage().await.unwrap(), None); + let path = filesystem.path().to_path_buf(); + tokio::fs::write(path.join("garbage"), b"old") + .await + .unwrap(); + drop(filesystem); + tokio::fs::create_dir_all(&path).await.unwrap(); + tokio::fs::write(path.join("garbage"), b"old") + .await + .unwrap(); + + let filesystem = filesystems.create_owned_empty(&id).await.unwrap(); + assert!(!filesystem.path().join("garbage").exists()); + filesystem.close_and_delete().await.unwrap(); + assert!(!path.exists()); +} + +#[test] +async fn seal_rejects_new_effects_without_waiting_for_existing_effects() { + let filesystems = AgentFilesystems::new(&FilesystemStorageConfig::default()).unwrap(); + let filesystem = filesystems.create_owned_empty(&agent_id()).await.unwrap(); + let runtime = filesystem.runtime(); + let effect = runtime.begin_effect().await.unwrap(); + + filesystem.seal(); + assert!(runtime.begin_effect().await.is_err()); + assert!(filesystem.path().exists()); + drop(effect); + filesystem.close_and_delete().await.unwrap(); +} + +#[test] +async fn conditional_seal_is_atomic_with_effect_admission() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let effect = runtime.begin_effect().await.unwrap(); + + assert!(!runtime.seal_if_no_active_effects()); + drop(effect); + assert!(runtime.seal_if_no_active_effects()); + assert!(runtime.begin_effect().await.is_err()); +} + +#[test] +async fn conditional_seal_races_effect_admission_atomically() { + for _ in 0..100 { + let runtime = AgentFilesystemRuntime::new_for_test(); + let barrier = Arc::new(tokio::sync::Barrier::new(3)); + let effect_runtime = runtime.clone(); + let effect_barrier = Arc::clone(&barrier); + let seal_runtime = runtime.clone(); + let seal_barrier = Arc::clone(&barrier); + + let effect = async move { + effect_barrier.wait().await; + effect_runtime.begin_effect().await + }; + let seal = async move { + seal_barrier.wait().await; + seal_runtime.seal_if_no_active_effects() + }; + let release = barrier.wait(); + let (effect, sealed, _) = tokio::join!(effect, seal, release); + + assert_ne!(effect.is_ok(), sealed); + } +} + +#[test] +async fn seal_rejects_admitted_effects_waiting_for_operation_lock() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let update = runtime.begin_update_effect().await.unwrap(); + let admitted = runtime.admit_effect().unwrap(); + let waiting = tokio::spawn(async move { admitted.begin().await }); + tokio::task::yield_now().await; + assert!(!waiting.is_finished()); + + runtime.seal(); + drop(update); + assert!(waiting.await.unwrap().is_err()); +} + +#[test] +async fn close_waits_for_an_existing_effect_before_deleting() { + let filesystems = AgentFilesystems::new(&FilesystemStorageConfig::default()).unwrap(); + let filesystem = filesystems.create_owned_empty(&agent_id()).await.unwrap(); + let path = filesystem.path().to_path_buf(); + let effect = filesystem.runtime().begin_effect().await.unwrap(); + + let close = tokio::spawn(filesystem.close_and_delete()); + tokio::task::yield_now().await; + assert!(!close.is_finished()); + assert!(path.exists()); + drop(effect); + close.await.unwrap().unwrap(); + assert!(!path.exists()); +} + +#[test] +async fn cancelled_close_keeps_async_deletion_owned() { + let root = tempfile::tempdir().unwrap(); + let runtime_backend: Arc = Arc::new( + unmanaged::UnmanagedAgentFilesystem::new(root.path().to_path_buf()), + ); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let deleted = Arc::new(AtomicBool::new(false)); + let used_fallback = Arc::new(AtomicBool::new(false)); + let lifecycle = Arc::new(Mutex::new(())) + .try_lock_owned() + .expect("new test lifecycle lock must be available"); + let provisioned = backend::ProvisionedAgentFilesystem::new( + runtime_backend, + Box::new(PausedCleanup { + started: Arc::clone(&started), + release: Arc::clone(&release), + deleted: Arc::clone(&deleted), + used_fallback: Arc::clone(&used_fallback), + }), + lifecycle, + ); + let filesystem = AgentFilesystem::new(provisioned, FilesystemPressureConfig::default()); + let close = tokio::spawn(filesystem.close_and_delete()); + tokio::time::timeout(std::time::Duration::from_secs(1), started.notified()) + .await + .unwrap(); + + close.abort(); + assert!(matches!(close.await, Err(error) if error.is_cancelled())); + assert!(!deleted.load(Ordering::Acquire)); + assert!(!used_fallback.load(Ordering::Acquire)); + + release.notify_one(); + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while !deleted.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!used_fallback.load(Ordering::Acquire)); +} + +#[test] +async fn reconstruction_settlement_waits_for_existing_effects() { + let filesystems = AgentFilesystems::new(&FilesystemStorageConfig::default()).unwrap(); + let filesystem = filesystems.create_owned_empty(&agent_id()).await.unwrap(); + let effect = filesystem.runtime().begin_effect().await.unwrap(); + { + let settle = filesystem.settle_reconstruction(); + tokio::pin!(settle); + + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), &mut settle) + .await + .is_err() + ); + drop(effect); + settle.await.unwrap(); + } + filesystem.close_and_delete().await.unwrap(); +} + +#[test] +async fn dropped_owner_defers_cleanup_and_retains_lifecycle_until_effects_finish() { + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + deterministic_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + let id = agent_id(); + let filesystem = filesystems.create_owned_empty(&id).await.unwrap(); + let path = filesystem.path().to_path_buf(); + let effect = filesystem.runtime().begin_effect().await.unwrap(); + drop(filesystem); + + let replacement = tokio::spawn({ + let filesystems = filesystems.clone(); + let id = id.clone(); + async move { filesystems.create_owned_empty(&id).await } + }); + tokio::task::yield_now().await; + assert!(!replacement.is_finished()); + assert!(path.exists()); + + drop(effect); + let replacement = tokio::time::timeout(std::time::Duration::from_secs(5), replacement) + .await + .unwrap() + .unwrap() + .unwrap(); + replacement.close_and_delete().await.unwrap(); +} + +#[test] +async fn deterministic_creation_is_exclusive_for_the_full_owner_lifetime() { + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + deterministic_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + let id = agent_id(); + let first = filesystems.create_owned_empty(&id).await.unwrap(); + tokio::fs::write(first.path().join("owned"), b"first") + .await + .unwrap(); + + let second = tokio::spawn({ + let filesystems = filesystems.clone(); + let id = id.clone(); + async move { filesystems.create_owned_empty(&id).await } + }); + tokio::task::yield_now().await; + assert!(!second.is_finished()); + assert!(first.path().join("owned").exists()); + + first.close_and_delete().await.unwrap(); + let second = second.await.unwrap().unwrap(); + assert!(!second.path().join("owned").exists()); + second.close_and_delete().await.unwrap(); +} + +#[test] +async fn cancelled_creation_does_not_remain_queued_for_lifecycle() { + let root = tempfile::tempdir().unwrap(); + let settings = FilesystemStorageConfig { + deterministic_root_dir: Some(root.path().to_path_buf()), + ..FilesystemStorageConfig::default() + }; + let filesystems = AgentFilesystems::new(&settings).unwrap(); + let id = agent_id(); + let first = filesystems.create_owned_empty(&id).await.unwrap(); + + let cancelled = tokio::spawn({ + let filesystems = filesystems.clone(); + let id = id.clone(); + async move { filesystems.create_owned_empty(&id).await } + }); + tokio::task::yield_now().await; + cancelled.abort(); + assert!(matches!(cancelled.await, Err(error) if error.is_cancelled())); + tokio::task::yield_now().await; + + let path = first.path().to_path_buf(); + let lifecycle_owners = LIFECYCLE_LOCKS + .get() + .unwrap() + .lock() + .unwrap() + .get(&path) + .unwrap() + .strong_count(); + assert_eq!(lifecycle_owners, 1); + + first.close_and_delete().await.unwrap(); + let replacement = tokio::time::timeout( + std::time::Duration::from_secs(5), + filesystems.create_owned_empty(&id), + ) + .await + .unwrap() + .unwrap(); + assert!( + std::fs::read_dir(replacement.path()) + .unwrap() + .next() + .is_none() + ); + replacement.close_and_delete().await.unwrap(); +} + +#[test] +async fn positioned_effect_does_not_wait_for_active_append() { + let runtime = AgentFilesystemRuntime::new_for_test(); + let append = runtime.begin_append_effect().await.unwrap(); + + let positioned = runtime.begin_effect().await.unwrap(); + + drop(positioned); + drop(append); +} diff --git a/golem-worker-executor/src/services/agent_filesystem/unmanaged.rs b/golem-worker-executor/src/services/agent_filesystem/unmanaged.rs new file mode 100644 index 0000000000..7ada3963f2 --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/unmanaged.rs @@ -0,0 +1,256 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::backend::{ + AgentFilesystemBackend, AgentFilesystemCleanup, FilesystemBackendProvisioner, + InitialFileMaterialization, ProvisionedAgentFilesystem, agent_filesystem_owner_path, +}; +use super::quota::observe_path_capacity; +use super::{ + FilesystemCapacity, FilesystemStorageError, OwnedAgentId, OwnedMutexGuard, Path, PathBuf, + RetryConfig, acquire_lifecycle_lock, create_materialization_parent, remove_and_verify, + remove_and_verify_blocking, rollback_creation, set_initial_file_permissions, + verify_fresh_directory, +}; +use async_trait::async_trait; +use std::sync::Arc; +use tokio::sync::Mutex; + +#[derive(Clone)] +pub(super) struct UnmanagedBackend { + deterministic_root: Option, + cleanup_retry: RetryConfig, +} + +impl UnmanagedBackend { + pub fn new(deterministic_root: Option, cleanup_retry: RetryConfig) -> Self { + Self { + deterministic_root, + cleanup_retry, + } + } + + async fn create_temporary(&self) -> Result { + let directory = tempfile::Builder::new() + .prefix("golem") + .tempdir() + .map_err(|error| { + FilesystemStorageError::io("create temporary directory", Path::new(""), error) + })?; + let lifecycle = Arc::new(Mutex::new(())) + .try_lock_owned() + .expect("new temporary filesystem lifecycle lock must be available"); + let root = directory.keep(); + let backend: Arc = + Arc::new(UnmanagedAgentFilesystem { root: root.clone() }); + let created = ProvisionedAgentFilesystem::new( + backend, + Box::new(UnmanagedCleanup { + root, + cleanup_retry: self.cleanup_retry.clone(), + }), + lifecycle, + ); + if let Err(error) = verify_fresh_directory(created.backend().root()).await { + return Err(created.rollback(error).await); + } + Ok(created) + } + + async fn create_deterministic( + &self, + root: PathBuf, + lifecycle: OwnedMutexGuard<()>, + ) -> Result { + remove_and_verify(&root, "remove stale runtime directory", &self.cleanup_retry).await?; + let parent = root + .parent() + .expect("deterministic agent filesystem path must have a parent"); + if let Err(error) = tokio::fs::create_dir_all(parent).await { + return Err(rollback_creation( + &root, + FilesystemStorageError::io("create runtime directory parent", parent, error), + &self.cleanup_retry, + ) + .await); + } + if let Err(error) = tokio::fs::create_dir(&root).await { + return Err(rollback_creation( + &root, + FilesystemStorageError::io("create fresh runtime directory", &root, error), + &self.cleanup_retry, + ) + .await); + } + + let backend: Arc = + Arc::new(UnmanagedAgentFilesystem { root: root.clone() }); + let created = ProvisionedAgentFilesystem::new( + backend, + Box::new(UnmanagedCleanup { + root, + cleanup_retry: self.cleanup_retry.clone(), + }), + lifecycle, + ); + if let Err(error) = verify_fresh_directory(created.backend().root()).await { + return Err(created.rollback(error).await); + } + Ok(created) + } +} + +#[async_trait] +impl FilesystemBackendProvisioner for UnmanagedBackend { + fn initial_file_cache_root(&self) -> Option<&Path> { + None + } + + async fn provision_for( + &self, + agent_id: &OwnedAgentId, + ) -> Result { + let Some(storage_root) = &self.deterministic_root else { + return self.create_temporary().await; + }; + let root = storage_root.join(agent_filesystem_owner_path(agent_id)); + let lifecycle = acquire_lifecycle_lock(&root).await; + let backend = self.clone(); + let error_path = root.clone(); + tokio::spawn(async move { backend.create_deterministic(root, lifecycle).await }) + .await + .map_err(|error| { + FilesystemStorageError::io( + "provision unmanaged agent filesystem", + &error_path, + std::io::Error::other(error), + ) + })? + } + + async fn observe_capacity(&self) -> Result { + Err(FilesystemStorageError::verification( + "observe capacity for unmanaged filesystem storage", + Path::new(""), + )) + } + + #[cfg(test)] + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +pub(super) struct UnmanagedAgentFilesystem { + root: PathBuf, +} + +impl UnmanagedAgentFilesystem { + #[cfg(test)] + pub(super) fn new(root: PathBuf) -> Self { + Self { root } + } +} + +#[async_trait] +impl AgentFilesystemBackend for UnmanagedAgentFilesystem { + fn root(&self) -> &Path { + &self.root + } + + fn create_staging_dir(&self) -> std::io::Result { + tempfile::Builder::new() + .prefix(".golem-initial-files-update-") + .tempdir_in(&self.root) + } + + async fn materialize_initial_file( + &self, + materialization: InitialFileMaterialization, + ) -> Result<(), FilesystemStorageError> { + let InitialFileMaterialization { + materialization_root, + source, + target, + read_only, + effect, + staging, + } = materialization; + let operation_target = target.clone(); + tokio::task::spawn_blocking(move || { + let _effect = effect; + let _staging = staging; + materialize_unmanaged( + &materialization_root, + source.path(), + &operation_target, + read_only, + ) + }) + .await + .map_err(|error| { + FilesystemStorageError::io( + "materialize unmanaged initial file", + &target, + std::io::Error::other(error), + ) + })? + .map_err(|error| { + FilesystemStorageError::io("materialize unmanaged initial file", &target, error) + }) + } + + async fn observe_capacity(&self) -> Result { + observe_path_capacity(&self.root).await + } + + #[cfg(test)] + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +struct UnmanagedCleanup { + root: PathBuf, + cleanup_retry: RetryConfig, +} + +#[async_trait] +impl AgentFilesystemCleanup for UnmanagedCleanup { + async fn delete(&mut self) -> Result<(), FilesystemStorageError> { + remove_and_verify(&self.root, "delete runtime directory", &self.cleanup_retry).await + } + + fn delete_blocking(&mut self) -> Result<(), FilesystemStorageError> { + remove_and_verify_blocking(&self.root) + } +} + +fn materialize_unmanaged( + root: &Path, + source: &Path, + target: &Path, + read_only: bool, +) -> std::io::Result<()> { + let parent = create_materialization_parent(root, target)?; + let mut temporary = tempfile::NamedTempFile::new_in(parent)?; + let mut source = std::fs::File::open(source)?; + std::io::copy(&mut source, &mut temporary)?; + temporary.as_file().sync_all()?; + set_initial_file_permissions(temporary.as_file(), read_only)?; + temporary + .persist_noclobber(target) + .map_err(|error| error.error)?; + Ok(()) +} diff --git a/golem-worker-executor/src/services/agent_filesystem/xfs.rs b/golem-worker-executor/src/services/agent_filesystem/xfs.rs new file mode 100644 index 0000000000..8039626c31 --- /dev/null +++ b/golem-worker-executor/src/services/agent_filesystem/xfs.rs @@ -0,0 +1,1613 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{ + AgentFilesystemStorageLimit, AgentFilesystemUpdateEffectLease, AgentFilesystemUsage, + FilesystemCapacity, FilesystemObjectLimitPolicyConfig, FilesystemStorageError, OwnedAgentId, + OwnedMutexGuard, ResolvedAgentFilesystemLimits, RetryState, acquire_lifecycle_lock, + backend::{ + AgentFilesystemBackend, AgentFilesystemCleanup, AgentFilesystemQuota, + FilesystemBackendProvisioner, InitialFileMaterialization, InstalledAgentFilesystemLimit, + ProvisionedAgentFilesystem, agent_filesystem_owner_path, + }, + create_materialization_parent, + quota::FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION, + quota::capacity_from_values, + quota::validate_observed_limits, + remove_and_verify, remove_and_verify_blocking, rollback_creation, set_initial_file_permissions, + verify_fresh_open_directory, +}; +use async_trait::async_trait; +use golem_common::model::RetryConfig; +use rustix::fs::{ + FlockOperation, Mode, OFlags, StatVfsMountFlags, flock, fstatfs, fstatvfs, ioctl_ficlone, + mkdirat, openat, +}; +use rustix::ioctl::{Getter, Setter, ioctl}; +use std::collections::HashMap; +use std::fs::File; +use std::num::NonZeroU32; +use std::os::fd::AsRawFd; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +const XFS_SUPER_MAGIC: u64 = 0x5846_5342; +const XFS_BASIC_BLOCK_BYTES: u64 = 512; +const XQM_PRJQUOTA: u32 = 2; +const Q_XGETQUOTA: u32 = (b'X' as u32) << 8 | 3; +const Q_XSETQLIM: u32 = (b'X' as u32) << 8 | 4; +const Q_XGETQSTATV: u32 = (b'X' as u32) << 8 | 8; +const FS_DQUOT_VERSION: i8 = 1; +const FS_QSTATV_VERSION1: i8 = 1; +const FS_PROJ_QUOTA: i8 = 1 << 1; +const FS_QUOTA_PDQ_ACCT: u16 = 1 << 4; +const FS_QUOTA_PDQ_ENFD: u16 = 1 << 5; +const FS_DQ_ISOFT: u16 = 1 << 0; +const FS_DQ_IHARD: u16 = 1 << 1; +const FS_DQ_BSOFT: u16 = 1 << 2; +const FS_DQ_BHARD: u16 = 1 << 3; +const FS_DQ_RTBSOFT: u16 = 1 << 4; +const FS_DQ_RTBHARD: u16 = 1 << 5; +const PROJECT_LIMIT_FIELDS: u16 = + FS_DQ_ISOFT | FS_DQ_IHARD | FS_DQ_BSOFT | FS_DQ_BHARD | FS_DQ_RTBSOFT | FS_DQ_RTBHARD; +const PROJECT_DATA_LIMIT_FIELDS: u16 = FS_DQ_ISOFT | FS_DQ_IHARD | FS_DQ_BSOFT | FS_DQ_BHARD; + +#[derive(Default)] +struct ProjectAllocator { + next: u32, + active: HashMap, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct FsDiskQuota { + d_version: i8, + d_flags: i8, + d_fieldmask: u16, + d_id: u32, + d_blk_hardlimit: u64, + d_blk_softlimit: u64, + d_ino_hardlimit: u64, + d_ino_softlimit: u64, + d_bcount: u64, + d_icount: u64, + d_itimer: i32, + d_btimer: i32, + d_iwarns: u16, + d_bwarns: u16, + d_itimer_hi: i8, + d_btimer_hi: i8, + d_rtbtimer_hi: i8, + d_padding2: i8, + d_rtb_hardlimit: u64, + d_rtb_softlimit: u64, + d_rtbcount: u64, + d_rtbtimer: i32, + d_rtbwarns: u16, + d_padding3: i16, + d_padding4: [i8; 8], +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct FsQuotaFileStatV { + qfs_ino: u64, + qfs_nblks: u64, + qfs_nextents: u32, + qfs_pad: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct FsQuotaStatV { + qs_version: i8, + qs_pad1: u8, + qs_flags: u16, + qs_incoredqs: u32, + qs_uquota: FsQuotaFileStatV, + qs_gquota: FsQuotaFileStatV, + qs_pquota: FsQuotaFileStatV, + qs_btimelimit: i32, + qs_itimelimit: i32, + qs_rtbtimelimit: i32, + qs_bwarnlimit: u16, + qs_iwarnlimit: u16, + qs_rtbwarnlimit: u16, + qs_pad3: u16, + qs_pad4: u32, + qs_pad2: [u64; 7], +} + +#[derive(Clone)] +pub(super) struct XfsBackend { + root: PathBuf, + root_fd: Arc, + allocator: Arc>, + filesystem_block_bytes: u64, + cleanup_retry: RetryConfig, + filesystem_object_limit_policy: FilesystemObjectLimitPolicyConfig, +} + +impl XfsBackend { + pub(super) fn new( + root: &Path, + cleanup_retry: &RetryConfig, + filesystem_object_limit_policy: &FilesystemObjectLimitPolicyConfig, + ) -> Result { + let root_fd = File::open(root) + .map_err(|error| FilesystemStorageError::io("open managed XFS root", root, error))?; + flock(&root_fd, FlockOperation::NonBlockingLockExclusive).map_err(|error| { + FilesystemStorageError::io( + "acquire exclusive ownership of managed XFS root", + root, + errno_to_io(error), + ) + })?; + let filesystem = fstatfs(&root_fd).map_err(|error| { + FilesystemStorageError::io("inspect managed XFS root", root, errno_to_io(error)) + })?; + if filesystem.f_type as u64 != XFS_SUPER_MAGIC { + return Err(FilesystemStorageError::verification( + "validate managed XFS root filesystem type", + root, + )); + } + let filesystem_block_bytes = u64::try_from(filesystem.f_bsize).map_err(|_| { + FilesystemStorageError::verification("validate managed XFS filesystem block size", root) + })?; + if filesystem_block_bytes == 0 + || !filesystem_block_bytes.is_multiple_of(XFS_BASIC_BLOCK_BYTES) + { + return Err(FilesystemStorageError::verification( + "validate managed XFS filesystem block size", + root, + )); + } + + let stable_root = PathBuf::from(format!("/proc/self/fd/{}", root_fd.as_raw_fd())); + std::fs::metadata(&stable_root).map_err(|error| { + FilesystemStorageError::io( + "open managed XFS root through its stable descriptor", + root, + error, + ) + })?; + + let backend = Self { + root: stable_root, + root_fd: Arc::new(root_fd), + allocator: Arc::new(Mutex::new(ProjectAllocator { + next: 1, + active: HashMap::new(), + })), + filesystem_block_bytes, + cleanup_retry: cleanup_retry.clone(), + filesystem_object_limit_policy: filesystem_object_limit_policy.clone(), + }; + backend.clear_root_project_assignment()?; + backend.validate_project_quota_state()?; + backend.validate_project_assignment(cleanup_retry)?; + + Ok(backend) + } + + pub(super) fn root(&self) -> &Path { + &self.root + } + + #[allow( + dead_code, + reason = "authoritative capacity observation is part of the backend interface" + )] + pub(super) fn observe_capacity(&self) -> std::io::Result { + let capacity = fstatvfs(&self.root_fd).map_err(errno_to_io)?; + if capacity.f_flag.contains(StatVfsMountFlags::RDONLY) { + return Err(std::io::Error::new( + std::io::ErrorKind::ReadOnlyFilesystem, + "managed XFS mount is read-only", + )); + } + capacity_from_values( + capacity.f_blocks, + capacity.f_bavail, + capacity.f_frsize, + capacity.f_files, + capacity.f_ffree, + ) + } + + pub(super) fn project_id(&self, file: &File) -> std::io::Result> { + let attributes = get_fsxattr(file)?; + Ok(NonZeroU32::new(attributes.fsx_projid)) + } + + pub(super) fn open_agent_parent( + &self, + environment: &str, + component: &str, + ) -> std::io::Result { + let environment = open_or_create_directory(&self.root_fd, environment)?; + open_or_create_directory(&environment, component) + } + + pub(super) fn open_entry(&self, parent: &File, name: &str) -> std::io::Result { + let entry = openat( + parent, + name, + OFlags::RDONLY | OFlags::NONBLOCK | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(errno_to_io)?; + Ok(File::from(entry)) + } + + pub(super) fn open_directory(&self, parent: &File, name: &str) -> std::io::Result { + let directory = openat( + parent, + name, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(errno_to_io)?; + Ok(File::from(directory)) + } + + pub(super) fn reserved_project(&self, owner: &Path) -> Option { + self.allocator + .lock() + .expect("XFS project allocator lock poisoned") + .active + .iter() + .find_map(|(project_id, active_owner)| (active_owner == owner).then_some(*project_id)) + } + + pub(super) fn reserve_existing_project( + &self, + project_id: NonZeroU32, + owner: &Path, + ) -> std::io::Result<()> { + let mut allocator = self + .allocator + .lock() + .expect("XFS project allocator lock poisoned"); + match allocator.active.get(&project_id) { + Some(active_owner) if active_owner == owner => Ok(()), + Some(active_owner) => Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "XFS project {project_id} is owned by {}", + active_owner.display() + ), + )), + None => { + allocator.active.insert(project_id, owner.to_path_buf()); + Ok(()) + } + } + } + + pub(super) fn reserve_project(&self, owner: &Path) -> std::io::Result { + let mut first_candidate = None; + loop { + let project_id = self.reserve_project_candidate(owner)?; + if first_candidate == Some(project_id) { + self.release_project(project_id); + return Err(std::io::Error::other( + "no reusable XFS project IDs are available", + )); + } + first_candidate.get_or_insert(project_id); + + let prepared = (|| { + let usage = self.project_usage(project_id.get())?; + if usage.allocated_bytes != 0 || usage.filesystem_objects != 0 { + return Ok(false); + } + self.clear_project_limits(project_id)?; + Ok(true) + })(); + + match prepared { + Ok(true) => return Ok(project_id), + Ok(false) => self.release_project(project_id), + Err(error) => { + self.release_project(project_id); + return Err(error); + } + } + } + } + + fn reserve_project_candidate(&self, owner: &Path) -> std::io::Result { + let mut allocator = self + .allocator + .lock() + .expect("XFS project allocator lock poisoned"); + let first = allocator.next.max(1); + let mut candidate = first; + loop { + let project_id = NonZeroU32::new(candidate).expect("candidate must be nonzero"); + if let std::collections::hash_map::Entry::Vacant(entry) = + allocator.active.entry(project_id) + { + entry.insert(owner.to_path_buf()); + allocator.next = candidate.checked_add(1).unwrap_or(1); + return Ok(project_id); + } + + candidate = candidate.checked_add(1).unwrap_or(1); + if candidate == first { + return Err(std::io::Error::other( + "no reusable XFS project IDs are available", + )); + } + } + } + + pub(super) fn release_project(&self, project_id: NonZeroU32) { + self.allocator + .lock() + .expect("XFS project allocator lock poisoned") + .active + .remove(&project_id); + } + + pub(super) fn assign_project( + &self, + file: &File, + project_id: NonZeroU32, + ) -> std::io::Result<()> { + set_project(file, project_id)?; + validate_project_attributes(get_fsxattr(file)?, project_id) + } + + pub(super) fn materialize_initial_file( + &self, + root: &Path, + project_id: NonZeroU32, + source: &Path, + target: &Path, + read_only: bool, + ) -> std::io::Result<()> { + let parent = create_materialization_parent(root, target)?; + { + let temporary = tempfile::NamedTempFile::new_in(parent)?; + let source = File::open(source)?; + if self.project_id(temporary.as_file())? != Some(project_id) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "managed XFS initial-file destination did not inherit its project identity", + )); + } + ioctl_ficlone(temporary.as_file(), &source).map_err(errno_to_io)?; + temporary.as_file().sync_all()?; + set_initial_file_permissions(temporary.as_file(), read_only)?; + temporary + .persist_noclobber(target) + .map_err(|error| error.error)?; + } + rustix::fs::syncfs(&self.root_fd).map_err(errno_to_io) + } + + #[allow( + dead_code, + reason = "authoritative usage observation is part of the backend interface" + )] + pub(super) fn usage(&self, project_id: NonZeroU32) -> std::io::Result { + self.project_usage(project_id.get()) + } + + pub(super) fn usage_and_limits( + &self, + runtime_root: &Path, + project_id: NonZeroU32, + policy_version: u32, + ) -> std::io::Result<(AgentFilesystemUsage, Option)> { + self.observe_project_quota_state()?; + let runtime_root = File::open(runtime_root)?; + validate_project_attributes(get_fsxattr(&runtime_root)?, project_id)?; + let quota = self.project_quota(project_id.get())?; + let usage = usage_from_quota_counts(quota.d_bcount, quota.d_rtbcount, quota.d_icount)?; + let limits = match (quota.d_blk_hardlimit, quota.d_ino_hardlimit) { + (0, 0) => None, + (0, _) | (_, 0) => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "XFS project retained only one member of the quota limit pair", + )); + } + (block_hard_limit, filesystem_objects) => Some(ResolvedAgentFilesystemLimits { + allocated_bytes: block_hard_limit + .checked_mul(XFS_BASIC_BLOCK_BYTES) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "XFS project byte limit exceeds u64", + ) + })?, + filesystem_objects, + filesystem_object_limit_policy_version: policy_version, + }), + }; + Ok((usage, limits)) + } + + pub(super) fn finish_project_cleanup(&self, project_id: NonZeroU32) -> std::io::Result<()> { + let mut usage = self.project_usage(project_id.get())?; + if usage.allocated_bytes != 0 || usage.filesystem_objects != 0 { + rustix::fs::syncfs(&self.root_fd).map_err(errno_to_io)?; + usage = self.project_usage(project_id.get())?; + } + if usage.allocated_bytes != 0 || usage.filesystem_objects != 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + format!( + "XFS project {project_id} still owns {} bytes and {} filesystem objects", + usage.allocated_bytes, usage.filesystem_objects + ), + )); + } + self.clear_project_limits(project_id) + } + + pub(super) fn install_project_limits( + &self, + project_id: NonZeroU32, + limits: ResolvedAgentFilesystemLimits, + ) -> std::io::Result { + if limits.allocated_bytes == 0 + || !limits + .allocated_bytes + .is_multiple_of(self.filesystem_block_bytes) + || limits.filesystem_objects == 0 + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "agent filesystem limits are not exactly representable by managed XFS", + )); + } + let block_hard_limit = limits.allocated_bytes / XFS_BASIC_BLOCK_BYTES; + let mut quota = FsDiskQuota { + d_version: FS_DQUOT_VERSION, + d_flags: FS_PROJ_QUOTA, + d_fieldmask: PROJECT_DATA_LIMIT_FIELDS, + d_id: project_id.get(), + d_blk_hardlimit: block_hard_limit, + d_ino_hardlimit: limits.filesystem_objects, + ..FsDiskQuota::default() + }; + self.set_project_quota_record(project_id, &mut quota)?; + + let mut installed = FsDiskQuota::default(); + self.get_project_quota(project_id.get(), &mut installed)?; + if installed.d_version != FS_DQUOT_VERSION + || installed.d_flags != FS_PROJ_QUOTA + || installed.d_id != project_id.get() + || installed.d_blk_hardlimit != block_hard_limit + || installed.d_blk_softlimit != 0 + || installed.d_ino_hardlimit != limits.filesystem_objects + || installed.d_ino_softlimit != 0 + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("XFS project {project_id} did not retain the complete quota limit pair"), + )); + } + Ok(limits) + } + + fn validate_project_quota_state(&self) -> Result<(), FilesystemStorageError> { + self.observe_project_quota_state().map_err(|error| { + FilesystemStorageError::io( + "validate managed XFS project quota accounting and enforcement", + &self.root, + error, + ) + })?; + + // Querying project zero proves that the executor has quota-query + // privileges without assigning a reusable project identity. + self.project_usage(0).map_err(|error| { + FilesystemStorageError::io( + "validate managed XFS project quota query permissions", + &self.root, + error, + ) + })?; + Ok(()) + } + + fn observe_project_quota_state(&self) -> std::io::Result<()> { + let mut state = FsQuotaStatV { + qs_version: FS_QSTATV_VERSION1, + ..FsQuotaStatV::default() + }; + self.get_quota_state(&mut state)?; + validate_project_quota_state_record(&state) + } + + fn clear_root_project_assignment(&self) -> Result<(), FilesystemStorageError> { + let mut attributes = get_fsxattr(&self.root_fd).map_err(|error| { + FilesystemStorageError::io( + "inspect managed XFS root project attributes", + &self.root, + error, + ) + })?; + attributes.fsx_projid = 0; + attributes.fsx_xflags &= !linux_raw_sys::general::FS_XFLAG_PROJINHERIT; + attributes.fsx_pad = [0; 8]; + set_fsxattr(&self.root_fd, attributes).map_err(|error| { + FilesystemStorageError::io( + "clear managed XFS root project inheritance", + &self.root, + error, + ) + })?; + let assigned = get_fsxattr(&self.root_fd).map_err(|error| { + FilesystemStorageError::io( + "verify managed XFS root project attributes", + &self.root, + error, + ) + })?; + if assigned.fsx_projid != 0 + || assigned.fsx_xflags & linux_raw_sys::general::FS_XFLAG_PROJINHERIT != 0 + { + return Err(FilesystemStorageError::verification( + "verify managed XFS root has neutral project identity", + &self.root, + )); + } + Ok(()) + } + + fn validate_project_assignment( + &self, + cleanup_retry: &RetryConfig, + ) -> Result<(), FilesystemStorageError> { + const PROBE_PROJECT_ID: u32 = 0x8000_0000; + let project_id = NonZeroU32::new(PROBE_PROJECT_ID).unwrap(); + let probe = self.root.join(".golem-xfs-project-probe"); + if probe.exists() { + std::fs::remove_dir_all(&probe).map_err(|error| { + FilesystemStorageError::cleanup_io( + "remove stale managed XFS startup probe", + &probe, + error, + ) + })?; + } + + let usage = self.project_usage(project_id.get()).map_err(|error| { + FilesystemStorageError::io("validate 32-bit XFS project quota query", &self.root, error) + })?; + if usage.allocated_bytes != 0 || usage.filesystem_objects != 0 { + return Err(FilesystemStorageError::verification( + "reserve unused 32-bit XFS startup probe project", + &self.root, + )); + } + self.clear_project_limits(project_id).map_err(|error| { + FilesystemStorageError::io( + "validate XFS project quota update permissions", + &self.root, + error, + ) + })?; + std::fs::create_dir(&probe).map_err(|error| { + FilesystemStorageError::io("create managed XFS startup probe", &probe, error) + })?; + if let Err(error) = self.probe_project_inheritance(&probe, project_id) { + let _ = std::fs::remove_dir_all(&probe); + return Err(FilesystemStorageError::io( + "validate managed XFS project assignment and inheritance", + &probe, + error, + )); + } + std::fs::remove_dir_all(&probe).map_err(|error| { + FilesystemStorageError::cleanup_io("remove managed XFS startup probe", &probe, error) + })?; + self.finish_project_cleanup_with_retry(project_id, cleanup_retry) + .map_err(|error| { + FilesystemStorageError::cleanup_io( + "clear managed XFS startup probe project", + &probe, + error, + ) + }) + } + + fn probe_project_inheritance( + &self, + probe: &Path, + project_id: NonZeroU32, + ) -> std::io::Result<()> { + let probe_directory = File::open(probe)?; + self.assign_project(&probe_directory, project_id)?; + let child = probe.join("child"); + std::fs::create_dir(&child)?; + let child = File::open(&child)?; + let child_id = self.project_id(&child)?; + if child_id != Some(project_id) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "XFS child did not inherit its project identity", + )); + } + let source_path = probe.join("reflink-source"); + let destination_path = probe.join("reflink-destination"); + std::fs::write(&source_path, b"golem-xfs-reflink-probe")?; + let source = File::open(&source_path)?; + let destination = File::create(&destination_path)?; + ioctl_ficlone(&destination, &source).map_err(errno_to_io)?; + if std::fs::read(&destination_path)? != b"golem-xfs-reflink-probe" { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "XFS reflink did not preserve probe contents", + )); + } + Ok(()) + } + + fn project_usage(&self, project_id: u32) -> std::io::Result { + let quota = self.project_quota(project_id)?; + usage_from_quota_counts(quota.d_bcount, quota.d_rtbcount, quota.d_icount) + } + + fn project_quota(&self, project_id: u32) -> std::io::Result { + let mut quota = FsDiskQuota::default(); + if let Err(error) = self.get_project_quota(project_id, &mut quota) { + if matches!(error.raw_os_error(), Some(libc::ENOENT) | Some(libc::ESRCH)) { + quota.d_version = FS_DQUOT_VERSION; + quota.d_flags = FS_PROJ_QUOTA; + quota.d_id = project_id; + return Ok(quota); + } + return Err(error); + } + if quota.d_version != FS_DQUOT_VERSION + || quota.d_flags != FS_PROJ_QUOTA + || quota.d_id != project_id + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "XFS returned an invalid project quota record", + )); + } + Ok(quota) + } + + fn clear_project_limits(&self, project_id: NonZeroU32) -> std::io::Result<()> { + let mut quota = FsDiskQuota { + d_version: FS_DQUOT_VERSION, + d_flags: FS_PROJ_QUOTA, + d_fieldmask: PROJECT_LIMIT_FIELDS, + d_id: project_id.get(), + ..FsDiskQuota::default() + }; + self.set_project_quota_record(project_id, &mut quota)?; + + let mut cleared = FsDiskQuota::default(); + if let Err(error) = self.get_project_quota(project_id.get(), &mut cleared) { + if matches!(error.raw_os_error(), Some(libc::ENOENT) | Some(libc::ESRCH)) { + return Ok(()); + } + return Err(error); + } + if cleared.d_blk_hardlimit != 0 + || cleared.d_blk_softlimit != 0 + || cleared.d_ino_hardlimit != 0 + || cleared.d_ino_softlimit != 0 + || cleared.d_rtb_hardlimit != 0 + || cleared.d_rtb_softlimit != 0 + || cleared.d_itimer != 0 + || cleared.d_btimer != 0 + || cleared.d_rtbtimer != 0 + || cleared.d_iwarns != 0 + || cleared.d_bwarns != 0 + || cleared.d_rtbwarns != 0 + || cleared.d_itimer_hi != 0 + || cleared.d_btimer_hi != 0 + || cleared.d_rtbtimer_hi != 0 + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("XFS project {project_id} retained quota limits or state"), + )); + } + Ok(()) + } + + fn finish_project_cleanup_with_retry( + &self, + project_id: NonZeroU32, + cleanup_retry: &RetryConfig, + ) -> std::io::Result<()> { + let attempts = cleanup_retry.max_attempts.max(1); + let mut delay = cleanup_retry.min_delay; + for attempt in 1..=attempts { + match self.finish_project_cleanup(project_id) { + Ok(()) => return Ok(()), + Err(error) if attempt == attempts => return Err(error), + Err(_) => { + std::thread::sleep(delay); + delay = delay + .mul_f64(cleanup_retry.multiplier) + .min(cleanup_retry.max_delay); + } + } + } + unreachable!("managed XFS cleanup always performs at least one attempt") + } + + fn get_project_quota(&self, project_id: u32, quota: &mut FsDiskQuota) -> std::io::Result<()> { + // SAFETY: Q_XGETQUOTA writes exactly one `fs_disk_quota` record. + unsafe { + self.quotactl_raw( + Q_XGETQUOTA, + project_id, + std::ptr::from_mut(quota).cast::(), + ) + } + } + + fn set_project_quota_record( + &self, + project_id: NonZeroU32, + quota: &mut FsDiskQuota, + ) -> std::io::Result<()> { + // SAFETY: Q_XSETQLIM reads exactly one `fs_disk_quota` record. + unsafe { + self.quotactl_raw( + Q_XSETQLIM, + project_id.get(), + std::ptr::from_mut(quota).cast::(), + ) + } + } + + fn get_quota_state(&self, state: &mut FsQuotaStatV) -> std::io::Result<()> { + // SAFETY: Q_XGETQSTATV writes exactly one `fs_quota_statv` record. + unsafe { + self.quotactl_raw( + Q_XGETQSTATV, + 0, + std::ptr::from_mut(state).cast::(), + ) + } + } + + unsafe fn quotactl_raw( + &self, + command: u32, + id: u32, + data: *mut libc::c_void, + ) -> std::io::Result<()> { + let operation = (command << 8) | (XQM_PRJQUOTA & 0xff); + // SAFETY: The caller guarantees that `data` points to the UAPI + // structure selected by `command` for the duration of the syscall. + let result = unsafe { + libc::syscall( + libc::SYS_quotactl_fd, + self.root_fd.as_raw_fd(), + operation, + id, + data, + ) + }; + if result == -1 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } +} + +impl XfsBackend { + async fn observe_capacity_async( + &self, + error_path: PathBuf, + ) -> Result { + let backend = self.clone(); + tokio::task::spawn_blocking(move || backend.observe_capacity()) + .await + .map_err(|error| { + FilesystemStorageError::io( + "observe managed XFS capacity", + &error_path, + std::io::Error::other(error), + ) + })? + .map_err(|error| { + FilesystemStorageError::io("observe managed XFS capacity", &error_path, error) + }) + } + + async fn provision( + &self, + agent_id: &OwnedAgentId, + lifecycle: OwnedMutexGuard<()>, + ) -> Result { + let environment = agent_id.environment_id.to_string(); + let component = agent_id.agent_id.component_id.to_string(); + let agent = agent_id.agent_id.agent_name_encoded(); + let owner = agent_filesystem_owner_path(agent_id); + let mut lifecycle = Some(lifecycle); + let parent = self + .open_agent_parent(&environment, &component) + .map_err(|error| { + FilesystemStorageError::io( + "open managed runtime directory parent", + &self + .root() + .join(owner.parent().expect("owner must have a parent")), + error, + ) + })?; + let cleanup_path = + PathBuf::from(format!("/proc/self/fd/{}", parent.as_raw_fd())).join(&agent); + + let disk_project = match tokio::fs::symlink_metadata(&cleanup_path).await { + Ok(metadata) if !metadata.file_type().is_symlink() => { + let backend = self.clone(); + let existing_entry = backend.open_entry(&parent, &agent).map_err(|error| { + FilesystemStorageError::cleanup_io( + "open stale managed XFS runtime path", + &cleanup_path, + error, + ) + })?; + tokio::task::spawn_blocking(move || backend.project_id(&existing_entry)) + .await + .map_err(|error| { + FilesystemStorageError::io( + "inspect stale managed XFS project", + &cleanup_path, + std::io::Error::other(error), + ) + })? + .map_err(|error| { + FilesystemStorageError::io( + "inspect stale managed XFS project", + &cleanup_path, + error, + ) + })? + } + Ok(_) => None, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => { + return Err(FilesystemStorageError::cleanup_io( + "inspect stale managed XFS runtime path", + &cleanup_path, + error, + )); + } + }; + let reserved_project = self.reserved_project(&owner); + let stale_project = match (disk_project, reserved_project) { + (Some(disk_project), Some(reserved_project)) if disk_project != reserved_project => { + return Err(FilesystemStorageError::cleanup_verification( + "match stale managed XFS path and reserved project", + &cleanup_path, + )); + } + (disk_project, reserved_project) => disk_project.or(reserved_project), + }; + + let mut stale_cleanup = if let Some(project_id) = stale_project { + let cleanup_parent = parent.try_clone().map_err(|error| { + FilesystemStorageError::cleanup_io( + "retain managed XFS runtime directory parent", + &cleanup_path, + error, + ) + })?; + self.reserve_existing_project(project_id, &owner) + .map_err(|error| { + FilesystemStorageError::cleanup_io( + "reserve stale managed XFS project for cleanup", + &cleanup_path, + error, + ) + })?; + Some(ManagedProjectCleanup::new( + XfsProjectCleanup::new( + self.clone(), + project_id, + cleanup_path.clone(), + self.cleanup_retry.clone(), + ), + cleanup_parent, + lifecycle + .take() + .expect("managed XFS lifecycle owner must exist"), + )) + } else { + None + }; + remove_and_verify( + &cleanup_path, + "remove stale managed XFS runtime directory", + &self.cleanup_retry, + ) + .await?; + if stale_project.is_some() { + stale_cleanup + .as_ref() + .expect("stale project cleanup owner must exist") + .project + .finish() + .await?; + lifecycle = Some( + stale_cleanup + .as_mut() + .expect("stale project cleanup owner must exist") + .disarm(), + ); + } + + tokio::fs::create_dir(&cleanup_path) + .await + .map_err(|error| { + FilesystemStorageError::io( + "create fresh managed runtime directory", + &cleanup_path, + error, + ) + })?; + + let project_id = self.reserve_project(&owner).map_err(|error| { + FilesystemStorageError::io("allocate managed XFS project", &cleanup_path, error) + }); + let project_id = match project_id { + Ok(project_id) => project_id, + Err(error) => { + return Err(rollback_creation(&cleanup_path, error, &self.cleanup_retry).await); + } + }; + let root_fd = match self.open_directory(&parent, &agent) { + Ok(root) => root, + Err(error) => { + self.release_project(project_id); + return Err(rollback_creation( + &cleanup_path, + FilesystemStorageError::io( + "open fresh managed runtime directory", + &cleanup_path, + error, + ), + &self.cleanup_retry, + ) + .await); + } + }; + let root = PathBuf::from(format!("/proc/self/fd/{}", root_fd.as_raw_fd())); + let assignment_result = self.assign_project(&root_fd, project_id).map_err(|error| { + FilesystemStorageError::io("assign managed XFS project", &root, error) + }); + let runtime_backend = Arc::new(XfsAgentFilesystem { + root, + backend: self.clone(), + project_id, + root_fd: Mutex::new(Some(root_fd)), + }); + let backend: Arc = runtime_backend.clone(); + let created = ProvisionedAgentFilesystem::new( + backend, + Box::new(XfsCleanup { + runtime_backend, + project: XfsProjectCleanup::new( + self.clone(), + project_id, + cleanup_path, + self.cleanup_retry.clone(), + ), + _parent: parent, + }), + lifecycle.expect("managed XFS lifecycle owner must exist"), + ); + if let Err(error) = assignment_result { + return Err(created.rollback(error).await); + } + if let Err(error) = verify_fresh_open_directory(created.backend().root()).await { + return Err(created.rollback(error).await); + } + + Ok(created) + } +} + +#[async_trait] +impl FilesystemBackendProvisioner for XfsBackend { + fn initial_file_cache_root(&self) -> Option<&Path> { + Some(self.root()) + } + + async fn provision_for( + &self, + agent_id: &OwnedAgentId, + ) -> Result { + let owner = agent_filesystem_owner_path(agent_id); + let lifecycle = acquire_lifecycle_lock(&self.root().join(owner)).await; + let backend = self.clone(); + let agent_id = agent_id.clone(); + let root = self.root().to_path_buf(); + tokio::spawn(async move { backend.provision(&agent_id, lifecycle).await }) + .await + .map_err(|error| { + FilesystemStorageError::io( + "provision managed XFS agent filesystem", + &root, + std::io::Error::other(error), + ) + })? + } + + async fn observe_capacity(&self) -> Result { + self.observe_capacity_async(self.root().to_path_buf()).await + } + + #[cfg(test)] + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +struct XfsAgentFilesystem { + root: PathBuf, + backend: XfsBackend, + project_id: NonZeroU32, + root_fd: Mutex>, +} + +impl XfsAgentFilesystem { + fn close(&self) { + self.root_fd + .lock() + .expect("managed XFS root descriptor lock poisoned") + .take(); + } +} + +#[async_trait] +impl AgentFilesystemBackend for XfsAgentFilesystem { + fn root(&self) -> &Path { + &self.root + } + + fn create_staging_dir(&self) -> std::io::Result { + let staging = tempfile::Builder::new() + .prefix(".golem-initial-files-update-") + .tempdir_in(&self.root)?; + let directory = File::open(staging.path())?; + self.backend.assign_project(&directory, self.project_id)?; + Ok(staging) + } + + async fn materialize_initial_file( + &self, + materialization: InitialFileMaterialization, + ) -> Result<(), FilesystemStorageError> { + let InitialFileMaterialization { + materialization_root, + source, + target, + read_only, + effect, + staging, + } = materialization; + let backend = self.backend.clone(); + let project_id = self.project_id; + let operation_target = target.clone(); + tokio::task::spawn_blocking(move || { + let _effect = effect; + let _staging = staging; + backend.materialize_initial_file( + &materialization_root, + project_id, + source.path(), + &operation_target, + read_only, + ) + }) + .await + .map_err(|error| { + FilesystemStorageError::io( + "reflink managed XFS initial file", + &target, + std::io::Error::other(error), + ) + })? + .map_err(|error| { + FilesystemStorageError::io("reflink managed XFS initial file", &target, error) + }) + } + + async fn observe_capacity(&self) -> Result { + self.backend.observe_capacity_async(self.root.clone()).await + } + + fn quota(&self) -> Option<&dyn AgentFilesystemQuota> { + Some(self) + } + + #[cfg(test)] + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[async_trait] +impl AgentFilesystemQuota for XfsAgentFilesystem { + async fn usage(&self) -> Result { + let root = self.root.clone(); + let backend = self.backend.clone(); + let project_id = self.project_id; + tokio::task::spawn_blocking(move || backend.usage(project_id)) + .await + .map_err(|error| { + FilesystemStorageError::io( + "observe managed XFS project usage", + &root, + std::io::Error::other(error), + ) + })? + .map_err(|error| { + FilesystemStorageError::io("observe managed XFS project usage", &root, error) + }) + } + + async fn failure_observations( + &self, + installed_limits: Option, + ) -> Result<(AgentFilesystemUsage, Option), FilesystemStorageError> + { + let root = self.root.clone(); + let backend = self.backend.clone(); + let project_id = self.project_id; + let policy_version = installed_limits + .map(|limits| limits.filesystem_object_limit_policy_version) + .unwrap_or(FILESYSTEM_OBJECT_LIMIT_POLICY_VERSION); + let observation_root = root.clone(); + let (usage, observed_limits) = tokio::task::spawn_blocking(move || { + backend.usage_and_limits(&observation_root, project_id, policy_version) + }) + .await + .map_err(|error| { + FilesystemStorageError::io( + "observe managed XFS project quota", + &root, + std::io::Error::other(error), + ) + })? + .map_err(|error| { + FilesystemStorageError::io("observe managed XFS project quota", &root, error) + })?; + validate_observed_limits(&root, installed_limits, observed_limits)?; + Ok((usage, observed_limits)) + } + + async fn install_limit( + &self, + limit: AgentFilesystemStorageLimit, + effect: AgentFilesystemUpdateEffectLease, + ) -> Result { + let limits = self.backend.filesystem_object_limit_policy.resolve(limit)?; + let root = self.root.clone(); + let backend = self.backend.clone(); + let project_id = self.project_id; + let usage = tokio::task::spawn_blocking(move || { + let _effect = effect; + backend.install_project_limits(project_id, limits)?; + backend.usage(project_id) + }) + .await + .map_err(|error| { + FilesystemStorageError::io( + "install managed XFS project limits", + &root, + std::io::Error::other(error), + ) + })? + .map_err(|error| { + FilesystemStorageError::io("install managed XFS project limits", &root, error) + })?; + Ok(InstalledAgentFilesystemLimit { limits, usage }) + } +} + +struct XfsCleanup { + runtime_backend: Arc, + project: XfsProjectCleanup, + _parent: File, +} + +#[async_trait] +impl AgentFilesystemCleanup for XfsCleanup { + async fn delete(&mut self) -> Result<(), FilesystemStorageError> { + self.runtime_backend.close(); + remove_and_verify( + &self.project.path, + "delete managed XFS runtime directory", + &self.project.cleanup_retry, + ) + .await?; + self.project.finish().await + } + + fn delete_blocking(&mut self) -> Result<(), FilesystemStorageError> { + self.runtime_backend.close(); + self.project.remove_and_finish_blocking() + } + + fn requires_background_drop_cleanup(&self) -> bool { + true + } +} + +#[cfg(test)] +pub(super) fn project_id_for_test(backend: &dyn AgentFilesystemBackend) -> NonZeroU32 { + backend + .as_any() + .downcast_ref::() + .expect("created filesystem backend must be XFS") + .project_id +} + +struct ManagedProjectCleanup { + project: XfsProjectCleanup, + parent: Option, + lifecycle: Option>, + armed: bool, +} + +impl ManagedProjectCleanup { + fn new(project: XfsProjectCleanup, parent: File, lifecycle: OwnedMutexGuard<()>) -> Self { + Self { + project, + parent: Some(parent), + lifecycle: Some(lifecycle), + armed: true, + } + } + + fn disarm(&mut self) -> OwnedMutexGuard<()> { + self.armed = false; + self.parent.take(); + self.lifecycle + .take() + .expect("managed XFS stale cleanup lifecycle owner must exist") + } +} + +impl Drop for ManagedProjectCleanup { + fn drop(&mut self) { + if self.armed { + let project = self.project.clone(); + let parent = self.parent.take(); + let lifecycle = self.lifecycle.take(); + std::thread::spawn(move || { + if let Err(error) = project.remove_and_finish_blocking() { + tracing::error!(error = %error, "Failed to clean reserved managed XFS project"); + } + drop((parent, lifecycle)); + }); + } + } +} + +#[derive(Clone)] +struct XfsProjectCleanup { + backend: XfsBackend, + project_id: NonZeroU32, + path: PathBuf, + cleanup_retry: RetryConfig, +} + +impl XfsProjectCleanup { + fn new( + backend: XfsBackend, + project_id: NonZeroU32, + path: PathBuf, + cleanup_retry: RetryConfig, + ) -> Self { + Self { + backend, + project_id, + path, + cleanup_retry, + } + } + + async fn finish(&self) -> Result<(), FilesystemStorageError> { + let mut retry = RetryState::new(&self.cleanup_retry); + loop { + retry.start_attempt(); + let backend = self.backend.clone(); + let project_id = self.project_id; + let attempt = + tokio::task::spawn_blocking(move || backend.finish_project_cleanup(project_id)) + .await + .map_err(|error| { + FilesystemStorageError::cleanup_io( + "verify and clear managed XFS project", + &self.path, + std::io::Error::other(error), + ) + })?; + match attempt { + Ok(()) => { + self.backend.release_project(self.project_id); + return Ok(()); + } + Err(error) => { + if !retry.failed_attempt().await { + return Err(FilesystemStorageError::cleanup_io( + "verify and clear managed XFS project", + &self.path, + error, + )); + } + } + } + } + } + + fn remove_and_finish_blocking(&self) -> Result<(), FilesystemStorageError> { + remove_and_verify_blocking(&self.path)?; + let attempts = self.cleanup_retry.max_attempts.max(1); + let mut delay = self.cleanup_retry.min_delay; + for attempt in 1..=attempts { + match self.backend.finish_project_cleanup(self.project_id) { + Ok(()) => { + self.backend.release_project(self.project_id); + return Ok(()); + } + Err(error) if attempt == attempts => { + return Err(FilesystemStorageError::cleanup_io( + "verify and clear managed XFS project", + &self.path, + error, + )); + } + Err(_) => { + std::thread::sleep(delay); + delay = delay + .mul_f64(self.cleanup_retry.multiplier) + .min(self.cleanup_retry.max_delay); + } + } + } + unreachable!("managed XFS cleanup always performs at least one attempt") + } +} + +fn get_fsxattr(file: &File) -> std::io::Result { + // SAFETY: The generated opcode and `fsxattr` type come from the same Linux + // UAPI version and the kernel initializes the complete output structure. + unsafe { + ioctl( + file, + Getter::< + { linux_raw_sys::ioctl::FS_IOC_FSGETXATTR as rustix::ioctl::Opcode }, + linux_raw_sys::general::fsxattr, + >::new(), + ) + } + .map_err(errno_to_io) +} + +fn validate_project_attributes( + attributes: linux_raw_sys::general::fsxattr, + project_id: NonZeroU32, +) -> std::io::Result<()> { + if attributes.fsx_projid != project_id.get() + || attributes.fsx_xflags & linux_raw_sys::general::FS_XFLAG_PROJINHERIT == 0 + { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "XFS project identity or inheritance did not persist", + )) + } else { + Ok(()) + } +} + +fn validate_project_quota_state_record(state: &FsQuotaStatV) -> std::io::Result<()> { + if state.qs_version != FS_QSTATV_VERSION1 + || state.qs_flags & (FS_QUOTA_PDQ_ACCT | FS_QUOTA_PDQ_ENFD) + != FS_QUOTA_PDQ_ACCT | FS_QUOTA_PDQ_ENFD + { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "managed XFS project quota accounting or enforcement is disabled", + )) + } else { + Ok(()) + } +} + +fn set_project(file: &File, project_id: NonZeroU32) -> std::io::Result<()> { + let mut attributes = get_fsxattr(file)?; + attributes.fsx_projid = project_id.get(); + attributes.fsx_xflags |= linux_raw_sys::general::FS_XFLAG_PROJINHERIT; + attributes.fsx_pad = [0; 8]; + set_fsxattr(file, attributes) +} + +fn set_fsxattr(file: &File, attributes: linux_raw_sys::general::fsxattr) -> std::io::Result<()> { + // SAFETY: The generated opcode and `fsxattr` type come from the same Linux + // UAPI version. The caller preserves existing settable attributes. + unsafe { + ioctl( + file, + Setter::< + { linux_raw_sys::ioctl::FS_IOC_FSSETXATTR as rustix::ioctl::Opcode }, + linux_raw_sys::general::fsxattr, + >::new(attributes), + ) + } + .map_err(errno_to_io) +} + +fn open_or_create_directory(parent: &File, name: &str) -> std::io::Result { + match mkdirat(parent, name, Mode::from_raw_mode(0o700)) { + Ok(()) | Err(rustix::io::Errno::EXIST) => {} + Err(error) => return Err(errno_to_io(error)), + } + let directory = openat( + parent, + name, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(errno_to_io)?; + Ok(File::from(directory)) +} + +fn usage_from_quota_counts( + data_basic_blocks: u64, + realtime_basic_blocks: u64, + filesystem_objects: u64, +) -> std::io::Result { + let basic_blocks = data_basic_blocks + .checked_add(realtime_basic_blocks) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "XFS project data and realtime usage exceeds u64 blocks", + ) + })?; + let allocated_bytes = basic_blocks + .checked_mul(XFS_BASIC_BLOCK_BYTES) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "XFS project usage exceeds u64 bytes", + ) + })?; + Ok(AgentFilesystemUsage { + allocated_bytes, + filesystem_objects, + }) +} + +fn errno_to_io(error: rustix::io::Errno) -> std::io::Error { + std::io::Error::from_raw_os_error(error.raw_os_error()) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_r::test; + + #[test] + fn capacity_uses_fragment_size_and_executor_available_counts() { + let capacity = capacity_from_values(10, 4, 4096, 20, 7).unwrap(); + + assert_eq!(capacity.total_bytes, 40_960); + assert_eq!(capacity.available_bytes, 16_384); + assert_eq!(capacity.total_filesystem_objects, 20); + assert_eq!(capacity.available_filesystem_objects, 7); + } + + #[test] + fn capacity_rejects_byte_overflow() { + assert!(capacity_from_values(u64::MAX, 1, 2, 0, 0).is_err()); + } + + #[test] + fn project_usage_uses_xfs_basic_block_units() { + let usage = usage_from_quota_counts(3, 2, 2).unwrap(); + + assert_eq!(usage.allocated_bytes, 2560); + assert_eq!(usage.filesystem_objects, 2); + } + + #[test] + fn project_usage_rejects_combined_block_overflow() { + assert!(usage_from_quota_counts(u64::MAX, 1, 0).is_err()); + } + + #[test] + fn quota_abi_layout_matches_linux_uapi() { + assert_eq!(std::mem::size_of::(), 112); + assert_eq!(std::mem::align_of::(), 8); + assert_eq!(std::mem::size_of::(), 160); + assert_eq!(std::mem::align_of::(), 8); + } + + #[test] + fn project_quota_state_requires_accounting_and_enforcement() { + let healthy = FsQuotaStatV { + qs_version: FS_QSTATV_VERSION1, + qs_flags: FS_QUOTA_PDQ_ACCT | FS_QUOTA_PDQ_ENFD, + ..FsQuotaStatV::default() + }; + assert!(validate_project_quota_state_record(&healthy).is_ok()); + + for flags in [0, FS_QUOTA_PDQ_ACCT, FS_QUOTA_PDQ_ENFD] { + let unhealthy = FsQuotaStatV { + qs_version: FS_QSTATV_VERSION1, + qs_flags: flags, + ..FsQuotaStatV::default() + }; + assert_eq!( + validate_project_quota_state_record(&unhealthy) + .unwrap_err() + .kind(), + std::io::ErrorKind::InvalidData + ); + } + } + + #[test] + fn project_attributes_require_expected_identity_and_inheritance() { + let project_id = NonZeroU32::new(17).unwrap(); + let healthy = linux_raw_sys::general::fsxattr { + fsx_xflags: linux_raw_sys::general::FS_XFLAG_PROJINHERIT, + fsx_extsize: 0, + fsx_nextents: 0, + fsx_projid: project_id.get(), + fsx_cowextsize: 0, + fsx_pad: [0; 8], + }; + assert!(validate_project_attributes(healthy, project_id).is_ok()); + + let wrong_project = linux_raw_sys::general::fsxattr { + fsx_projid: project_id.get() + 1, + ..healthy + }; + assert_eq!( + validate_project_attributes(wrong_project, project_id) + .unwrap_err() + .kind(), + std::io::ErrorKind::InvalidData + ); + + let no_inheritance = linux_raw_sys::general::fsxattr { + fsx_xflags: 0, + ..healthy + }; + assert_eq!( + validate_project_attributes(no_inheritance, project_id) + .unwrap_err() + .kind(), + std::io::ErrorKind::InvalidData + ); + } +} diff --git a/golem-worker-executor/src/services/agent_memory_meter.rs b/golem-worker-executor/src/services/agent_memory_meter.rs index c1736f4521..2beda45ff8 100644 --- a/golem-worker-executor/src/services/agent_memory_meter.rs +++ b/golem-worker-executor/src/services/agent_memory_meter.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::services::byte_time_accumulator::{ByteTimeAccumulator, ByteTimeSettlement}; use crate::services::resource_limits::AtomicResourceEntry; use golem_common::model::agent::AgentMode; use std::sync::atomic::{AtomicU64, Ordering}; @@ -21,6 +22,10 @@ use std::time::Instant; pub(crate) const BYTE_NANOSECONDS_PER_GB_SECOND: u128 = (1024_u128 * 1024 * 1024) * 1_000_000_000; #[derive(Clone, Debug)] +/// Leaf meter for linear-memory byte-time and memory-limit state. +/// +/// `AgentResourceBilling` owns permit-window lifecycle transitions and invokes this meter +/// under the transition lock shared with filesystem storage accounting. pub struct AgentMemoryMeter { inner: Arc, } @@ -48,9 +53,7 @@ struct State { bytes: u64, active: bool, stopped: bool, - last_sample: Instant, - pending_gb_seconds: i64, - pending_byte_nanoseconds: u128, + usage: ByteTimeAccumulator, } impl AgentMemoryMeter { @@ -71,9 +74,7 @@ impl AgentMemoryMeter { bytes, active, stopped: false, - last_sample: now, - pending_gb_seconds: 0, - pending_byte_nanoseconds: 0, + usage: ByteTimeAccumulator::new(BYTE_NANOSECONDS_PER_GB_SECOND, now), }), }), } @@ -110,19 +111,28 @@ impl AgentMemoryMeter { Arc::ptr_eq(&self.inner, &other.inner) } - pub fn resume(&self, bytes: u64, now: Instant) { + /// Changes only memory metering. Resource-window lifecycle transitions must use + /// `AgentResourceBilling` so filesystem storage changes at the same timestamp. + pub fn resume(&self, bytes: u64, now: Instant) -> bool { self.inner.transition(now, |state| { - if !state.stopped { + if state.stopped { + false + } else { state.bytes = bytes; state.active = true; + true } - }); + }) } + /// Changes only memory metering. Resource-window lifecycle transitions must use + /// `AgentResourceBilling` so filesystem storage changes at the same timestamp. pub fn pause(&self, now: Instant) { self.inner.transition(now, |state| state.active = false); } + /// Changes only memory metering. Resource-window lifecycle transitions must use + /// `AgentResourceBilling` so filesystem storage changes at the same timestamp. pub fn stop(&self, now: Instant) { let settlement = { let mut state = self.inner.state.lock().unwrap(); @@ -135,9 +145,8 @@ impl AgentMemoryMeter { Some(state.take_settlement()) } }; - if let Some((units, remainder)) = settlement { - self.inner.record(units); - self.inner.transfer_remainder(remainder); + if let Some(settlement) = settlement { + self.inner.record_settlement(settlement); } } @@ -145,14 +154,35 @@ impl AgentMemoryMeter { self.inner.transition(now, |state| state.bytes = bytes); } + pub(crate) fn sample(&self, now: Instant) { + self.inner.transition(now, |_| {}); + } + pub fn flush(&self, now: Instant) { - let units = { - let mut state = self.inner.state.lock().unwrap(); - state.accrue(now); - std::mem::take(&mut state.pending_gb_seconds) - }; + let units = self.take_units(now); self.inner.record(units); } + + pub(crate) fn take_units(&self, now: Instant) -> i64 { + let mut state = self.inner.state.lock().unwrap(); + state.accrue(now); + state.usage.take_units() + } + + pub(crate) fn take_settlement(&self) -> ByteTimeSettlement { + self.inner.state.lock().unwrap().take_settlement() + } + + pub(crate) fn take_abort_settlement(&self) -> Option { + let mut state = self.inner.state.lock().unwrap(); + if state.stopped { + None + } else { + state.active = false; + state.stopped = true; + Some(state.take_settlement()) + } + } } impl Inner { @@ -170,53 +200,32 @@ impl Inner { } } - fn transfer_remainder(&self, remainder: u128) { - if remainder != 0 - && let Some(entry) = self.entry.upgrade() - { - entry.record_memory_remainder(self.mode, remainder); + fn record_settlement(&self, settlement: ByteTimeSettlement) { + if let Some(entry) = self.entry.upgrade() { + entry.record_memory_settlement(self.mode, settlement); } } } impl State { fn accrue(&mut self, now: Instant) { - if now <= self.last_sample { - return; - } - - let elapsed = now.saturating_duration_since(self.last_sample).as_nanos(); - self.last_sample = now; - if self.active && !self.stopped { - self.pending_byte_nanoseconds = self - .pending_byte_nanoseconds - .saturating_add((self.bytes as u128).saturating_mul(elapsed)); - } - - let units = self.pending_byte_nanoseconds / BYTE_NANOSECONDS_PER_GB_SECOND; - self.pending_byte_nanoseconds %= BYTE_NANOSECONDS_PER_GB_SECOND; - self.pending_gb_seconds = self - .pending_gb_seconds - .saturating_add(units.min(i64::MAX as u128) as i64); + let bytes = (self.active && !self.stopped).then_some(self.bytes); + self.usage.accrue(now, bytes); } - fn take_settlement(&mut self) -> (i64, u128) { - ( - std::mem::take(&mut self.pending_gb_seconds), - std::mem::take(&mut self.pending_byte_nanoseconds), - ) + fn take_settlement(&mut self) -> ByteTimeSettlement { + self.usage.take_settlement() } } impl Drop for Inner { fn drop(&mut self) { - let (units, remainder) = { + let settlement = { let state = self.state.get_mut().unwrap(); state.accrue(Instant::now()); state.take_settlement() }; - self.record(units); - self.transfer_remainder(remainder); + self.record_settlement(settlement); } } @@ -239,11 +248,11 @@ mod tests { meter.pause(now + Duration::from_secs(2)); meter.pause(now + Duration::from_secs(3)); - meter.resume(gib(1), now + Duration::from_secs(4)); - meter.resume(gib(1), now + Duration::from_secs(5)); + assert!(meter.resume(gib(1), now + Duration::from_secs(4))); + assert!(meter.resume(gib(1), now + Duration::from_secs(5))); meter.stop(now + Duration::from_secs(7)); meter.stop(now + Duration::from_secs(8)); - meter.resume(gib(1), now + Duration::from_secs(9)); + assert!(!meter.resume(gib(1), now + Duration::from_secs(9))); meter.flush(now + Duration::from_secs(10)); assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 5); @@ -336,7 +345,7 @@ mod tests { let meter = AgentMemoryMeter::new(AgentMode::Durable, gib(1), true, entry.clone(), now); meter.pause(now + Duration::from_secs(2)); - meter.resume(gib(2), now + Duration::from_secs(3)); + assert!(meter.resume(gib(2), now + Duration::from_secs(3))); meter.set_bytes(gib(3), now + Duration::from_secs(4)); assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 0); diff --git a/golem-worker-executor/src/services/agent_resource_billing.rs b/golem-worker-executor/src/services/agent_resource_billing.rs new file mode 100644 index 0000000000..c0b6b92274 --- /dev/null +++ b/golem-worker-executor/src/services/agent_resource_billing.rs @@ -0,0 +1,638 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::services::agent_filesystem::{ + AgentFilesystemRuntime, AgentFilesystemUsage, FilesystemStorageError, +}; +use crate::services::agent_storage_meter::{AgentStorageMeter, FilesystemUsageObservation}; +use crate::services::byte_time_accumulator::ByteTimeSettlement; +use crate::services::linear_memory::LinearMemoryTracker; +use crate::services::resource_limits::AtomicResourceEntry; +use golem_common::model::agent::AgentMode; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Instant; + +pub(crate) trait FilesystemUsageObserver: Send + Sync { + fn is_active(&self) -> bool { + true + } + + fn begin_observation(&self) -> FilesystemUsageObservation; + fn complete_observation( + &self, + observation: FilesystemUsageObservation, + usage: Option, + now: Instant, + ); + fn fail_observation(&self, observation: FilesystemUsageObservation) -> bool; +} + +#[derive(Clone, Debug)] +/// Long-lived owner of the memory and storage meters for one resident agent. +/// +/// Each `open`/`close` pair starts and settles one permit-owned billing interval. The shared +/// transition lock keeps both leaf meters on the same monotonic timeline. +pub(crate) struct AgentResourceBilling { + state: Arc, +} + +#[derive(Debug)] +struct AgentResourceBillingState { + mode: AgentMode, + entry: Weak, + linear_memory: LinearMemoryTracker, + transition: Arc>, + storage: AgentStorageMeter, +} + +impl AgentResourceBilling { + pub(crate) fn new( + mode: AgentMode, + linear_memory: LinearMemoryTracker, + entry: Arc, + now: Instant, + ) -> Self { + let transition = linear_memory.resource_transition(); + Self { + state: Arc::new(AgentResourceBillingState { + mode, + entry: Arc::downgrade(&entry), + linear_memory, + transition, + storage: AgentStorageMeter::new(mode, entry, now), + }), + } + } + + pub(crate) fn is_same_billing(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.state, &other.state) + } + + pub(crate) fn is_active(&self) -> bool { + self.state.storage.is_active() + } + + pub(crate) async fn open( + &self, + filesystem: &AgentFilesystemRuntime, + ) -> Result<(), FilesystemStorageError> { + let _admission_pause = filesystem.pause_effect_admission(); + filesystem.drain().await; + let usage = filesystem.usage().await?; + let now = Instant::now(); + if self.open_at(usage.map(|usage| usage.allocated_bytes), now) { + Ok(()) + } else { + Err(FilesystemStorageError::resource_billing_transition( + "open resource window with a stopped memory meter", + )) + } + } + + pub(crate) async fn close( + &self, + filesystem: &AgentFilesystemRuntime, + ) -> Result<(), FilesystemStorageError> { + let _admission_pause = filesystem.pause_effect_admission(); + filesystem.drain().await; + filesystem.wait_for_usage_completion_debounce().await; + let observation = self.begin_close_observation().ok_or_else(|| { + FilesystemStorageError::resource_billing_transition( + "begin terminal resource-window close", + ) + })?; + let usage = match filesystem.usage().await { + Ok(usage) => usage, + Err(error) => { + self.abort(); + return Err(error); + } + }; + let now = Instant::now(); + if self.close_at(observation, usage.map(|usage| usage.allocated_bytes), now) { + Ok(()) + } else { + Err(FilesystemStorageError::resource_billing_transition( + "complete terminal resource-window close", + )) + } + } + + pub(crate) fn flush(&self, now: Instant) { + let _transition = self.state.transition.lock().unwrap(); + let memory_units = self.state.linear_memory.meter().take_units(now); + let storage_units = self.state.storage.flush(now); + self.state.record_usage(memory_units, storage_units); + } + + pub(crate) fn abort(&self) { + let _transition = self.state.transition.lock().unwrap(); + let storage_settlement = self.state.storage.abort(); + if let Some(storage_settlement) = storage_settlement { + let memory_settlement = self + .state + .linear_memory + .meter() + .take_abort_settlement() + .unwrap_or_default(); + self.state + .record_settlement(memory_settlement, storage_settlement); + } + } + + pub(crate) fn enforce_memory_limit(&self, limit: u64) { + self.state + .linear_memory + .enforce_resource_memory_limit(limit); + } + + fn open_at(&self, allocated_bytes: Option, now: Instant) -> bool { + let _transition = self.state.transition.lock().unwrap(); + // Component instantiation reconciles every Wasmtime memory into this canonical tracker + // before the first window opens. Memory cannot shrink while the resident worker is idle, + // so this tracker read is the authoritative opening observation. + if !self + .state + .linear_memory + .meter() + .resume(self.state.linear_memory.current_bytes(), now) + { + return false; + } + self.state.storage.open(allocated_bytes, now); + true + } + + fn close_at( + &self, + observation: FilesystemUsageObservation, + allocated_bytes: Option, + now: Instant, + ) -> bool { + let _transition = self.state.transition.lock().unwrap(); + let storage_settlement = self.state.storage.close(observation, allocated_bytes, now); + if let Some(storage_settlement) = storage_settlement { + self.state.linear_memory.meter().pause(now); + let memory_settlement = self.state.linear_memory.meter().take_settlement(); + self.state + .record_settlement(memory_settlement, storage_settlement); + true + } else { + false + } + } + + fn begin_close_observation(&self) -> Option { + let _transition = self.state.transition.lock().unwrap(); + self.state.storage.begin_close() + } + + #[cfg(test)] + pub(crate) fn open_for_test(&self, allocated_bytes: Option, now: Instant) -> bool { + self.open_at(allocated_bytes, now) + } + + #[cfg(test)] + pub(crate) fn begin_close_for_test(&self) -> Option { + self.begin_close_observation() + } + + #[cfg(test)] + pub(crate) fn close_for_test( + &self, + observation: FilesystemUsageObservation, + allocated_bytes: Option, + now: Instant, + ) -> bool { + self.close_at(observation, allocated_bytes, now) + } +} + +impl FilesystemUsageObserver for AgentResourceBilling { + fn is_active(&self) -> bool { + AgentResourceBilling::is_active(self) + } + + fn begin_observation(&self) -> FilesystemUsageObservation { + self.state.storage.begin_observation() + } + + fn complete_observation( + &self, + observation: FilesystemUsageObservation, + usage: Option, + now: Instant, + ) { + let _transition = self.state.transition.lock().unwrap(); + let accepted = self.state.storage.complete_observation( + observation, + usage.map(|usage| usage.allocated_bytes), + now, + ); + if accepted { + self.state.linear_memory.meter().sample(now); + } + } + + fn fail_observation(&self, observation: FilesystemUsageObservation) -> bool { + let _transition = self.state.transition.lock().unwrap(); + let settlement = self.state.storage.fail_observation(observation); + if let Some(storage_settlement) = settlement { + let memory_settlement = self + .state + .linear_memory + .meter() + .take_abort_settlement() + .unwrap_or_default(); + self.state + .record_settlement(memory_settlement, storage_settlement); + true + } else { + false + } + } +} + +impl AgentResourceBillingState { + fn record_usage(&self, memory_units: i64, storage_units: i64) { + if let Some(entry) = self.entry.upgrade() { + entry.record_resource_usage(self.mode, memory_units, storage_units); + } + } + + fn record_settlement(&self, memory: ByteTimeSettlement, storage: ByteTimeSettlement) { + if let Some(entry) = self.entry.upgrade() { + entry.record_resource_settlement(self.mode, memory, storage); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::active_workers::MemoryGrant; + use std::time::Duration; + use test_r::test; + + fn meter(now: Instant) -> (Arc, AgentResourceBilling) { + let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); + let memory = LinearMemoryTracker::new( + 1024 * 1024 * 1024, + 1024 * 1024 * 1024, + AgentMode::Durable, + false, + entry.clone(), + Arc::new(Mutex::new(MemoryGrant::inert(0))), + now, + ); + let meter = AgentResourceBilling::new(AgentMode::Durable, memory, entry.clone(), now); + (entry, meter) + } + + #[test] + fn storage_is_prospective_and_final_level_has_zero_duration() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(100), t0); + let increase = meter.begin_observation(); + meter.complete_observation( + increase, + Some(AgentFilesystemUsage { + allocated_bytes: 300, + filesystem_objects: 0, + }), + t0 + Duration::from_secs(2), + ); + let close = meter.begin_close_for_test().unwrap(); + meter.close_for_test(close, Some(900), t0 + Duration::from_secs(5)); + meter.flush(t0 + Duration::from_secs(20)); + + assert_eq!(entry.durable_byte_seconds_delta(), 100 * 2 + 300 * 3); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 5); + } + + #[test] + fn permit_to_start_gap_and_paused_changes_are_unbilled() { + let acquired = Instant::now(); + let (entry, meter) = meter(acquired); + meter.open_for_test(Some(100), acquired + Duration::from_secs(3)); + let close = meter.begin_close_for_test().unwrap(); + meter.close_for_test(close, Some(200), acquired + Duration::from_secs(4)); + let paused = meter.begin_observation(); + meter.complete_observation( + paused, + Some(AgentFilesystemUsage { + allocated_bytes: 900, + filesystem_objects: 0, + }), + acquired + Duration::from_secs(8), + ); + meter.flush(acquired + Duration::from_secs(10)); + + assert_eq!(entry.durable_byte_seconds_delta(), 100); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 1); + } + + #[test] + fn stale_observation_cannot_replace_a_newer_level() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(100), t0); + let stale = meter.begin_observation(); + let current = meter.begin_observation(); + meter.complete_observation( + current, + Some(AgentFilesystemUsage { + allocated_bytes: 300, + filesystem_objects: 0, + }), + t0 + Duration::from_secs(1), + ); + meter.complete_observation( + stale, + Some(AgentFilesystemUsage { + allocated_bytes: 10, + filesystem_objects: 0, + }), + t0 + Duration::from_secs(2), + ); + let close = meter.begin_close_for_test().unwrap(); + meter.close_for_test(close, Some(300), t0 + Duration::from_secs(3)); + meter.flush(t0 + Duration::from_secs(3)); + + assert_eq!(entry.durable_byte_seconds_delta(), 100 + 300 * 2); + } + + #[test] + fn terminal_close_rejects_a_newer_sampler_completion() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(100), t0); + let close = meter.begin_close_for_test().unwrap(); + let newer_sample = meter.begin_observation(); + meter.complete_observation( + newer_sample, + Some(AgentFilesystemUsage { + allocated_bytes: 300, + filesystem_objects: 0, + }), + t0 + Duration::from_secs(2), + ); + + meter.close_for_test(close, Some(900), t0 + Duration::from_secs(5)); + let idle_sample = meter.begin_observation(); + meter.complete_observation( + idle_sample, + Some(AgentFilesystemUsage { + allocated_bytes: 1200, + filesystem_objects: 0, + }), + t0 + Duration::from_secs(10), + ); + meter.flush(t0 + Duration::from_secs(20)); + + assert!(!meter.is_active()); + assert_eq!(entry.durable_byte_seconds_delta(), 100 * 5); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 5); + } + + #[test] + fn failed_observation_aborts_both_meters_without_estimating_failed_interval() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(100), t0); + let completed = meter.begin_observation(); + meter.complete_observation( + completed, + Some(AgentFilesystemUsage { + allocated_bytes: 300, + filesystem_objects: 0, + }), + t0 + Duration::from_secs(2), + ); + + let failed = meter.begin_observation(); + assert!(meter.fail_observation(failed)); + meter.flush(t0 + Duration::from_secs(20)); + + assert!(!meter.is_active()); + assert_eq!(entry.durable_byte_seconds_delta(), 200); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 2); + } + + #[test] + fn failed_observation_prevents_a_partial_resource_window_reopen() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + assert!(meter.open_for_test(Some(100), t0)); + let failed = meter.begin_observation(); + assert!(meter.fail_observation(failed)); + + assert!(!meter.open_for_test(Some(300), t0 + Duration::from_secs(1))); + meter.flush(t0 + Duration::from_secs(10)); + + assert!(!meter.is_active()); + assert_eq!(entry.durable_byte_seconds_delta(), 0); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 0); + } + + #[test] + fn failed_observation_after_close_is_rejected() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(100), t0); + let failed = meter.begin_observation(); + let close = meter.begin_close_for_test().unwrap(); + + meter.close_for_test(close, Some(300), t0 + Duration::from_secs(2)); + assert!(!meter.fail_observation(failed)); + meter.flush(t0 + Duration::from_secs(10)); + + assert!(!meter.is_active()); + assert_eq!(entry.durable_byte_seconds_delta(), 200); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 2); + } + + #[test] + fn abort_after_close_cannot_discard_the_closed_window() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(100), t0); + let close = meter.begin_close_for_test().unwrap(); + + assert!(meter.close_for_test(close, Some(100), t0 + Duration::from_secs(2))); + meter.abort(); + + assert_eq!(entry.durable_byte_seconds_delta(), 200); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 2); + } + + #[test] + fn older_failed_observation_after_newer_sample_is_rejected() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(100), t0); + let failed = meter.begin_observation(); + let completed = meter.begin_observation(); + meter.complete_observation( + completed, + Some(AgentFilesystemUsage { + allocated_bytes: 300, + filesystem_objects: 0, + }), + t0 + Duration::from_secs(2), + ); + + assert!(!meter.fail_observation(failed)); + let close = meter.begin_close_for_test().unwrap(); + meter.close_for_test(close, Some(300), t0 + Duration::from_secs(5)); + meter.flush(t0 + Duration::from_secs(10)); + + assert_eq!(entry.durable_byte_seconds_delta(), 100 * 2 + 300 * 3); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 5); + } + + #[test] + fn unsupported_storage_keeps_memory_billing() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(None, t0); + let close = meter.begin_close_for_test().unwrap(); + meter.close_for_test(close, None, t0 + Duration::from_secs(2)); + meter.flush(t0 + Duration::from_secs(2)); + + assert_eq!(entry.durable_byte_seconds_delta(), 0); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 2); + } + + #[test] + fn storage_decrease_is_prospective() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(300), t0); + let decrease = meter.begin_observation(); + meter.complete_observation( + decrease, + Some(AgentFilesystemUsage { + allocated_bytes: 100, + filesystem_objects: 0, + }), + t0 + Duration::from_secs(2), + ); + let close = meter.begin_close_for_test().unwrap(); + meter.close_for_test(close, Some(100), t0 + Duration::from_secs(5)); + meter.flush(t0 + Duration::from_secs(5)); + + assert_eq!(entry.durable_byte_seconds_delta(), 300 * 2 + 100 * 3); + } + + #[test] + async fn opening_observation_failure_starts_neither_meter() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + let runtime = AgentFilesystemRuntime::new_for_test_with_failed_observations(); + + assert!(meter.open(&runtime).await.is_err()); + meter.flush(t0 + Duration::from_secs(10)); + + assert!(!meter.is_active()); + assert_eq!(entry.durable_byte_seconds_delta(), 0); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 0); + } + + #[test] + async fn close_owns_effect_admission_until_the_final_observation() { + let t0 = Instant::now(); + let (_, meter) = meter(t0); + let runtime = AgentFilesystemRuntime::new_for_test(); + meter.open(&runtime).await.unwrap(); + let active_effect = runtime.begin_effect().await.unwrap(); + let close_meter = meter.clone(); + let close_runtime = runtime.clone(); + let close = tokio::spawn(async move { close_meter.close(&close_runtime).await }); + + while !runtime.effect_admission_is_paused() { + tokio::task::yield_now().await; + } + assert!(runtime.begin_effect().await.is_err()); + drop(active_effect); + close.await.unwrap().unwrap(); + assert!(runtime.begin_effect().await.is_ok()); + } + + #[test] + fn failed_close_discards_the_unsampled_tail() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(100), t0); + let completed = meter.begin_observation(); + meter.complete_observation( + completed, + Some(AgentFilesystemUsage { + allocated_bytes: 300, + filesystem_objects: 0, + }), + t0 + Duration::from_secs(1), + ); + meter.abort(); + meter.flush(t0 + Duration::from_secs(10)); + + assert_eq!(entry.durable_byte_seconds_delta(), 100); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 1); + } + + #[test] + fn sampler_failure_cannot_abort_a_window_owned_by_terminal_close() { + let t0 = Instant::now(); + let (entry, meter) = meter(t0); + meter.open_for_test(Some(100), t0); + let close = meter.begin_close_for_test().unwrap(); + let sampler = meter.begin_observation(); + + assert!(!meter.fail_observation(sampler)); + meter.close_for_test(close, Some(300), t0 + Duration::from_secs(5)); + meter.flush(t0 + Duration::from_secs(5)); + + assert_eq!(entry.durable_byte_seconds_delta(), 500); + assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 5); + } + + #[test] + fn account_storage_remainder_crosses_short_windows() { + let t0 = Instant::now(); + let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); + for offset in [0, 400] { + let memory = LinearMemoryTracker::new( + 0, + 0, + AgentMode::Durable, + false, + entry.clone(), + Arc::new(Mutex::new(MemoryGrant::inert(0))), + t0, + ); + let meter = AgentResourceBilling::new( + AgentMode::Durable, + memory, + entry.clone(), + t0 + Duration::from_millis(offset), + ); + meter.open_for_test(Some(1), t0 + Duration::from_millis(offset)); + let close = meter.begin_close_for_test().unwrap(); + meter.close_for_test(close, Some(1), t0 + Duration::from_millis(offset + 600)); + } + + assert_eq!(entry.durable_byte_seconds_delta(), 1); + } +} diff --git a/golem-worker-executor/src/services/agent_storage_meter.rs b/golem-worker-executor/src/services/agent_storage_meter.rs index 3f27f0fbd5..3c59d1fa43 100644 --- a/golem-worker-executor/src/services/agent_storage_meter.rs +++ b/golem-worker-executor/src/services/agent_storage_meter.rs @@ -12,232 +12,197 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Per-agent byte-second metering. Clones share one meter; dropping the last clone records any -//! whole byte-seconds accumulated since the previous flush. - +use crate::services::byte_time_accumulator::{ByteTimeAccumulator, ByteTimeSettlement}; use crate::services::resource_limits::AtomicResourceEntry; use golem_common::model::agent::AgentMode; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, Weak}; use std::time::Instant; -#[derive(Clone, Debug)] -pub struct AgentStorageMeter { - inner: Arc, -} +const BYTE_NANOSECONDS_PER_BYTE_SECOND: u128 = 1_000_000_000; -impl AgentStorageMeter { - /// Whether both handles refer to the same underlying meter. - /// - /// Deliberately not `PartialEq`: this is identity, not value equality — two meters - /// holding identical byte counts are still different meters, and unregistration must - /// only remove the exact handle it was given. - pub fn is_same_meter(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.inner, &other.inner) - } +#[derive(Clone, Copy, Debug)] +pub(crate) struct FilesystemUsageObservation { + pub(crate) generation: u64, + pub(crate) sequence: u64, } +/// Leaf meter for authoritative filesystem-storage levels and byte-time integration. +/// +/// `AgentResourceBilling` invokes lifecycle methods under the transition lock shared with +/// memory accounting. Observation generations and sequences reject stale asynchronous results. #[derive(Debug)] -struct Inner { +pub(crate) struct AgentStorageMeter { mode: AgentMode, entry: Weak, - state: Mutex, + generation: AtomicU64, + sequence: AtomicU64, + state: Mutex, } #[derive(Debug)] -struct State { - bytes: u64, - last_sample: Instant, - pending_byte_nanoseconds: u128, +struct AgentStorageMeterState { + active: bool, + closing: bool, + allocated_bytes: Option, + generation: u64, + applied_sequence: u64, + usage: ByteTimeAccumulator, } impl AgentStorageMeter { - pub fn new(mode: AgentMode, bytes: u64, entry: Arc, now: Instant) -> Self { + pub(crate) fn new(mode: AgentMode, entry: Arc, now: Instant) -> Self { Self { - inner: Arc::new(Inner { - mode, - entry: Arc::downgrade(&entry), - state: Mutex::new(State { - bytes, - last_sample: now, - pending_byte_nanoseconds: 0, - }), + mode, + entry: Arc::downgrade(&entry), + generation: AtomicU64::new(0), + sequence: AtomicU64::new(0), + state: Mutex::new(AgentStorageMeterState { + active: false, + closing: false, + allocated_bytes: None, + generation: 0, + applied_sequence: 0, + usage: ByteTimeAccumulator::new(BYTE_NANOSECONDS_PER_BYTE_SECOND, now), }), } } - pub fn on_acquire(&self, bytes: u64, now: Instant) { - self.inner.update_bytes(bytes, true, now); - } - - pub fn on_release(&self, bytes: u64, now: Instant) { - self.inner.update_bytes(bytes, false, now); - } - - pub fn flush(&self, now: Instant) { - self.inner.integrate(now); - } -} - -impl Inner { - fn update_bytes(&self, bytes: u64, acquire: bool, now: Instant) { - let byte_seconds = { - let mut state = self.state.lock().unwrap(); - let byte_seconds = state.take_whole_byte_seconds(now); - state.bytes = if acquire { - state.bytes.saturating_add(bytes) - } else { - state.bytes.saturating_sub(bytes) - }; - byte_seconds - }; - self.record(byte_seconds); - } - - fn integrate(&self, now: Instant) { - let byte_seconds = self.state.lock().unwrap().take_whole_byte_seconds(now); - self.record(byte_seconds); - } - - fn record(&self, byte_seconds: i64) { - if byte_seconds == 0 { - return; - } - if let Some(entry) = self.entry.upgrade() { - entry.record_storage_byte_seconds(self.mode, byte_seconds); + pub(crate) fn is_active(&self) -> bool { + self.state.lock().unwrap().active + } + + pub(crate) fn open(&self, allocated_bytes: Option, now: Instant) { + let generation = self + .generation + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + value.checked_add(1) + }) + .expect("resource-window generation overflowed") + + 1; + let sequence = self.next_sequence(); + let mut state = self.state.lock().unwrap(); + state.accrue(now); + state.active = true; + state.closing = false; + state.allocated_bytes = allocated_bytes; + state.generation = generation; + state.applied_sequence = sequence; + } + + pub(crate) fn begin_observation(&self) -> FilesystemUsageObservation { + FilesystemUsageObservation { + generation: self.generation.load(Ordering::Acquire), + sequence: self.next_sequence(), } } -} -impl State { - fn take_whole_byte_seconds(&mut self, now: Instant) -> i64 { - if now <= self.last_sample { - return 0; + pub(crate) fn begin_close(&self) -> Option { + let mut state = self.state.lock().unwrap(); + if !state.active || state.closing { + return None; } - - let elapsed_nanoseconds = now.saturating_duration_since(self.last_sample).as_nanos(); - self.last_sample = now; - if self.bytes == 0 { - return 0; + state.closing = true; + Some(FilesystemUsageObservation { + generation: state.generation, + sequence: self.next_sequence(), + }) + } + + pub(crate) fn complete_observation( + &self, + observation: FilesystemUsageObservation, + allocated_bytes: Option, + now: Instant, + ) -> bool { + let mut state = self.state.lock().unwrap(); + let accepted = state.active + && !state.closing + && observation.generation == state.generation + && observation.sequence > state.applied_sequence; + if accepted { + state.accrue(now); + state.allocated_bytes = allocated_bytes; + state.applied_sequence = observation.sequence; } - self.pending_byte_nanoseconds = self - .pending_byte_nanoseconds - .saturating_add((self.bytes as u128).saturating_mul(elapsed_nanoseconds)); - if self.pending_byte_nanoseconds < 1_000_000_000 { - return 0; + accepted + } + + pub(crate) fn close( + &self, + observation: FilesystemUsageObservation, + allocated_bytes: Option, + now: Instant, + ) -> Option { + let mut state = self.state.lock().unwrap(); + let accepted = state.active && state.closing && observation.generation == state.generation; + if !accepted { + return None; + } + state.accrue(now); + state.allocated_bytes = allocated_bytes; + state.applied_sequence = state.applied_sequence.max(observation.sequence); + state.active = false; + state.closing = false; + Some(state.usage.take_settlement()) + } + + pub(crate) fn fail_observation( + &self, + observation: FilesystemUsageObservation, + ) -> Option { + let mut state = self.state.lock().unwrap(); + let accepted = state.active + && !state.closing + && observation.generation == state.generation + && observation.sequence > state.applied_sequence; + accepted.then(|| { + state.active = false; + state.usage.take_settlement() + }) + } + + pub(crate) fn flush(&self, now: Instant) -> i64 { + let mut state = self.state.lock().unwrap(); + state.accrue(now); + state.usage.take_units() + } + + pub(crate) fn abort(&self) -> Option { + let mut state = self.state.lock().unwrap(); + state.active.then(|| { + state.active = false; + state.closing = false; + state.usage.take_settlement() + }) + } + + fn next_sequence(&self) -> u64 { + self.sequence + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + value.checked_add(1) + }) + .expect("filesystem usage observation sequence overflowed") + + 1 + } + + fn record_settlement(&self, settlement: ByteTimeSettlement) { + if let Some(entry) = self.entry.upgrade() { + entry.record_storage_settlement(self.mode, settlement); } - - let byte_seconds = self.pending_byte_nanoseconds / 1_000_000_000; - self.pending_byte_nanoseconds %= 1_000_000_000; - byte_seconds.min(i64::MAX as u128) as i64 } } -impl Drop for Inner { - fn drop(&mut self) { - self.integrate(Instant::now()); +impl AgentStorageMeterState { + fn accrue(&mut self, now: Instant) { + let allocated_bytes = self.active.then_some(self.allocated_bytes).flatten(); + self.usage.accrue(now, allocated_bytes); } } -#[cfg(test)] -mod tests { - use super::*; - use proptest::prelude::*; - use std::time::Duration; - use test_r::test; - - #[test] - fn integrates_acquire_release_and_flush() { - let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); - let now = Instant::now(); - let meter = AgentStorageMeter::new(AgentMode::Durable, 10, entry.clone(), now); - - meter.on_acquire(5, now + Duration::from_secs(2)); - meter.on_release(3, now + Duration::from_secs(4)); - meter.flush(now + Duration::from_secs(5)); - - assert_eq!(entry.durable_byte_seconds_delta(), 62); - } - - #[test] - fn meters_ephemeral_storage_separately() { - let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); - let now = Instant::now(); - let meter = AgentStorageMeter::new(AgentMode::Ephemeral, 10, entry.clone(), now); - - meter.flush(now + Duration::from_secs(3)); - - assert_eq!(entry.ephemeral_byte_seconds_delta(), 30); - assert_eq!(entry.durable_byte_seconds_delta(), 0); - } - - #[test] - fn ignores_a_stale_flush_timestamp() { - let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); - let now = Instant::now(); - let meter = AgentStorageMeter::new(AgentMode::Durable, 10, entry.clone(), now); - - meter.on_acquire(5, now + Duration::from_secs(2)); - meter.flush(now + Duration::from_secs(1)); - meter.flush(now + Duration::from_secs(4)); - - assert_eq!(entry.durable_byte_seconds_delta(), 50); - } - - #[test] - fn retains_sub_byte_second_remainder_without_division() { - let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); - let now = Instant::now(); - let meter = AgentStorageMeter::new(AgentMode::Durable, 1024, entry.clone(), now); - - meter.flush(now + Duration::from_micros(1)); - assert_eq!(entry.durable_byte_seconds_delta(), 0); - - meter.flush(now + Duration::from_secs(1)); - assert_eq!(entry.durable_byte_seconds_delta(), 1024); - } - - #[test] - fn cloned_meter_flushes_shared_state() { - let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); - let now = Instant::now(); - let meter = AgentStorageMeter::new(AgentMode::Durable, 0, entry.clone(), now); - let flusher = meter.clone(); - - meter.on_acquire(10, now + Duration::from_secs(1)); - flusher.flush(now + Duration::from_secs(3)); - - assert_eq!(entry.durable_byte_seconds_delta(), 20); - } - - proptest! { - #[test] - fn integrates_arbitrary_monotonic_storage_changes( - operations in prop::collection::vec((0u8..3, 0u64..1024, 1u64..5), 1..100), - ) { - let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); - let mut now = Instant::now(); - let meter = AgentStorageMeter::new(AgentMode::Durable, 10, entry.clone(), now); - let mut bytes = 10u64; - let mut expected = 0u64; - - for (operation, amount, elapsed_seconds) in operations { - now += Duration::from_secs(elapsed_seconds); - expected += bytes * elapsed_seconds; - - match operation { - 0 => { - meter.on_acquire(amount, now); - bytes = bytes.saturating_add(amount); - } - 1 => { - meter.on_release(amount, now); - bytes = bytes.saturating_sub(amount); - } - _ => meter.flush(now), - } - } - - prop_assert_eq!(entry.durable_byte_seconds_delta(), expected as i64); - } +impl Drop for AgentStorageMeter { + fn drop(&mut self) { + let settlement = self.state.get_mut().unwrap().usage.take_settlement(); + self.record_settlement(settlement); } } diff --git a/golem-worker-executor/src/services/byte_time_accumulator.rs b/golem-worker-executor/src/services/byte_time_accumulator.rs new file mode 100644 index 0000000000..8135444906 --- /dev/null +++ b/golem-worker-executor/src/services/byte_time_accumulator.rs @@ -0,0 +1,93 @@ +// Copyright 2024-2026 Golem Cloud +// +// Licensed under the Golem Source License v1.1 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://license.golem.cloud/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Instant; + +#[derive(Debug)] +pub(crate) struct ByteTimeAccumulator { + byte_nanoseconds_per_unit: u128, + last_sample: Instant, + pending_units: u128, + pending_byte_nanoseconds: u128, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct ByteTimeSettlement { + pub(crate) units: u128, + pub(crate) remainder: u128, +} + +impl ByteTimeAccumulator { + pub(crate) fn new(byte_nanoseconds_per_unit: u128, now: Instant) -> Self { + assert!(byte_nanoseconds_per_unit != 0); + Self { + byte_nanoseconds_per_unit, + last_sample: now, + pending_units: 0, + pending_byte_nanoseconds: 0, + } + } + + pub(crate) fn accrue(&mut self, now: Instant, bytes: Option) { + if now <= self.last_sample { + return; + } + + let elapsed = now.saturating_duration_since(self.last_sample).as_nanos(); + self.last_sample = now; + if let Some(bytes) = bytes { + self.pending_byte_nanoseconds = self + .pending_byte_nanoseconds + .saturating_add(u128::from(bytes).saturating_mul(elapsed)); + } + + let units = self.pending_byte_nanoseconds / self.byte_nanoseconds_per_unit; + self.pending_byte_nanoseconds %= self.byte_nanoseconds_per_unit; + self.pending_units = self.pending_units.saturating_add(units); + } + + pub(crate) fn take_units(&mut self) -> i64 { + let units = self.pending_units.min(i64::MAX as u128) as i64; + self.pending_units -= units as u128; + units + } + + pub(crate) fn take_settlement(&mut self) -> ByteTimeSettlement { + ByteTimeSettlement { + units: std::mem::take(&mut self.pending_units), + remainder: std::mem::take(&mut self.pending_byte_nanoseconds), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use test_r::test; + + #[test] + fn units_above_the_batch_range_remain_pending() { + let now = Instant::now(); + let mut accumulator = ByteTimeAccumulator::new(1, now); + accumulator.accrue(now + Duration::from_nanos(2), Some(u64::MAX)); + + assert_eq!(accumulator.take_units(), i64::MAX); + assert_eq!(accumulator.take_units(), i64::MAX); + assert_eq!(accumulator.take_units(), i64::MAX); + assert_eq!(accumulator.take_units(), i64::MAX); + assert_eq!(accumulator.take_units(), 2); + assert_eq!(accumulator.take_units(), 0); + } +} diff --git a/golem-worker-executor/src/services/events.rs b/golem-worker-executor/src/services/events.rs index dbff6babdd..0cc6771bd3 100644 --- a/golem-worker-executor/src/services/events.rs +++ b/golem-worker-executor/src/services/events.rs @@ -15,6 +15,7 @@ use golem_common::model::{AgentId, AgentInvocationOutput, IdempotencyKey}; use golem_service_base::error::worker_executor::WorkerExecutorError; use tokio::sync::broadcast::error::RecvError; +use uuid::Uuid; pub struct Events { sender: tokio::sync::broadcast::Sender, @@ -80,6 +81,7 @@ pub enum Event { }, WorkerLoaded { agent_id: AgentId, + start_attempt: Uuid, result: Result<(), WorkerExecutorError>, }, } diff --git a/golem-worker-executor/src/services/file_loader.rs b/golem-worker-executor/src/services/file_loader.rs index 46ece34574..e10c466693 100644 --- a/golem-worker-executor/src/services/file_loader.rs +++ b/golem-worker-executor/src/services/file_loader.rs @@ -28,9 +28,6 @@ use tempfile::TempDir; use tokio::io::AsyncWriteExt; use tracing::debug; -use crate::metrics::storage::record_filesystem_pool_released; -use crate::services::active_workers::{FilesystemStoragePermit, FilesystemStorageSemaphore}; - // Opaque token for read-only files. This is used to ensure that the file is not deleted while it is in use. // Make sure to not drop this token until you are done with the file. #[derive(Debug, Clone)] @@ -38,10 +35,24 @@ pub struct FileUseToken { _handle: Arc, } -/// Interface for loading files and making them available to workers. -/// -/// This will hardlink to a temporary directory to avoid copying files between workers. Beware -/// that hardlinking is only possible within the same filesystem. +#[derive(Debug, Clone)] +pub(crate) struct InitialFileSource { + path: PathBuf, + size: u64, + _token: FileUseToken, +} + +impl InitialFileSource { + pub(crate) fn path(&self) -> &Path { + &self.path + } + + pub(crate) fn size(&self) -> u64 { + self.size + } +} + +/// Interface for loading immutable, content-addressed initial-file sources. pub struct FileLoader { initial_agent_files_service: Arc, cache_dir: TempDir, @@ -52,156 +63,63 @@ pub struct FileLoader { // We need to ensure that no one else is using the file while we are deleting it. // To do that, give every file a unique number. item_counter: AtomicU64, - /// Executor-wide storage semaphore. When `Some`, acquiring a new cache - /// entry (i.e. downloading the file for the first time) also acquires - /// semaphore permits proportional to the file size. The permit is embedded - /// in the cache entry and released automatically when the last - /// `FileUseToken` holding that entry is dropped. Subsequent workers that - /// hardlink to the same cached file do not acquire additional permits. - filesystem_storage_semaphore: Option>, } impl FileLoader { pub fn new( initial_agent_files_service: Arc, - filesystem_storage_semaphore: Option>, + cache_parent: Option<&Path>, ) -> Result { - let cache_dir = tempfile::Builder::new() - .prefix("golem-initial-component-files") - .tempdir()?; + let mut cache = tempfile::Builder::new(); + cache.prefix("golem-initial-component-files"); + let cache_dir = match cache_parent { + Some(parent) => cache.tempdir_in(parent)?, + None => cache.tempdir()?, + }; Ok(Self { initial_agent_files_service, cache: Mutex::new(HashMap::new()), cache_dir, item_counter: AtomicU64::new(0), - filesystem_storage_semaphore, }) } - /// Read-only files can be safely shared between workers. Download once to cache and hardlink to target. - /// The file will only be valid until the token is dropped. - /// - /// `file_size` is the size of the file in bytes. It is used to acquire the - /// executor-wide storage semaphore permit on the first download (cache miss). - /// Cache hits (hardlinks) do not acquire additional permits. - pub async fn get_read_only_to( + pub(crate) async fn get_source( &self, environment_id: EnvironmentId, key: AgentFileContentHash, - target: &PathBuf, file_size: u64, - ) -> Result { - self.get_read_only_to_impl(environment_id, key, target, file_size) - .await - .map_err(|e| { - WorkerExecutorError::initial_file_download_failed( - target.display().to_string(), - e.to_string(), - ) - }) - } - - /// Read-write files are copied to target. - pub async fn get_read_write_to( - &self, - environment_id: EnvironmentId, - key: AgentFileContentHash, - target: &PathBuf, - ) -> Result<(), WorkerExecutorError> { - self.get_read_write_to_impl(environment_id, key, target) + ) -> Result { + let cache_entry = self + .get_or_add_cache_entry(environment_id, key, file_size) .await - .map_err(|e| { + .map_err(|error| { WorkerExecutorError::initial_file_download_failed( - target.display().to_string(), - e.to_string(), + key.to_string(), + error.to_string(), ) - }) - } - - async fn get_read_only_to_impl( - &self, - environment_id: EnvironmentId, - key: AgentFileContentHash, - target: &PathBuf, - file_size: u64, - ) -> Result { - if let Some(parent) = target.parent() { - tokio::fs::create_dir_all(parent).await?; - }; - - let cache_entry = self - .get_or_add_cache_entry(environment_id, key, file_size) - .await?; - - // peek at the cache entry. It's fine to not hold the lock here. - // as long as we keep a ref to the cache entry, the file will not be deleted - let cache_entry_path = { + })?; + let (path, size) = { let cache_entry_guard = cache_entry.lock().await; - cache_entry_guard - .as_ref() - .map_err(|e| anyhow!(e.clone()))? - .path - .clone() + let entry = cache_entry_guard.as_ref().map_err(|error| { + WorkerExecutorError::initial_file_download_failed(key.to_string(), error.clone()) + })?; + (entry.path.clone(), entry.size) }; - - debug!( - "Hardlinking {} to {}", - cache_entry_path.display(), - target.display() - ); - tokio::fs::hard_link(&cache_entry_path, target).await?; - - Ok(FileUseToken { - _handle: cache_entry, - }) - } - - async fn get_read_write_to_impl( - &self, - environment_id: EnvironmentId, - key: AgentFileContentHash, - target: &PathBuf, - ) -> Result<(), anyhow::Error> { - if let Some(parent) = target.parent() { - tokio::fs::create_dir_all(parent).await?; - }; - - // fast path for files that are already in cache - { - let cache_guard = self.cache.lock().await; - let cache_entry = cache_guard.get(&key).and_then(|weak| weak.upgrade()); - - // make sure we drop the lock to not block other threads - drop(cache_guard); - - if let Some(cache_entry) = cache_entry { - // peek at the cache entry. It's fine to not hold the lock here. - // as long as we keep a ref to the cache entry, the file will not be deleted - let cache_entry_path = { - let cache_entry_guard = cache_entry.lock().await; - cache_entry_guard - .as_ref() - .map_err(|e| anyhow!(e.clone()))? - .path - .clone() - }; - - // copy the file to the target - debug!( - "Copying {} to {}", - cache_entry_path.display(), - target.display() - ); - tokio::fs::copy(&cache_entry_path, target).await?; - return Ok(()); - } + if size != file_size { + return Err(WorkerExecutorError::initial_file_download_failed( + key.to_string(), + format!("Cached initial file size {size} does not match declared size {file_size}"), + )); } - - // alternative, download the file directly to the target - self.download_file_to_path(environment_id, target, key) - .await?; - Ok(()) + Ok(InitialFileSource { + path, + size, + _token: FileUseToken { + _handle: cache_entry, + }, + }) } async fn get_or_add_cache_entry( @@ -243,53 +161,19 @@ impl FileLoader { .fetch_add(1, std::sync::atomic::Ordering::Relaxed); let path = self.cache_dir.path().join(counter.to_string()); - // Acquire the executor-wide storage permit before touching the - // filesystem. This is a cache miss: we are about to write - // `file_size` new bytes to the shared cache directory. If the - // pool is exhausted we fail immediately; the worker will be - // retried once space is freed. - let filesystem_storage_permit = { - if let Some(sem) = &self.filesystem_storage_semaphore { - match sem.try_acquire(file_size).await { - Some(permit) => Some(permit), - None => { - *prelocked_entry = - Err("Executor storage pool exhausted".to_string()); - self.cache.lock().await.remove(&key); - return Err(anyhow!( - "Executor storage pool exhausted for initial component file" - )); - } - } - } else { - None - } - }; - match self - .download_file_to_path_as_read_only(environment_id, &path, key) + .download_file_to_path_as_read_only(environment_id, &path, key, file_size) .await { Ok(()) => { // we successfully downloaded the file and set it to read-only, set the cache entry to the file *prelocked_entry = Ok(InitializedCacheEntry { path: path.clone(), - filesystem_storage_permit_bytes: if filesystem_storage_permit.is_some() - { - // Round up to permit granularity (1 permit = 1 KB) so the - // released byte count matches what was actually acquired. - crate::services::active_workers::filesystem_storage_bytes_rounded_up( - file_size, - ) - } else { - 0 - }, - filesystem_storage_permit, + size: file_size, }); } Err(e) => { // we failed to set the file to read-only, we need to fail the entry, remove it from the cache and return the error - // filesystem_storage_permit is dropped here, returning permits to the semaphore *prelocked_entry = Err(format!("Other thread failed to download: {e}")); self.cache.lock().await.remove(&key); @@ -307,21 +191,27 @@ impl FileLoader { environment_id: EnvironmentId, path: &Path, key: AgentFileContentHash, + expected_size: u64, ) -> Result<(), anyhow::Error> { - self.download_file_to_path(environment_id, path, key) + let temporary = tempfile::NamedTempFile::new_in(self.cache_dir.path())?; + self.download_file(environment_id, &temporary, key, expected_size) .await?; - self.set_path_read_only(path).await?; + crate::services::agent_filesystem::set_initial_file_permissions(temporary.as_file(), true)?; + temporary.as_file().sync_all()?; + temporary + .persist_noclobber(path) + .map_err(|error| error.error)?; Ok(()) } - async fn download_file_to_path( + async fn download_file( &self, environment_id: EnvironmentId, - path: &Path, + temporary: &tempfile::NamedTempFile, key: AgentFileContentHash, + expected_size: u64, ) -> Result<(), anyhow::Error> { - debug!("Downloading {} to {}", key, path.display()); - + debug!("Downloading {} to immutable cache", key); let mut data = self .initial_agent_files_service .get(environment_id, key) @@ -329,21 +219,37 @@ impl FileLoader { .map_err(|e| anyhow!(e))? .ok_or_else(|| anyhow!("File not found"))?; - let file = tokio::fs::File::create(path).await?; + let file = tokio::fs::File::from_std(temporary.reopen()?); let mut writer = tokio::io::BufWriter::new(file); + let mut hasher = blake3::Hasher::new(); + let mut actual_size = 0u64; while let Some(chunk) = data.try_next().await.map_err(|e| anyhow!(e))? { + actual_size = actual_size + .checked_add(chunk.len() as u64) + .ok_or_else(|| anyhow!("Downloaded initial file size overflowed"))?; + if actual_size > expected_size { + return Err(anyhow!( + "Downloaded initial file size exceeds declared size {expected_size}" + )); + } + hasher.update(&chunk); writer.write_all(&chunk).await?; } writer.flush().await?; - Ok(()) - } - - async fn set_path_read_only(&self, path: &Path) -> Result<(), anyhow::Error> { - let mut perms = tokio::fs::metadata(path).await?.permissions(); - perms.set_readonly(true); - tokio::fs::set_permissions(path, perms).await?; + writer.get_ref().sync_all().await?; + let actual = hasher.finalize(); + if actual != *key.0.as_blake3_hash() { + return Err(anyhow!( + "Downloaded initial file content hash does not match {key}" + )); + } + if actual_size != expected_size { + return Err(anyhow!( + "Downloaded initial file size {actual_size} does not match declared size {expected_size}" + )); + } Ok(()) } } @@ -363,27 +269,13 @@ type CacheEntry = Mutex>; #[derive(Debug)] struct InitializedCacheEntry { path: PathBuf, - /// Storage semaphore permit held for the lifetime of this cache entry. - /// Acquired on cache miss (first download); `None` when no semaphore is - /// configured. Only returned to the executor pool if the file is - /// successfully deleted - filesystem_storage_permit: Option, - /// Byte count corresponding to `filesystem_storage_permit`, for metrics. - filesystem_storage_permit_bytes: u64, + size: u64, } impl InitializedCacheEntry { #[cfg(test)] - fn new_for_test( - path: PathBuf, - permit: Option, - permit_bytes: u64, - ) -> Self { - Self { - path, - filesystem_storage_permit: permit, - filesystem_storage_permit_bytes: permit_bytes, - } + fn new_for_test(path: PathBuf) -> Self { + Self { path, size: 0 } } } @@ -391,42 +283,24 @@ impl Drop for InitializedCacheEntry { fn drop(&mut self) { debug!("Removing file {}", self.path.display()); std::fs::remove_file(&self.path).expect("Failed to remove cached component file — executor filesystem is in an inconsistent state"); - // File deleted successfully — disk space is freed, return permits to the pool. - let permit = self.filesystem_storage_permit.take(); - if permit.is_some() && self.filesystem_storage_permit_bytes > 0 { - record_filesystem_pool_released(self.filesystem_storage_permit_bytes); - } - drop(permit); } } #[cfg(test)] mod tests { use super::*; - use crate::services::active_workers::FilesystemStorageSemaphore; use golem_common::model::environment::EnvironmentId; use golem_common::widen_infallible; use golem_service_base::replayable_stream::ReplayableStream as _; use golem_service_base::service::initial_agent_files::InitialAgentFilesService; use golem_service_base::storage::blob::memory::InMemoryBlobStorage; - use std::time::Duration; use test_r::test; test_r::enable!(); - /// Build a `FileLoader` + semaphore sharing a single in-memory blob store, - /// and upload `content` so it can be fetched via `get_read_only_to`. - /// - /// Returns `(loader, semaphore, content_hash, env_id)`. - async fn setup( - pool_bytes: usize, - content: &[u8], - ) -> ( - FileLoader, - Arc, - AgentFileContentHash, - EnvironmentId, - ) { + /// Build a `FileLoader` sharing a single in-memory blob store, + /// and upload `content` so it can be fetched as a verified source. + async fn setup(content: &[u8]) -> (FileLoader, AgentFileContentHash, EnvironmentId) { let blob = Arc::new(InMemoryBlobStorage::new()); // One service instance for uploading, one for the loader — both share @@ -434,11 +308,7 @@ mod tests { let upload_svc = Arc::new(InitialAgentFilesService::new(blob.clone())); let loader_svc = Arc::new(InitialAgentFilesService::new(blob)); - let semaphore = Arc::new(FilesystemStorageSemaphore::new( - pool_bytes, - Duration::from_millis(1), - )); - let loader = FileLoader::new(loader_svc, Some(semaphore.clone())).unwrap(); + let loader = FileLoader::new(loader_svc, None).unwrap(); let env_id = EnvironmentId::new(); let data: Vec = content.to_vec(); @@ -451,107 +321,28 @@ mod tests { .await .unwrap(); - (loader, semaphore, hash, env_id) + (loader, hash, env_id) } - /// A single `get_read_only_to` for a fresh file acquires permits equal to - /// the file size (rounded up to KB). #[test] - async fn ro_first_load_acquires_semaphore_permits() { - let content = b"hello world"; // 11 bytes → rounds up to 1 KB = 1 permit - let pool_bytes = 4 * 1024; - let (loader, semaphore, hash, env_id) = setup(pool_bytes, content).await; - - let dir = tempfile::tempdir().unwrap(); - let _token = loader - .get_read_only_to( - env_id, - hash, - &dir.path().join("f.txt"), - content.len() as u64, - ) - .await - .unwrap(); - - assert_eq!(semaphore.available_bytes(), 3 * 1024); - } - - /// A second `get_read_only_to` for the **same content hash** must NOT - /// consume additional semaphore permits — the file is already in the local - /// filesystem cache and only a hardlink is created, adding zero disk blocks. - #[test] - async fn ro_second_load_of_same_file_does_not_consume_additional_permits() { + async fn source_leases_keep_the_verified_cache_entry_alive() { let content = b"hello world"; - let pool_bytes = 4 * 1024; - let (loader, semaphore, hash, env_id) = setup(pool_bytes, content).await; + let (loader, hash, env_id) = setup(content).await; - let dir = tempfile::tempdir().unwrap(); - let _t1 = loader - .get_read_only_to( - env_id, - hash, - &dir.path().join("f1.txt"), - content.len() as u64, - ) + let source1 = loader + .get_source(env_id, hash, content.len() as u64) .await .unwrap(); - - let permits_after_first = semaphore.available_bytes(); - - let _t2 = loader - .get_read_only_to( - env_id, - hash, - &dir.path().join("f2.txt"), - content.len() as u64, - ) + let path = source1.path().to_path_buf(); + let source2 = loader + .get_source(env_id, hash, content.len() as u64) .await .unwrap(); - - assert_eq!( - semaphore.available_bytes(), - permits_after_first, - "second load of cached RO file must not consume extra semaphore permits" - ); - } - - /// When all `FileUseToken`s for a cached entry are dropped, the semaphore - /// permits are returned to the pool. - #[test] - async fn ro_permits_released_when_all_tokens_dropped() { - let content = b"hello world"; - let pool_bytes = 4 * 1024; - let (loader, semaphore, hash, env_id) = setup(pool_bytes, content).await; - - let dir = tempfile::tempdir().unwrap(); - let t1 = loader - .get_read_only_to( - env_id, - hash, - &dir.path().join("f1.txt"), - content.len() as u64, - ) - .await - .unwrap(); - let t2 = loader - .get_read_only_to( - env_id, - hash, - &dir.path().join("f2.txt"), - content.len() as u64, - ) - .await - .unwrap(); - - let after_load = semaphore.available_bytes(); - drop(t1); - assert_eq!(semaphore.available_bytes(), after_load, "t2 still alive"); - drop(t2); - assert_eq!( - semaphore.available_bytes(), - pool_bytes as u64, - "all tokens dropped — full pool must be restored" - ); + assert_eq!(source2.path(), path); + drop(source1); + assert!(path.exists()); + drop(source2); + assert!(!path.exists()); } /// When file deletion fails on drop (e.g. the file was already removed by @@ -563,7 +354,7 @@ mod tests { let result = std::panic::catch_unwind(|| { let nonexistent = std::path::PathBuf::from("/tmp/golem-test-nonexistent-file-12345.wasm"); - let entry = InitializedCacheEntry::new_for_test(nonexistent, None, 0); + let entry = InitializedCacheEntry::new_for_test(nonexistent); drop(entry); }); assert!( @@ -572,30 +363,15 @@ mod tests { ); } - /// When the semaphore pool is exhausted, `get_read_only_to` fails with an - /// error and the cache entry is cleaned up — a subsequent load of the same - /// hash (after the pool is freed) must succeed and download fresh. #[test] - async fn ro_load_fails_and_cleans_up_when_pool_exhausted() { + async fn source_load_rejects_incorrect_declared_size() { let content = b"hello world"; - let pool_bytes = 0; // 0 bytes → 0 permits → all acquires fail immediately - let (loader, semaphore, hash, env_id) = setup(pool_bytes, content).await; + let (loader, hash, env_id) = setup(content).await; - let dir = tempfile::tempdir().unwrap(); - - // Load must fail because the pool is empty. let result = loader - .get_read_only_to( - env_id, - hash, - &dir.path().join("f.txt"), - content.len() as u64, - ) + .get_source(env_id, hash, content.len() as u64 + 1) .await; - assert!(result.is_err(), "expected failure when pool is exhausted"); - // The failed load must have cleaned up the stale cache entry so the - // pool is still at 0 permits (no leak). - assert_eq!(semaphore.available_bytes(), 0); + assert!(result.is_err()); } } diff --git a/golem-worker-executor/src/services/golem_config.rs b/golem-worker-executor/src/services/golem_config.rs index 5e7e147013..b897fc7196 100644 --- a/golem-worker-executor/src/services/golem_config.rs +++ b/golem-worker-executor/src/services/golem_config.rs @@ -1776,47 +1776,12 @@ impl Default for MemoryConfig { } } -/// Configuration for the executor-wide worker storage semaphore. -/// -/// The semaphore pool size is `total_worker_filesystem_storage_bytes`. Workers acquire -/// permits proportional to their estimated storage usage; when the pool is -/// exhausted, idle workers are evicted to free space. Use -/// `total_worker_filesystem_storage_bytes` in tests to create a small, -/// predictable pool. -/// -/// # Permit release vs actual disk reclaim — configure with headroom -/// -/// When a worker is evicted its storage semaphore permits are released at the -/// moment `RunningWorker` drops, which is **slightly before** the worker's -/// temp directory is deleted from disk. The directory is removed when the -/// invocation task fully unwinds (dropping the wasmtime `Store` and its -/// contained `TempDir`). In practice this gap is sub-millisecond, but it means -/// the semaphore can briefly report available space that has not yet been -/// reclaimed on disk. -/// -/// This is the same race that exists for the memory semaphore -/// (`MemoryConfig::total_memory`): memory permits are released when -/// `RunningWorker` drops, before the wasmtime linear memory is actually freed. -/// It has never caused problems in production because the semaphore is not -/// configured to 100% of physical capacity. -/// -/// **Recommended practice:** assuming the executor's temp directory has a -/// dedicated volume (e.g. a pod-local tmpfs or block device mounted at `/tmp`), -/// set `total_worker_filesystem_storage_bytes` to around 80–90% of that volume's -/// capacity. The headroom absorbs the transient over-commitment window -/// described above and any filesystem metadata overhead for the temp directory -/// tree itself. +/// Configuration for managed agent filesystems and their cleanup. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FilesystemStorageConfig { - /// Override the total storage pool size (bytes). When `None`, the default - /// of 10 GB is used. Set to a small value in tests to trigger eviction. - /// - /// Should be set to ~80–90% of the dedicated volume capacity, not 100% — - /// see the `FilesystemStorageConfig` doc comment for the rationale. - #[serde(alias = "total_worker_filesystem_storage_bytes_override")] - pub total_worker_filesystem_storage_bytes: Option, - #[serde(with = "humantime_serde")] - pub acquire_retry_delay: Duration, + /// Retry policy for deleting and verifying runtime filesystem directories. + /// `max_attempts` includes the initial deletion attempt. + pub cleanup_retry: RetryConfig, /// When set, use deterministic per-agent directory names rooted at this /// path instead of random OS temp directories. The directory structure is: /// @@ -1828,31 +1793,138 @@ pub struct FilesystemStorageConfig { /// Directories are cleaned up when the worker is dropped, just like temp /// dirs. When `None` (the default), random temp directories are used. pub deterministic_root_dir: Option, + /// Dedicated XFS root managed through project quotas. Managed mode is + /// fail-closed and cannot be combined with `deterministic_root_dir`. + pub managed_xfs_root_dir: Option, + /// Private policy for deriving an agent's filesystem-object hard limit + /// proportionally from its allocated-byte limit, with fixed bounds. + pub filesystem_object_limit_policy: FilesystemObjectLimitPolicyConfig, + /// Physical capacity watermarks for managed filesystem pressure recovery. + #[serde(default)] + pub pressure: FilesystemPressureConfig, } -impl FilesystemStorageConfig { - /// The total number of bytes available to the storage semaphore pool. - pub fn worker_filesystem_storage(&self) -> usize { - self.total_worker_filesystem_storage_bytes - .unwrap_or(10 * 1024 * 1024 * 1024) // 10 GB default - as usize +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FilesystemPressureConfig { + /// Available bytes at or below which physical pressure is active. + pub minimum_available_bytes: u64, + /// Available bytes required after reclamation before retrying a mutation. + pub target_available_bytes: u64, + /// Available filesystem objects at or below which object pressure is active. + pub minimum_available_filesystem_objects: u64, + /// Available filesystem objects required after reclamation before retrying a mutation. + pub target_available_filesystem_objects: u64, + /// Number of fresh observations allowed after each completed deletion. + pub reclamation_observation_attempts: u32, + /// Delay between post-deletion observations while reclamation settles. + #[serde(with = "humantime_serde")] + pub reclamation_observation_delay: Duration, +} + +impl Default for FilesystemPressureConfig { + fn default() -> Self { + Self { + minimum_available_bytes: 64 * 1024 * 1024, + target_available_bytes: 128 * 1024 * 1024, + minimum_available_filesystem_objects: 8_192, + target_available_filesystem_objects: 16_384, + reclamation_observation_attempts: 4, + reclamation_observation_delay: Duration::from_millis(25), + } } } -impl SafeDisplay for FilesystemStorageConfig { +impl SafeDisplay for FilesystemPressureConfig { fn to_safe_string(&self) -> String { let mut result = String::new(); - if let Some(limit) = &self.total_worker_filesystem_storage_bytes { - let _ = writeln!(&mut result, "total worker storage bytes: {limit}"); + let _ = writeln!( + &mut result, + "minimum available bytes: {}", + self.minimum_available_bytes + ); + let _ = writeln!( + &mut result, + "target available bytes: {}", + self.target_available_bytes + ); + let _ = writeln!( + &mut result, + "minimum available filesystem objects: {}", + self.minimum_available_filesystem_objects + ); + let _ = writeln!( + &mut result, + "target available filesystem objects: {}", + self.target_available_filesystem_objects + ); + let _ = writeln!( + &mut result, + "reclamation observation attempts: {}", + self.reclamation_observation_attempts + ); + let _ = writeln!( + &mut result, + "reclamation observation delay: {:?}", + self.reclamation_observation_delay + ); + result + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FilesystemObjectLimitPolicyConfig { + /// Number of filesystem objects granted per GiB of allocated storage. + pub objects_per_gib: u64, + /// Object quota floor for small storage allocations. + pub minimum_objects: u64, + /// Object quota ceiling for large storage allocations. + pub maximum_objects: u64, +} + +impl Default for FilesystemObjectLimitPolicyConfig { + fn default() -> Self { + Self { + objects_per_gib: 32_768, + minimum_objects: 8192, + maximum_objects: 131_072, } + } +} + +impl SafeDisplay for FilesystemObjectLimitPolicyConfig { + fn to_safe_string(&self) -> String { + let mut result = String::new(); + let _ = writeln!(&mut result, "objects per GiB: {}", self.objects_per_gib); + let _ = writeln!(&mut result, "minimum objects: {}", self.minimum_objects); + let _ = writeln!(&mut result, "maximum objects: {}", self.maximum_objects); + result + } +} + +impl SafeDisplay for FilesystemStorageConfig { + fn to_safe_string(&self) -> String { + let mut result = String::new(); + let _ = writeln!(&mut result, "cleanup retry:"); let _ = writeln!( &mut result, - "acquire retry delay: {:?}", - self.acquire_retry_delay + "{}", + self.cleanup_retry.to_safe_string_indented() ); if let Some(root) = &self.deterministic_root_dir { let _ = writeln!(&mut result, "deterministic root dir: {}", root.display()); } + if let Some(root) = &self.managed_xfs_root_dir { + let _ = writeln!(&mut result, "managed XFS root dir: {}", root.display()); + } + let _ = writeln!(&mut result, "filesystem object limit policy:"); + let _ = writeln!( + &mut result, + "{}", + self.filesystem_object_limit_policy + .to_safe_string_indented() + ); + let _ = writeln!(&mut result, "pressure:"); + let _ = writeln!(&mut result, "{}", self.pressure.to_safe_string_indented()); result } } @@ -1860,9 +1932,17 @@ impl SafeDisplay for FilesystemStorageConfig { impl Default for FilesystemStorageConfig { fn default() -> Self { Self { - total_worker_filesystem_storage_bytes: None, - acquire_retry_delay: Duration::from_millis(500), + cleanup_retry: RetryConfig { + max_attempts: 4, + min_delay: Duration::from_millis(25), + max_delay: Duration::from_millis(250), + multiplier: 4.0, + max_jitter_factor: None, + }, deterministic_root_dir: None, + managed_xfs_root_dir: None, + filesystem_object_limit_policy: FilesystemObjectLimitPolicyConfig::default(), + pressure: FilesystemPressureConfig::default(), } } } diff --git a/golem-worker-executor/src/services/linear_memory.rs b/golem-worker-executor/src/services/linear_memory.rs index 2f074aa6ba..e6c35517e2 100644 --- a/golem-worker-executor/src/services/linear_memory.rs +++ b/golem-worker-executor/src/services/linear_memory.rs @@ -46,7 +46,7 @@ struct Inner { retained_growth_grant: Arc>, reconciling: AtomicBool, replaying: AtomicBool, - transitions: Mutex<()>, + transitions: Arc>, resource_entry: Arc, meter: AgentMemoryMeter, } @@ -73,9 +73,9 @@ impl LinearMemoryTracker { retained_growth_grant, reconciling: AtomicBool::new(true), replaying: AtomicBool::new(replaying), - transitions: Mutex::new(()), + transitions: Arc::new(Mutex::new(())), resource_entry: resource_entry.clone(), - meter: AgentMemoryMeter::new(mode, bytes, true, resource_entry, now), + meter: AgentMemoryMeter::new(mode, bytes, false, resource_entry, now), }), }; tracker @@ -247,19 +247,12 @@ impl LinearMemoryTracker { .store(true, Ordering::Release); } - pub fn resume(&self, now: Instant) { - let _transition = self.inner.transitions.lock().unwrap(); - self.inner.meter.resume(self.current_bytes(), now); + pub(crate) fn resource_transition(&self) -> Arc> { + Arc::clone(&self.inner.transitions) } - pub fn pause(&self, now: Instant) { - let _transition = self.inner.transitions.lock().unwrap(); - self.inner.meter.pause(now); - } - - pub fn stop(&self, now: Instant) { - let _transition = self.inner.transitions.lock().unwrap(); - self.inner.meter.stop(now); + pub(crate) fn enforce_resource_memory_limit(&self, limit: u64) { + self.inner.meter.enforce_limit(limit); } pub fn meter(&self) -> &AgentMemoryMeter { diff --git a/golem-worker-executor/src/services/mod.rs b/golem-worker-executor/src/services/mod.rs index 137e32e265..261d580819 100644 --- a/golem-worker-executor/src/services/mod.rs +++ b/golem-worker-executor/src/services/mod.rs @@ -13,11 +13,14 @@ // limitations under the License. pub mod active_workers; +pub mod agent_filesystem; pub mod agent_memory_meter; +pub mod agent_resource_billing; pub mod agent_storage_meter; pub mod agent_types; pub mod agent_webhooks; pub mod blob_store; +mod byte_time_accumulator; pub mod card; pub mod card_interest; pub mod compilation_limiter; diff --git a/golem-worker-executor/src/services/resource_limits.rs b/golem-worker-executor/src/services/resource_limits.rs index d18c0852d8..dbec14bbc7 100644 --- a/golem-worker-executor/src/services/resource_limits.rs +++ b/golem-worker-executor/src/services/resource_limits.rs @@ -17,8 +17,12 @@ use crate::metrics::resources::{ record_memory_gb_seconds, record_resource_usage_batch_update_failure, record_storage_byte_seconds, }; -use crate::services::agent_memory_meter::{AgentMemoryMeter, BYTE_NANOSECONDS_PER_GB_SECOND}; -use crate::services::agent_storage_meter::AgentStorageMeter; +use crate::services::agent_filesystem::{ + AgentFilesystemRuntime, AgentFilesystemStorageLimit, FilesystemStorageError, +}; +use crate::services::agent_memory_meter::BYTE_NANOSECONDS_PER_GB_SECOND; +use crate::services::agent_resource_billing::AgentResourceBilling; +use crate::services::byte_time_accumulator::ByteTimeSettlement; use crate::services::golem_config::ResourceLimitsConfig; use async_trait::async_trait; use chrono::Utc; @@ -32,7 +36,7 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicI64, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; -use tokio::sync::OnceCell; +use tokio::sync::{Mutex as AsyncMutex, OnceCell}; use tokio_util::sync::CancellationToken; use tracing::debug; use tracing::{Instrument, error, info_span}; @@ -45,24 +49,19 @@ pub struct AtomicResourceEntry { delta: AtomicI64, // any fuel consumption that is currently in flight to the server in_flight_delta: AtomicI64, - durable_byte_seconds_delta: AtomicI64, - ephemeral_byte_seconds_delta: AtomicI64, - storage_meters: Arc>, - memory_gb_seconds_delta: AtomicI64, in_flight_memory_gb_seconds_delta: AtomicI64, - durable_memory_gb_seconds_delta: AtomicI64, - ephemeral_memory_gb_seconds_delta: AtomicI64, in_flight_durable_memory_gb_seconds_delta: AtomicI64, in_flight_ephemeral_memory_gb_seconds_delta: AtomicI64, - memory_usage_transition: Mutex<()>, - memory_remainder: Mutex, - memory_meters: Arc>, + account_usage_accumulator: Mutex, + resource_billings: Arc>, // Current (cached) value of the account level worker memory limits max_memory: AtomicUsize, // Current (cached) value of the account level worker function table element limits max_table_elements: AtomicUsize, // Current (cached) value of the account level per-worker disk space limit max_disk_space: AtomicU64, + filesystem_limit_update: AsyncMutex<()>, + agent_filesystems: scc::HashMap, // Unix timestamp (seconds) of the last time fuel/memory were refreshed from // the server. Used by the background loop to detect idle accounts whose // cached limits have grown stale (e.g. after a plan change or monthly reset). @@ -103,6 +102,113 @@ struct CapturedUsageUpdate { ephemeral_memory_gb_seconds_delta: i64, } +#[derive(Debug, Default)] +/// Account-local consumption settled by resident meters but not yet captured for registry delivery. +/// +/// These values are usage, not reservations. Whole units remain as `u128` until `capture` removes +/// a wire-sized batch; sub-unit byte-nanosecond remainders carry across short-lived agent windows. +struct AccountUsageAccumulator { + durable_memory_gb_seconds: u128, + ephemeral_memory_gb_seconds: u128, + durable_storage_byte_seconds: u128, + ephemeral_storage_byte_seconds: u128, + memory_remainder: u128, + durable_storage_remainder: u128, + ephemeral_storage_remainder: u128, +} + +#[derive(Debug, Eq, PartialEq)] +struct CapturedAccountUsage { + memory_gb_seconds: i64, + durable_memory_gb_seconds: i64, + ephemeral_memory_gb_seconds: i64, + durable_storage_byte_seconds: i64, + ephemeral_storage_byte_seconds: i64, +} + +impl AccountUsageAccumulator { + fn add_memory(&mut self, mode: AgentMode, units: u128) { + let pending = match mode { + AgentMode::Durable => &mut self.durable_memory_gb_seconds, + AgentMode::Ephemeral => &mut self.ephemeral_memory_gb_seconds, + }; + *pending = pending.saturating_add(units); + } + + fn add_storage(&mut self, mode: AgentMode, units: u128) { + let pending = match mode { + AgentMode::Durable => &mut self.durable_storage_byte_seconds, + AgentMode::Ephemeral => &mut self.ephemeral_storage_byte_seconds, + }; + *pending = pending.saturating_add(units); + } + + fn add_memory_settlement(&mut self, mode: AgentMode, settlement: ByteTimeSettlement) { + self.memory_remainder = self.memory_remainder.saturating_add(settlement.remainder); + let remainder_units = self.memory_remainder / BYTE_NANOSECONDS_PER_GB_SECOND; + self.memory_remainder %= BYTE_NANOSECONDS_PER_GB_SECOND; + self.add_memory(mode, settlement.units.saturating_add(remainder_units)); + } + + fn add_storage_settlement(&mut self, mode: AgentMode, settlement: ByteTimeSettlement) { + let remainder = match mode { + AgentMode::Durable => &mut self.durable_storage_remainder, + AgentMode::Ephemeral => &mut self.ephemeral_storage_remainder, + }; + *remainder = remainder.saturating_add(settlement.remainder); + let remainder_units = *remainder / 1_000_000_000; + *remainder %= 1_000_000_000; + self.add_storage(mode, settlement.units.saturating_add(remainder_units)); + } + + fn is_active(&self) -> bool { + self.durable_memory_gb_seconds != 0 + || self.ephemeral_memory_gb_seconds != 0 + || self.durable_storage_byte_seconds != 0 + || self.ephemeral_storage_byte_seconds != 0 + } + + #[cfg(test)] + fn memory(&self, mode: AgentMode) -> u128 { + match mode { + AgentMode::Durable => self.durable_memory_gb_seconds, + AgentMode::Ephemeral => self.ephemeral_memory_gb_seconds, + } + } + + fn storage(&self, mode: AgentMode) -> u128 { + match mode { + AgentMode::Durable => self.durable_storage_byte_seconds, + AgentMode::Ephemeral => self.ephemeral_storage_byte_seconds, + } + } + + fn capture(&mut self) -> CapturedAccountUsage { + let durable_memory = take_bounded(&mut self.durable_memory_gb_seconds, i64::MAX as u128); + let ephemeral_memory = take_bounded( + &mut self.ephemeral_memory_gb_seconds, + i64::MAX as u128 - durable_memory, + ); + let durable_storage = + take_bounded(&mut self.durable_storage_byte_seconds, i64::MAX as u128); + let ephemeral_storage = + take_bounded(&mut self.ephemeral_storage_byte_seconds, i64::MAX as u128); + CapturedAccountUsage { + memory_gb_seconds: (durable_memory + ephemeral_memory) as i64, + durable_memory_gb_seconds: durable_memory as i64, + ephemeral_memory_gb_seconds: ephemeral_memory as i64, + durable_storage_byte_seconds: durable_storage as i64, + ephemeral_storage_byte_seconds: ephemeral_storage as i64, + } + } +} + +fn take_bounded(pending: &mut u128, maximum: u128) -> u128 { + let captured = (*pending).min(maximum); + *pending -= captured; + captured +} + impl AtomicResourceEntry { /// Sentinel value used in the database and service config to represent /// "unlimited" for the concurrent agents per executor limit. @@ -115,6 +221,8 @@ impl AtomicResourceEntry { /// Same 10^18 value — fits in i64 (TOML max), safe for SQLite REAL, /// consistent with other unlimited sentinels in this codebase. pub const UNLIMITED_OPLOG_WRITES_PER_SECOND: u64 = 1_000_000_000_000_000_000; + // XFS supports block sizes up to 64 KiB, so this remains exactly representable. + pub(crate) const EFFECTIVELY_UNLIMITED_DISK_SPACE: u64 = u64::MAX - u16::MAX as u64; pub fn new( fuel: u64, @@ -177,21 +285,16 @@ impl AtomicResourceEntry { fuel: AtomicU64::new(fuel), delta: AtomicI64::new(0), in_flight_delta: AtomicI64::new(0), - durable_byte_seconds_delta: AtomicI64::new(0), - ephemeral_byte_seconds_delta: AtomicI64::new(0), - storage_meters: Arc::new(scc::HashMap::new()), - memory_gb_seconds_delta: AtomicI64::new(0), in_flight_memory_gb_seconds_delta: AtomicI64::new(0), - durable_memory_gb_seconds_delta: AtomicI64::new(0), - ephemeral_memory_gb_seconds_delta: AtomicI64::new(0), in_flight_durable_memory_gb_seconds_delta: AtomicI64::new(0), in_flight_ephemeral_memory_gb_seconds_delta: AtomicI64::new(0), - memory_usage_transition: Mutex::new(()), - memory_remainder: Mutex::new(0), - memory_meters: Arc::new(scc::HashMap::new()), + account_usage_accumulator: Mutex::new(AccountUsageAccumulator::default()), + resource_billings: Arc::new(scc::HashMap::new()), max_memory: AtomicUsize::new(max_memory), max_table_elements: AtomicUsize::new(max_table_elements), max_disk_space: AtomicU64::new(max_disk_space), + filesystem_limit_update: AsyncMutex::new(()), + agent_filesystems: scc::HashMap::new(), last_refresh_secs: AtomicI64::new(Utc::now().timestamp()), per_invocation_http_call_limit: AtomicU64::new(per_invocation_http_call_limit), per_invocation_rpc_call_limit: AtomicU64::new(per_invocation_rpc_call_limit), @@ -301,73 +404,173 @@ impl AtomicResourceEntry { self.max_disk_space.load(Ordering::Acquire) } - pub(crate) fn register_storage_meter( + pub(crate) async fn register_agent_filesystem( &self, owned_agent_id: OwnedAgentId, - meter: AgentStorageMeter, - ) { - self.storage_meters.upsert_sync(owned_agent_id, meter); + runtime: AgentFilesystemRuntime, + ) -> Result<(), FilesystemStorageError> { + let _update = self.filesystem_limit_update.lock().await; + runtime + .set_allocated_byte_limit(AgentFilesystemStorageLimit { + allocated_bytes: self.max_disk_space_limit(), + }) + .await?; + self.agent_filesystems.upsert_sync(owned_agent_id, runtime); + Ok(()) } - pub(crate) fn unregister_storage_meter( + pub(crate) fn unregister_agent_filesystem( &self, owned_agent_id: &OwnedAgentId, - meter: &AgentStorageMeter, + runtime: &AgentFilesystemRuntime, ) { - self.storage_meters - .remove_if_sync(owned_agent_id, |registered| registered.is_same_meter(meter)); + self.agent_filesystems + .remove_if_sync(owned_agent_id, |registered| { + registered.is_same_runtime(runtime) + }); } - pub fn flush_storage_meters(&self, now: Instant) { - self.storage_meters.iter_sync(|_, meter| { - meter.flush(now); + #[doc(hidden)] + pub async fn apply_agent_filesystem_limit( + &self, + allocated_bytes: u64, + ) -> Result<(), (OwnedAgentId, FilesystemStorageError)> { + let _update = self.filesystem_limit_update.lock().await; + self.max_disk_space + .store(allocated_bytes, Ordering::Release); + let mut filesystems = Vec::new(); + self.agent_filesystems.iter_sync(|owned_agent_id, runtime| { + filesystems.push((owned_agent_id.clone(), runtime.clone())); true }); + let mut first_error = None; + for (owned_agent_id, runtime) in filesystems { + if let Err(error) = runtime + .set_allocated_byte_limit(AgentFilesystemStorageLimit { allocated_bytes }) + .await + && first_error.is_none() + { + first_error = Some((owned_agent_id, error)); + } + } + first_error.map_or(Ok(()), Err) } pub fn record_storage_byte_seconds(&self, mode: AgentMode, amount: i64) { - let delta = match mode { - AgentMode::Durable => &self.durable_byte_seconds_delta, - AgentMode::Ephemeral => &self.ephemeral_byte_seconds_delta, - }; - delta.fetch_add(amount, Ordering::Relaxed); + if amount > 0 { + self.account_usage_accumulator + .lock() + .unwrap() + .add_storage(mode, amount as u128); + } } - pub(crate) fn register_memory_meter( + pub(crate) fn record_resource_usage( + &self, + mode: AgentMode, + memory_gb_seconds: i64, + storage_byte_seconds: i64, + ) { + let mut accumulator = self.account_usage_accumulator.lock().unwrap(); + if memory_gb_seconds > 0 { + accumulator.add_memory(mode, memory_gb_seconds as u128); + } + if storage_byte_seconds > 0 { + accumulator.add_storage(mode, storage_byte_seconds as u128); + } + } + + pub(crate) fn record_resource_settlement( + &self, + mode: AgentMode, + memory: ByteTimeSettlement, + storage: ByteTimeSettlement, + ) { + let mut accumulator = self.account_usage_accumulator.lock().unwrap(); + accumulator.add_memory_settlement(mode, memory); + accumulator.add_storage_settlement(mode, storage); + } + + #[cfg(test)] + pub(crate) fn record_storage_remainder(&self, mode: AgentMode, remainder: u128) { + if remainder == 0 { + return; + } + self.account_usage_accumulator + .lock() + .unwrap() + .add_storage_settlement( + mode, + ByteTimeSettlement { + units: 0, + remainder, + }, + ); + } + + pub(crate) fn record_storage_settlement( + &self, + mode: AgentMode, + settlement: ByteTimeSettlement, + ) { + self.account_usage_accumulator + .lock() + .unwrap() + .add_storage_settlement(mode, settlement); + } + + pub(crate) fn register_resource_billing( &self, owned_agent_id: OwnedAgentId, - meter: AgentMemoryMeter, + billing: AgentResourceBilling, ) { - self.memory_meters.upsert_sync(owned_agent_id, meter); + self.resource_billings.upsert_sync(owned_agent_id, billing); } - pub(crate) fn unregister_memory_meter( + pub(crate) fn unregister_resource_billing( &self, owned_agent_id: &OwnedAgentId, - meter: &AgentMemoryMeter, + billing: &AgentResourceBilling, ) { - self.memory_meters - .remove_if_sync(owned_agent_id, |registered| registered.is_same_meter(meter)); + self.resource_billings + .remove_if_sync(owned_agent_id, |registered| { + registered.is_same_billing(billing) + }); } - fn flush_memory_meters(&self, now: Instant) { - self.memory_meters.iter_sync(|_, meter| { - meter.flush(now); + fn flush_resource_billings(&self, now: Instant) { + self.resource_billings.iter_sync(|_, billing| { + billing.flush(now); true }); } + /// Flushes resident resource meters and returns the local durable storage delta. + /// + /// This is test support for production-context executor tests that provide their own + /// `ResourceLimits` implementation and therefore observe this entry directly. It only + /// samples registered meters synchronously and never enqueues or wakes worker work. + #[doc(hidden)] + pub fn flush_durable_storage_byte_seconds_for_test(&self) -> i64 { + self.flush_resource_billings(Instant::now()); + self.account_usage_accumulator + .lock() + .unwrap() + .storage(AgentMode::Durable) + .min(i64::MAX as u128) as i64 + } + fn enforce_memory_limit(&self, limit: u64) { - self.memory_meters.iter_sync(|_, meter| { - meter.enforce_limit(limit); + self.resource_billings.iter_sync(|_, billing| { + billing.enforce_memory_limit(limit); true }); } fn capture_usage_update(&self, refresh_threshold_secs: i64) -> Option { - self.flush_memory_meters(Instant::now()); + self.flush_resource_billings(Instant::now()); let active = self.delta.load(Ordering::Acquire) != 0 - || self.memory_gb_seconds_delta.load(Ordering::Acquire) != 0 + || self.account_usage_accumulator.lock().unwrap().is_active() || self.unsynced_http_calls.load(Ordering::Acquire) > 0 || self.unsynced_rpc_calls.load(Ordering::Acquire) > 0; let stale = self.secs_since_last_refresh() >= refresh_threshold_secs; @@ -376,28 +579,15 @@ impl AtomicResourceEntry { return None; } - self.flush_storage_meters(Instant::now()); let fuel_delta = self.delta.swap(0, Ordering::AcqRel); - let ( - memory_gb_seconds_delta, - durable_memory_gb_seconds_delta, - ephemeral_memory_gb_seconds_delta, - ) = { - let _transition = self.memory_usage_transition.lock().unwrap(); - ( - self.memory_gb_seconds_delta.swap(0, Ordering::AcqRel), - self.durable_memory_gb_seconds_delta - .swap(0, Ordering::AcqRel), - self.ephemeral_memory_gb_seconds_delta - .swap(0, Ordering::AcqRel), - ) - }; + let captured_usage = self.account_usage_accumulator.lock().unwrap().capture(); + let memory_gb_seconds_delta = captured_usage.memory_gb_seconds; + let durable_memory_gb_seconds_delta = captured_usage.durable_memory_gb_seconds; + let ephemeral_memory_gb_seconds_delta = captured_usage.ephemeral_memory_gb_seconds; + let durable_storage_byte_seconds_delta = captured_usage.durable_storage_byte_seconds; + let ephemeral_storage_byte_seconds_delta = captured_usage.ephemeral_storage_byte_seconds; let http_count = self.unsynced_http_calls.swap(0, Ordering::AcqRel); let rpc_count = self.unsynced_rpc_calls.swap(0, Ordering::AcqRel); - let durable_storage_byte_seconds_delta = - self.durable_byte_seconds_delta.swap(0, Ordering::AcqRel); - let ephemeral_storage_byte_seconds_delta = - self.ephemeral_byte_seconds_delta.swap(0, Ordering::AcqRel); if http_count > 0 { self.syncing_http_calls .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { @@ -447,62 +637,46 @@ impl AtomicResourceEntry { } pub fn record_memory_gb_seconds(&self, mode: AgentMode, amount: i64) { - if amount == 0 { - return; + if amount > 0 { + self.account_usage_accumulator + .lock() + .unwrap() + .add_memory(mode, amount as u128); } - let _transition = self.memory_usage_transition.lock().unwrap(); - self.record_memory_gb_seconds_locked(mode, amount); - } - - fn record_memory_gb_seconds_locked(&self, mode: AgentMode, amount: i64) { - self.memory_gb_seconds_delta - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |delta| { - Some(delta.saturating_add(amount)) - }) - .ok(); - let mode_delta = self.memory_delta_for_mode(mode); - mode_delta - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |delta| { - Some(delta.saturating_add(amount)) - }) - .ok(); } - pub fn record_memory_remainder(&self, mode: AgentMode, remainder: u128) { - if remainder == 0 { - return; - } - let _transition = self.memory_usage_transition.lock().unwrap(); - let units = { - let mut account_remainder = self.memory_remainder.lock().unwrap(); - *account_remainder = account_remainder.saturating_add(remainder); - let units = *account_remainder / BYTE_NANOSECONDS_PER_GB_SECOND; - *account_remainder %= BYTE_NANOSECONDS_PER_GB_SECOND; - units.min(i64::MAX as u128) as i64 - }; - self.record_memory_gb_seconds_locked(mode, units); + pub(crate) fn record_memory_settlement(&self, mode: AgentMode, settlement: ByteTimeSettlement) { + self.account_usage_accumulator + .lock() + .unwrap() + .add_memory_settlement(mode, settlement); } #[cfg(test)] pub(crate) fn memory_gb_seconds_delta(&self, mode: AgentMode) -> i64 { - self.memory_delta_for_mode(mode).load(Ordering::Acquire) - } - - fn memory_delta_for_mode(&self, mode: AgentMode) -> &AtomicI64 { - match mode { - AgentMode::Durable => &self.durable_memory_gb_seconds_delta, - AgentMode::Ephemeral => &self.ephemeral_memory_gb_seconds_delta, - } + self.account_usage_accumulator + .lock() + .unwrap() + .memory(mode) + .min(i64::MAX as u128) as i64 } #[cfg(test)] pub(crate) fn durable_byte_seconds_delta(&self) -> i64 { - self.durable_byte_seconds_delta.load(Ordering::Acquire) + self.account_usage_accumulator + .lock() + .unwrap() + .storage(AgentMode::Durable) + .min(i64::MAX as u128) as i64 } #[cfg(test)] pub(crate) fn ephemeral_byte_seconds_delta(&self) -> i64 { - self.ephemeral_byte_seconds_delta.load(Ordering::Acquire) + self.account_usage_accumulator + .lock() + .unwrap() + .storage(AgentMode::Ephemeral) + .min(i64::MAX as u128) as i64 } /// Returns the number of HTTP calls remaining in this billing period from the @@ -668,12 +842,12 @@ impl ResourceLimitsGrpc { } /// Builds and sends a single batch to the registry covering: - /// - active accounts with non-zero fuel, HTTP call, or RPC call deltas - /// - stale accounts, including storage-only accounts, past the refresh threshold + /// - active accounts with non-zero fuel, memory, storage, HTTP, or RPC deltas + /// - otherwise-idle accounts past the refresh threshold /// /// On success, updates all entries via `update_last_known_limits`. On - /// failure, resets in-flight deltas for active accounts so they are not - /// double-counted next cycle; stale idle accounts are retried next tick. + /// failure, drops the captured batch under the accepted bounded-loss semantics and + /// resets in-flight quota tracking; stale idle accounts are retried next tick. async fn send_batch(&self, refresh_threshold_secs: i64) { async { let mut entries = Vec::new(); @@ -880,9 +1054,22 @@ impl ResourceLimitsGrpc { updated_limits.max_table_elements_per_worker as usize, Ordering::Release, ); - entry - .max_disk_space - .store(updated_limits.max_disk_space_per_worker, Ordering::Release); + let filesystem_limit_updated = match entry + .apply_agent_filesystem_limit(updated_limits.max_disk_space_per_worker) + .await + { + Ok(()) => true, + Err((owned_agent_id, error)) => { + error!( + account_id = %account_id, + agent_id = %owned_agent_id, + limit = updated_limits.max_disk_space_per_worker, + error = %error, + "Failed to apply managed agent filesystem limit" + ); + false + } + }; entry.per_invocation_http_call_limit.store( updated_limits.per_invocation_http_call_limit, Ordering::Release, @@ -906,9 +1093,11 @@ impl ResourceLimitsGrpc { entry .oplog_writes_per_second .store(updated_limits.oplog_writes_per_second, Ordering::Release); - entry - .last_refresh_secs - .store(Utc::now().timestamp(), Ordering::Release); + if filesystem_limit_updated { + entry + .last_refresh_secs + .store(Utc::now().timestamp(), Ordering::Release); + } } } @@ -980,7 +1169,7 @@ impl ResourceLimits for ResourceLimitsDisabled { u64::MAX, usize::MAX, usize::MAX, - u64::MAX, + AtomicResourceEntry::EFFECTIVELY_UNLIMITED_DISK_SPACE, AtomicResourceEntry::UNLIMITED_CONCURRENT_AGENTS, ))) } @@ -989,6 +1178,8 @@ impl ResourceLimits for ResourceLimitsDisabled { #[cfg(test)] mod tests { use super::*; + use crate::services::active_workers::MemoryGrant; + use crate::services::linear_memory::LinearMemoryTracker; use golem_common::model::AgentId; use golem_common::model::agent::{AgentTypeName, RegisteredAgentType, ResolvedAgentType}; use golem_common::model::application::{ApplicationId, ApplicationName}; @@ -1014,97 +1205,75 @@ mod tests { test_r::enable!(); #[test] - fn stale_storage_meter_unregister_preserves_reloaded_meter() { + fn fresh_tick_flushes_resource_billing_before_activity_check() { let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); let owned_agent_id = OwnedAgentId::new( EnvironmentId(Uuid::new_v4()), &AgentId { component_id: ComponentId(Uuid::new_v4()), - agent_id: "storage-meter-race".to_string(), + agent_id: "deferred-memory-meter-flush".to_string(), }, ); let now = Instant::now(); - let old = AgentStorageMeter::new(AgentMode::Durable, 1, entry.clone(), now); - let reloaded = AgentStorageMeter::new(AgentMode::Durable, 1, entry.clone(), now); - - entry.register_storage_meter(owned_agent_id.clone(), old.clone()); - entry.register_storage_meter(owned_agent_id.clone(), reloaded.clone()); - entry.unregister_storage_meter(&owned_agent_id, &old); + let memory = LinearMemoryTracker::new( + 1024 * 1024 * 1024, + 1024 * 1024 * 1024, + AgentMode::Durable, + false, + entry.clone(), + Arc::new(Mutex::new(MemoryGrant::inert(0))), + now, + ); + let billing = AgentResourceBilling::new(AgentMode::Durable, memory, entry.clone(), now); + entry.register_resource_billing(owned_agent_id, billing.clone()); + billing.open_for_test(None, now); + let close = billing.begin_close_for_test().unwrap(); + billing.close_for_test(close, None, now + Duration::from_secs(3)); - let registered = entry.storage_meters.get_sync(&owned_agent_id).unwrap(); - assert!(registered.get().is_same_meter(&reloaded)); + let captured = entry.capture_usage_update(i64::MAX).unwrap(); + assert_eq!(captured.update.memory_gb_seconds_delta, 3); } #[test] - fn flush_storage_meters_integrates_inline() { - let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); - let owned_agent_id = OwnedAgentId::new( - EnvironmentId(Uuid::new_v4()), - &AgentId { - component_id: ComponentId(Uuid::new_v4()), - agent_id: "inline-storage-meter-flush".to_string(), + fn account_usage_accumulator_emits_oversized_settlements_in_bounded_batches() { + let mut accumulator = AccountUsageAccumulator::default(); + let oversized = i64::MAX as u128 + 7; + accumulator.add_memory_settlement( + AgentMode::Durable, + ByteTimeSettlement { + units: oversized, + remainder: 0, }, ); - let now = Instant::now(); - let meter = AgentStorageMeter::new(AgentMode::Durable, 10, entry.clone(), now); - entry.register_storage_meter(owned_agent_id, meter); - - entry.flush_storage_meters(now + Duration::from_secs(3)); - - assert_eq!(entry.durable_byte_seconds_delta(), 30); - } - - /// `send_batch` only integrates the meters on ticks that actually ship an update, - /// so a meter routinely goes several ticks without being flushed. Time spanned by - /// those skipped ticks must still be billed, exactly once. - #[test] - fn skipped_flushes_are_billed_once_when_the_next_flush_happens() { - let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); - let owned_agent_id = OwnedAgentId::new( - EnvironmentId(Uuid::new_v4()), - &AgentId { - component_id: ComponentId(Uuid::new_v4()), - agent_id: "deferred-storage-meter-flush".to_string(), + accumulator.add_storage_settlement( + AgentMode::Durable, + ByteTimeSettlement { + units: oversized, + remainder: 0, }, ); - let now = Instant::now(); - let meter = AgentStorageMeter::new(AgentMode::Durable, 10, entry.clone(), now); - entry.register_storage_meter(owned_agent_id, meter); - - // A tick that ships integrates the first second. - entry.flush_storage_meters(now + Duration::from_secs(1)); - assert_eq!(entry.durable_byte_seconds_delta(), 10); - - // The next two ticks are skipped because the account is idle and its limits are - // still fresh. The following flush must bill both of those seconds, and neither - // re-bill the first nor drop anything. - entry.flush_storage_meters(now + Duration::from_secs(3)); - assert_eq!(entry.durable_byte_seconds_delta(), 30); - } - #[test] - fn fresh_tick_flushes_memory_meter_before_activity_check() { - let entry = Arc::new(AtomicResourceEntry::new(0, 0, 0, 0, 0)); - let owned_agent_id = OwnedAgentId::new( - EnvironmentId(Uuid::new_v4()), - &AgentId { - component_id: ComponentId(Uuid::new_v4()), - agent_id: "deferred-memory-meter-flush".to_string(), - }, + assert_eq!( + accumulator.capture(), + CapturedAccountUsage { + memory_gb_seconds: i64::MAX, + durable_memory_gb_seconds: i64::MAX, + ephemeral_memory_gb_seconds: 0, + durable_storage_byte_seconds: i64::MAX, + ephemeral_storage_byte_seconds: 0, + } ); - let now = Instant::now(); - let meter = AgentMemoryMeter::new( - AgentMode::Durable, - 1024 * 1024 * 1024, - true, - entry.clone(), - now, + assert_eq!( + accumulator.capture(), + CapturedAccountUsage { + memory_gb_seconds: 7, + durable_memory_gb_seconds: 7, + ephemeral_memory_gb_seconds: 0, + durable_storage_byte_seconds: 7, + ephemeral_storage_byte_seconds: 0, + } ); - entry.register_memory_meter(owned_agent_id, meter.clone()); - meter.pause(now + Duration::from_secs(3)); - - let captured = entry.capture_usage_update(i64::MAX).unwrap(); - assert_eq!(captured.update.memory_gb_seconds_delta, 3); + assert!(!accumulator.is_active()); } // ------------------------------------------------------------------------- @@ -2094,7 +2263,7 @@ mod tests { } #[test] - async fn send_batch_keeps_storage_only_delta_until_refresh_interval() { + async fn send_batch_treats_storage_only_delta_as_activity() { let mock = Arc::new(MockRegistryService::new(1000, 512)); let svc = make_grpc(mock.clone()); let id = account_id(); @@ -2103,8 +2272,29 @@ mod tests { svc.send_batch(NO_IDLE_REFRESH_THRESHOLD_SECS).await; - assert_eq!(entry.durable_byte_seconds_delta(), 100); - assert!(mock.last_batch_updates.lock().unwrap().is_empty()); + assert_eq!(entry.durable_byte_seconds_delta(), 0); + assert_eq!( + mock.last_batch_update(id) + .durable_storage_byte_seconds_delta, + 100 + ); + } + + #[test] + fn storage_remainders_do_not_cross_agent_modes() { + let entry = AtomicResourceEntry::new(0, 0, 0, 0, 0); + + entry.record_storage_remainder(AgentMode::Durable, 600_000_000); + entry.record_storage_remainder(AgentMode::Ephemeral, 600_000_000); + + assert_eq!(entry.durable_byte_seconds_delta(), 0); + assert_eq!(entry.ephemeral_byte_seconds_delta(), 0); + + entry.record_storage_remainder(AgentMode::Durable, 400_000_000); + entry.record_storage_remainder(AgentMode::Ephemeral, 400_000_000); + + assert_eq!(entry.durable_byte_seconds_delta(), 1); + assert_eq!(entry.ephemeral_byte_seconds_delta(), 1); } #[test] @@ -2299,6 +2489,8 @@ mod tests { assert_eq!(entry.in_flight_delta.load(Ordering::Acquire), 0); assert_eq!(entry.memory_gb_seconds_delta(AgentMode::Durable), 0); + assert_eq!(entry.durable_byte_seconds_delta(), 0); + assert_eq!(entry.ephemeral_byte_seconds_delta(), 0); assert_eq!( entry .in_flight_memory_gb_seconds_delta diff --git a/golem-worker-executor/src/worker/invocation_loop.rs b/golem-worker-executor/src/worker/invocation_loop.rs index 3ed46e9cdd..313518801b 100644 --- a/golem-worker-executor/src/worker/invocation_loop.rs +++ b/golem-worker-executor/src/worker/invocation_loop.rs @@ -13,18 +13,18 @@ // limitations under the License. use crate::model::{ReadFileResult, TrapType}; -use crate::services::events::Event; +use crate::services::agent_filesystem::AgentFilesystem; +use crate::services::agent_resource_billing::AgentResourceBilling; use crate::services::golem_config::SnapshotPolicy; -use crate::services::linear_memory::LinearMemoryTracker; use crate::services::oplog::{CommitLevel, EphemeralOplog, OplogOps}; -use crate::services::{HasEvents, HasOplog, HasWorker}; +use crate::services::{HasOplog, HasShardService, HasWorker}; use crate::worker::invocation::{ InvocationMode, InvokeResult, invoke_observed_and_traced, lower_invocation, }; use crate::worker::status_checkpointer; use crate::worker::{ - FinalWorkerState, PendingWorkerInterrupt, QueuedWorkerInvocation, RetryDecision, RunningWorker, - Worker, WorkerCommand, WorkerInterruptState, WorkerTrace, + CreateWorkerInstanceError, FinalWorkerState, PendingWorkerInterrupt, QueuedWorkerInvocation, + RetryDecision, RunningWorker, Worker, WorkerCommand, WorkerInterruptState, WorkerTrace, }; use crate::workerctx::{PublicWorkerIo, UpdateManagement, WorkerCtx}; use anyhow::anyhow; @@ -51,14 +51,16 @@ use golem_common::model::agent::structural_format::format_structural_typed; use golem_common::related_span; use golem_common::tracing::TraceOrigin; use std::collections::VecDeque; +use std::future::Future; use std::ops::DerefMut; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::Duration; use tokio::sync::RwLock; use tokio::sync::mpsc::UnboundedReceiver; use tokio::task::JoinHandle; use tracing::{Instrument, Level, debug, error, span, warn}; +use uuid::Uuid; use wasmtime::Store; use wasmtime::component::Instance; @@ -92,29 +94,51 @@ pub struct InvocationLoop { pub oom_retry_count: u32, /// Concurrent-agent permit owned by this invocation loop task. Released /// (set to `None`) when the agent goes idle, re-acquired when it wakes up. - /// Only actively running agents hold a permit. Dropped automatically when - /// the task is aborted (e.g. `RunningWorker::stop()`). - pub concurrent_agent_permit: Option, + /// Only actively running agents hold a permit. Normal stops close the command + /// channel and await cooperative loop exit; this field's drop is only a fallback + /// for task cancellation or panic. + pub(super) permit_state: + ConcurrentAgentPermitState, + pub idle_since_millis: Arc, /// `ResumeReplay` is not represented in the internal queue, so we track it /// explicitly to avoid evicting a worker that is blocked waking up for it. pub resume_replay_pending: Arc, + pub start_attempt: Uuid, /// What this worker's phase spans link back to, and the fields they carry. pub worker_trace: WorkerTrace, } +impl Drop for InvocationLoop { + fn drop(&mut self) { + self.permit_state.release(); + } +} + /// Outcome of creating the worker instance for one iteration of the invocation loop. enum CreateInstanceResult { Created { instance: Instance, store: Mutex>, + filesystem: Box, }, - /// Wasm executing during instantiation trapped with an [`InterruptKind`] (e.g. a fuel - /// suspension from the epoch deadline callback). The worker itself was created successfully. + /// Instance creation was interrupted by a recoverable condition, such as fuel or filesystem + /// quota exhaustion. The worker metadata remains valid and queued work must be preserved. Interrupted(InterruptKind), /// Instance creation failed; the worker was already stopped with the startup failure. Failed, } +async fn settle_reconstructed_filesystem( + filesystem: &AgentFilesystem, +) -> Result<(), WorkerExecutorError> { + filesystem + .settle_reconstruction() + .await + .map_err(|error| WorkerExecutorError::runtime(error.to_string()))?; + debug!("Agent filesystem reconstruction settled"); + Ok(()) +} + impl InvocationLoop { async fn pending_interrupt(&self) -> Option<(InterruptKind, RetryDecision)> { take_pending_interrupt(&self.interrupt_signal) @@ -137,9 +161,23 @@ impl InvocationLoop { let mut deferred_wakeups = VecDeque::new(); 'outer: loop { - let (instance, store) = match self.create_instance().await { - CreateInstanceResult::Created { instance, store } => (instance, store), + self.release_terminal_interrupt().await; + if let Err(error) = self.parent.shard_service().check_worker(&agent_id) { + debug!(%agent_id, "Worker generation not started because its shard is not assigned"); + self.parent.complete_startup(self.start_attempt, Err(error)); + self.release_concurrent_agent_permit(); + self.stop_unloaded(None).await; + break; + } + self.acquire_concurrent_agent_permit().await; + let (instance, store, filesystem) = match self.create_instance().await { + CreateInstanceResult::Created { + instance, + store, + filesystem, + } => (instance, store, *filesystem), CreateInstanceResult::Interrupted(kind) => { + self.release_concurrent_agent_permit(); let pending_interrupt = take_pending_interrupt(&self.interrupt_signal).await; let kind = pending_interrupt .map(|interrupt| interrupt.kind) @@ -163,6 +201,7 @@ impl InvocationLoop { ); continue; } else { + self.parent.complete_startup(self.start_attempt, Ok(())); self.stop_unloaded(None).await; break; } @@ -171,6 +210,10 @@ impl InvocationLoop { self.parent .add_and_commit_oplog(OplogEntry::interrupted()) .await; + self.parent.complete_startup( + self.start_attempt, + Err(WorkerExecutorError::Interrupted { kind }), + ); self.stop_unloaded(None).await; break; } @@ -178,15 +221,50 @@ impl InvocationLoop { } CreateInstanceResult::Failed => { // early return, can't retry a failed instance creation + self.release_concurrent_agent_permit(); break; } }; + let resource_billing = store.lock().await.data().durable_ctx().resource_billing(); + if let Err(error) = resource_billing.open(&filesystem.runtime()).await { + let error = WorkerExecutorError::runtime(format!( + "Failed to open worker resource billing window: {error}" + )); + self.parent + .complete_startup(self.start_attempt, Err(error.clone())); + self.release_concurrent_agent_permit(); + filesystem.seal(); + drop(store); + let cleanup_result = filesystem.close_and_delete().await; + match cleanup_result { + Ok(()) => self.stop_unloaded(Some(error)).await, + Err(cleanup) => { + self.stop_cleanup_failed(WorkerExecutorError::runtime(format!( + "{error}; {cleanup}" + ))) + .await; + } + } + break; + } - let mut final_decision = self.recover_instance_state(&instance, &store).await; + let (mut final_decision, recovery_failure) = match self + .recover_instance_state(&instance, &store, &filesystem) + .await + { + Ok(decision) => (decision, None), + Err(error) => { + self.parent + .complete_startup(self.start_attempt, Err(error.clone())); + (Some(RetryDecision::None), Some(error)) + } + }; let mut final_interrupt = None; let mut cleanup_ephemeral_worker = false; - if let Some((kind, decision)) = self.pending_interrupt().await { + if recovery_failure.is_none() + && let Some((kind, decision)) = self.pending_interrupt().await + { debug!( %agent_id, ?decision, @@ -198,13 +276,17 @@ impl InvocationLoop { final_decision = Some(decision); } - if final_decision.is_none() { - let linear_memory = store - .lock() + if final_decision.is_none() + && !self + .parent + .complete_startup_success(self.start_attempt) .await - .data() - .durable_ctx() - .linear_memory_tracker(); + { + final_decision = Some(RetryDecision::None); + } + + if final_decision.is_none() { + let resource_billing = store.lock().await.data().durable_ctx().resource_billing(); let mut inner_loop = InnerInvocationLoop { receiver: &mut self.receiver, active: self.active.clone(), @@ -214,10 +296,12 @@ impl InvocationLoop { interrupt_signal: self.interrupt_signal.clone(), instance: &instance, store: &store, - linear_memory, + filesystem: &filesystem, + resource_billing, invocations_since_snapshot: 0, idle_snapshot_task: None, - concurrent_agent_permit: &mut self.concurrent_agent_permit, + permit_state: &mut self.permit_state, + idle_since_millis: self.idle_since_millis.clone(), resume_replay_pending: self.resume_replay_pending.clone(), worker_trace: self.worker_trace.clone(), deferred_wakeups: &mut deferred_wakeups, @@ -235,20 +319,36 @@ impl InvocationLoop { cleanup_ephemeral_worker = result.cleanup_ephemeral_worker; } + let resource_close_error = self + .close_resource_window_and_release(&store, &filesystem) + .await + .err(); + if let Some(error) = resource_close_error.as_ref() { + warn!(error = %error, "Resource-window close failed; reconstructing or unloading the worker"); + } + final_decision = + decision_after_resource_close(final_decision, resource_close_error.is_some()); + self.suspend_worker(&store).await; if let Some(kind) = final_interrupt { self.record_retry_interrupt_failure(&store, kind).await; } + let mut runtime = Some((instance, store, filesystem)); + match final_decision { None | Some(RetryDecision::None) => { + let cleanup_error = Self::close_runtime(&mut runtime).await; debug!( %agent_id, "Invocation queue loop notifying parent about being stopped" ); - self.stop_unloaded( - cleanup_ephemeral_worker.then(super::inactive_ephemeral_agent_error), + self.stop_closed( + cleanup_error, + recovery_failure.or(resource_close_error).or_else(|| { + cleanup_ephemeral_worker.then(super::inactive_ephemeral_agent_error) + }), ) .await; if cleanup_ephemeral_worker { @@ -259,25 +359,35 @@ impl InvocationLoop { } Some(RetryDecision::TryStop(ts)) => { if ts < *self.parent.last_resume_request.lock().await { + if let Some(error) = Self::close_runtime(&mut runtime).await { + self.stop_cleanup_failed(error).await; + break; + } debug!( %agent_id, "Suspend request ignored because there was a resume request since it" ); continue; } else { + let cleanup_error = Self::close_runtime(&mut runtime).await; debug!( %agent_id, "Invocation queue loop notifying parent about being stopped" ); - self.stop_unloaded(None).await; + self.stop_closed(cleanup_error, resource_close_error).await; break; } } Some(RetryDecision::Immediate) => { + if let Some(error) = Self::close_runtime(&mut runtime).await { + self.stop_cleanup_failed(error).await; + break; + } debug!(%agent_id, "Invocation queue loop triggering restart immediately"); continue; } Some(RetryDecision::Delayed(delay)) => { + debug_assert!(resource_close_error.is_none()); debug!( %agent_id, ?delay, @@ -288,6 +398,10 @@ impl InvocationLoop { loop { tokio::select! { _ = &mut sleep => { + if let Some(error) = Self::close_runtime(&mut runtime).await { + self.stop_cleanup_failed(error).await; + break 'outer; + } debug!(%agent_id, "Invocation queue loop restarting after delay"); continue 'outer; } @@ -295,8 +409,9 @@ impl InvocationLoop { let command = match command { Some(command) => command, None => { + let cleanup_error = Self::close_runtime(&mut runtime).await; debug!(%agent_id, "Invocation queue loop command channel closed during delayed retry"); - self.stop_unloaded(None).await; + self.stop_closed(cleanup_error, None).await; break 'outer; } }; @@ -304,19 +419,38 @@ impl InvocationLoop { if let Some((kind, decision)) = self.pending_interrupt().await { debug!(%agent_id, ?decision, "Invocation queue loop interrupted during delayed retry"); if !matches!(kind, InterruptKind::Restart | InterruptKind::Jump) { - self.record_retry_interrupt_failure(&store, kind).await; + let store = &runtime.as_ref().expect("runtime must be open").1; + self.record_retry_interrupt_failure(store, kind).await; } match decision { RetryDecision::Immediate => { + if let Some(error) = Self::close_runtime(&mut runtime).await { + self.stop_cleanup_failed(error).await; + break 'outer; + } Self::defer_wakeup(&mut deferred_wakeups, command); continue 'outer; } RetryDecision::None => { - self.stop_unloaded(None).await; + let cleanup_error = Self::close_runtime(&mut runtime).await; + self.stop_closed(cleanup_error, None).await; break 'outer; } - RetryDecision::Delayed(_) | RetryDecision::TryStop(_) | RetryDecision::ReacquirePermits => { - unreachable!("interrupt decisions are only immediate or none") + RetryDecision::TryStop(timestamp) => { + let cleanup_error = Self::close_runtime(&mut runtime).await; + if timestamp < *self.parent.last_resume_request.lock().await { + if let Some(error) = cleanup_error { + self.stop_cleanup_failed(error).await; + break 'outer; + } + Self::defer_wakeup(&mut deferred_wakeups, command); + continue 'outer; + } + self.stop_closed(cleanup_error, None).await; + break 'outer; + } + RetryDecision::Delayed(_) | RetryDecision::ReacquirePermits => { + unreachable!("queued interrupts do not delay or reacquire permits") } } } @@ -328,11 +462,19 @@ impl InvocationLoop { } WorkerCommand::WorkAvailable => { debug!(%agent_id, "Invocation queue loop woke up during delayed retry"); + if let Some(error) = Self::close_runtime(&mut runtime).await { + self.stop_cleanup_failed(error).await; + break 'outer; + } Self::defer_wakeup(&mut deferred_wakeups, WorkerCommand::WorkAvailable); continue 'outer; } WorkerCommand::ResumeReplay => { debug!(%agent_id, "Invocation queue loop woke up for resume replay during delayed retry"); + if let Some(error) = Self::close_runtime(&mut runtime).await { + self.stop_cleanup_failed(error).await; + break 'outer; + } Self::defer_wakeup(&mut deferred_wakeups, WorkerCommand::ResumeReplay); continue 'outer; } @@ -342,31 +484,137 @@ impl InvocationLoop { } } Some(RetryDecision::ReacquirePermits) => { + if let Some(error) = Self::close_runtime(&mut runtime).await { + self.stop_cleanup_failed(error).await; + break; + } let delay = get_delay(self.parent.oom_retry_config(), self.oom_retry_count); debug!( %agent_id, ?delay, "Invocation queue loop dropping memory permits and triggering restart" ); - let _ = Worker::restart_on_oom( + let pending_startup_attempt = self.parent.pending_startup_attempt(); + if let Err(error) = Worker::restart_on_oom( self.parent.clone(), true, delay, self.oom_retry_count + 1, + pending_startup_attempt, ) - .await; + .await + { + warn!("Failed to restart worker after releasing memory permits: {error}"); + } break; } } } + self.release_terminal_interrupt().await; + } + + async fn release_terminal_interrupt(&self) { + self.parent.filesystem_limit_interrupt.lock().await.take(); + self.interrupt_signal.lock().await.release_terminal_claim(); + } + + async fn acquire_concurrent_agent_permit(&mut self) { + if self.permit_state.is_none() { + let agent_id = self.owned_agent_id.agent_id(); + let permit = self + .parent + .registered_concurrent_account + .acquire(agent_id) + .instrument(agent_phase_span!(self, "acquire_concurrent_agent_permit")) + .await; + self.permit_state.install(permit); + } + } + + fn release_concurrent_agent_permit(&mut self) { + self.permit_state.release(); + } + + async fn close_resource_window_and_release( + &mut self, + store: &Mutex>, + filesystem: &AgentFilesystem, + ) -> Result<(), WorkerExecutorError> { + filesystem.seal(); + let resource_billing = store.lock().await.data().durable_ctx().resource_billing(); + if resource_billing.is_active() { + self.permit_state + .close_then_release(async { + resource_billing + .close(&filesystem.runtime()) + .await + .map_err(|error| { + WorkerExecutorError::runtime(format!( + "Failed to close worker resource billing window: {error}" + )) + }) + }) + .await + } else { + self.release_concurrent_agent_permit(); + Ok(()) + } } async fn stop_unloaded(&self, startup_failure: Option) { + self.parent.complete_startup( + self.start_attempt, + Err(startup_failure.clone().unwrap_or_else(|| { + WorkerExecutorError::unknown("Worker stopped before startup completed") + })), + ); + let pending_failure = startup_failure.clone(); + self.parent + .stop_internal( + true, + pending_failure, + FinalWorkerState::Unloaded { startup_failure }, + ) + .await; + } + + async fn stop_cleanup_failed(&self, error: WorkerExecutorError) { + self.parent + .complete_startup(self.start_attempt, Err(error.clone())); + let pending_failure = error.clone(); self.parent - .stop_internal(true, None, FinalWorkerState::Unloaded { startup_failure }) + .stop_internal( + true, + Some(pending_failure), + FinalWorkerState::CleanupFailed(error), + ) .await; } + async fn stop_closed( + &self, + cleanup_error: Option, + startup_failure: Option, + ) { + match cleanup_error { + Some(error) => self.stop_cleanup_failed(error).await, + None => self.stop_unloaded(startup_failure).await, + } + } + + async fn close_runtime( + runtime: &mut Option<(Instance, Mutex>, AgentFilesystem)>, + ) -> Option { + let (_instance, store, filesystem) = runtime.take()?; + let resource_billing = store.lock().await.data().durable_ctx().resource_billing(); + debug_assert!(!resource_billing.is_active()); + drop(store); + filesystem.close_and_delete().await.err().map(|error| { + error!(error = %error, "Failed to delete agent runtime filesystem"); + WorkerExecutorError::runtime(error.to_string()) + }) + } + fn archive_ephemeral_oplog(&self) { let oplog = self.parent.oplog.clone(); tokio::spawn(async move { @@ -404,40 +652,37 @@ impl InvocationLoop { async { debug!("Creating the worker instance"); match RunningWorker::create_instance(self.parent.clone()).await { - Ok((instance, store)) => { - self.parent.events().publish(Event::WorkerLoaded { - agent_id: self.owned_agent_id.agent_id(), - result: Ok(()), - }); - CreateInstanceResult::Created { instance, store } - } - // Wasm executing during instantiation was interrupted (e.g. suspended by the fuel - // check in the epoch deadline callback). The worker exists — its metadata and - // `Create` oplog entry are already persisted — so this is not a creation failure: - // creation waiters are released successfully and the caller parks or restarts the - // worker like any other interrupt. - Err(WorkerExecutorError::Interrupted { kind }) => { + Ok((instance, store, filesystem)) => CreateInstanceResult::Created { + instance, + store, + filesystem: Box::new(filesystem), + }, + // Instance creation was interrupted by a recoverable condition. The worker exists + // and its metadata and `Create` oplog entry are already persisted, so the caller + // parks or restarts the worker without exposing an unprepared runtime. + Err(CreateWorkerInstanceError { + error: WorkerExecutorError::Interrupted { kind }, + filesystem_cleanup_failed: false, + }) => { debug!("Worker instantiation interrupted: {kind:?}"); - self.parent.events().publish(Event::WorkerLoaded { - agent_id: self.owned_agent_id.agent_id(), - result: Ok(()), - }); CreateInstanceResult::Interrupted(kind) } - Err(err) => { + Err(CreateWorkerInstanceError { + error: err, + filesystem_cleanup_failed, + }) => { warn!("Failed to start the worker: {err}"); - self.parent.events().publish(Event::WorkerLoaded { - agent_id: self.owned_agent_id.agent_id(), - result: Err(err.clone()), - }); self.parent - .stop_internal( - true, - Some(err.clone()), - FinalWorkerState::Unloaded { - startup_failure: Some(err), - }, - ) + .complete_startup(self.start_attempt, Err(err.clone())); + let final_state = if filesystem_cleanup_failed { + FinalWorkerState::CleanupFailed(err.clone()) + } else { + FinalWorkerState::Unloaded { + startup_failure: Some(err.clone()), + } + }; + self.parent + .stop_internal(true, Some(err), final_state) .await; CreateInstanceResult::Failed } @@ -454,7 +699,8 @@ impl InvocationLoop { &self, instance: &Instance, store: &Mutex>, - ) -> Option { + filesystem: &AgentFilesystem, + ) -> Result, WorkerExecutorError> { async { debug!("Preparing the worker instance"); let mut store = store.lock().await; @@ -474,23 +720,16 @@ impl InvocationLoop { match prepare_result { Ok(decision) => { + if decision.is_none() { + settle_reconstructed_filesystem(filesystem).await?; + } debug!("Recovery decision from prepare_instance: {decision:?}"); - decision + Ok(decision) } Err(err) => { warn!("Failed to start the worker: {err}"); store.data().set_suspended(); - - self.parent - .stop_internal( - true, - Some(err.clone()), - FinalWorkerState::Unloaded { - startup_failure: Some(err), - }, - ) - .await; - Some(RetryDecision::None) // early return, we can't retry this + Err(err) } } } @@ -536,13 +775,16 @@ struct InnerInvocationLoop<'a, Ctx: WorkerCtx> { interrupt_signal: Arc>, instance: &'a Instance, store: &'a Mutex>, - linear_memory: LinearMemoryTracker, + filesystem: &'a AgentFilesystem, + resource_billing: AgentResourceBilling, invocations_since_snapshot: u64, idle_snapshot_task: Option>, /// Mutable reference to the concurrent-agent permit held by the outer /// `InvocationLoop`. Set to `None` when entering idle (releasing the /// permit back to the semaphore pool) and re-acquired on wake. - concurrent_agent_permit: &'a mut Option, + permit_state: + &'a mut ConcurrentAgentPermitState, + idle_since_millis: Arc, resume_replay_pending: Arc, deferred_wakeups: &'a mut VecDeque, /// What this worker's phase spans link back to, and the fields they carry. @@ -575,13 +817,31 @@ impl InnerInvocationLoop<'_, Ctx> { // Entering idle: release the concurrent-agent permit so other agents // from the same account can start without evicting this one. self.check_no_active_tail_work_on_idle().await; - self.release_concurrent_agent_permit(); + if let Err(error) = self.release_concurrent_agent_permit().await { + error!(error = %error, "Failed to close worker resource billing window"); + return InnerInvocationLoopResult { + retry_decision: Some(RetryDecision::Immediate), + final_interrupt: None, + cleanup_ephemeral_worker: false, + }; + } + mark_idle(&self.idle_since_millis); self.waiting_for_command.store(true, Ordering::Release); while let Some(cmd) = self.next_wakeup_or_initial().await { + if matches!(cmd, WorkerCommand::InternalStatusChanged) + && !self.internal_status_change_requires_permit().await + { + continue; + } + // Waking from idle: re-acquire the concurrent-agent permit before // processing any commands. - self.acquire_concurrent_agent_permit().await; self.waiting_for_command.store(false, Ordering::Release); + if let Err(error) = self.acquire_concurrent_agent_permit().await { + error!(error = %error, "Failed to open worker resource billing window"); + final_decision = Some(RetryDecision::Immediate); + break; + } let outcome = match cmd { WorkerCommand::WorkAvailable | WorkerCommand::InternalStatusChanged => { loop { @@ -644,7 +904,12 @@ impl InnerInvocationLoop<'_, Ctx> { // Returning to idle: release the concurrent-agent permit. self.check_no_active_tail_work_on_idle().await; - self.release_concurrent_agent_permit(); + if let Err(error) = self.release_concurrent_agent_permit().await { + error!(error = %error, "Failed to close worker resource billing window"); + final_decision = Some(RetryDecision::Immediate); + break; + } + mark_idle(&self.idle_since_millis); self.waiting_for_command.store(true, Ordering::Release); } self.abort_idle_snapshot_task(); @@ -659,6 +924,22 @@ impl InnerInvocationLoop<'_, Ctx> { } } + async fn internal_status_change_requires_permit(&self) -> bool { + if !self.active.read().await.is_empty() + || self.interrupt_signal.lock().await.has_interrupt() + { + return true; + } + + let status = self.parent.get_non_detached_last_known_status().await; + !status.pending_updates.is_empty() + || !status.pending_invocations.is_empty() + || !matches!( + self.periodic_snapshot_action(&status), + PeriodicSnapshotAction::NotNeeded + ) + } + async fn next_wakeup_or_initial(&mut self) -> Option { match self.deferred_wakeups.pop_front() { Some(command) => Some(command), @@ -696,20 +977,30 @@ impl InnerInvocationLoop<'_, Ctx> { /// Release the concurrent-agent permit back to the semaphore pool. /// Called when the agent enters idle state. No-op if already released. - fn release_concurrent_agent_permit(&mut self) { - if let Some(permit) = self.concurrent_agent_permit.take() { - self.linear_memory.pause(std::time::Instant::now()); + async fn release_concurrent_agent_permit(&mut self) -> Result<(), WorkerExecutorError> { + if self.permit_state.is_some() { debug!(agent_id = %self.owned_agent_id.agent_id, "Releasing concurrent-agent permit (entering idle)"); - drop(permit); + let runtime = self.filesystem.runtime(); + let result = self + .permit_state + .close_then_release(async { + self.resource_billing + .close(&runtime) + .await + .map_err(|error| WorkerExecutorError::runtime(error.to_string())) + }) + .await; + return result; } + Ok(()) } /// Re-acquire the concurrent-agent permit from the scheduler. /// Called when the agent wakes from idle to process a command. /// The scheduler ensures FIFO ordering within the account so that a worker /// that just finished goes to the back of the queue. - async fn acquire_concurrent_agent_permit(&mut self) { - if self.concurrent_agent_permit.is_none() { + async fn acquire_concurrent_agent_permit(&mut self) -> Result<(), WorkerExecutorError> { + if self.permit_state.is_none() { let span = agent_phase_span!(self, "acquire_concurrent_agent_permit"); let agent_id = self.owned_agent_id.agent_id(); let registered_concurrent_account = self.parent.registered_concurrent_account.clone(); @@ -719,9 +1010,13 @@ impl InnerInvocationLoop<'_, Ctx> { } .instrument(span) .await; - *self.concurrent_agent_permit = Some(permit); - self.linear_memory.resume(std::time::Instant::now()); + self.resource_billing + .open(&self.filesystem.runtime()) + .await + .map_err(|error| WorkerExecutorError::runtime(error.to_string()))?; + self.permit_state.install(permit); } + Ok(()) } async fn next_wakeup(&mut self) -> Option { @@ -947,7 +1242,13 @@ impl InnerInvocationLoop<'_, Ctx> { async { let mut store = self.store.lock().await; - let resume_replay_result = Ctx::resume_replay(&mut *store, self.instance, true).await; + let resume_replay_result = + match Ctx::resume_replay(&mut *store, self.instance, true).await { + Ok(None) => settle_reconstructed_filesystem(self.filesystem) + .await + .map(|()| None), + other => other, + }; match resume_replay_result { Ok(None) => CommandOutcome::Continue, @@ -996,6 +1297,73 @@ impl InnerInvocationLoop<'_, Ctx> { } } +pub(super) struct ConcurrentAgentPermitState { + permit: Option, + held: Arc, +} + +impl ConcurrentAgentPermitState { + pub(super) fn new(permit: Option, held: Arc) -> Self { + held.store(permit.is_some(), Ordering::Release); + Self { permit, held } + } + + fn is_some(&self) -> bool { + self.permit.is_some() + } + + fn is_none(&self) -> bool { + self.permit.is_none() + } + + fn install(&mut self, permit: T) { + debug_assert!(self.permit.is_none()); + self.permit = Some(permit); + self.held.store(true, Ordering::Release); + } + + fn release(&mut self) { + if let Some(permit) = self.permit.take() { + drop(permit); + self.held.store(false, Ordering::Release); + } else { + debug_assert!(!self.held.load(Ordering::Acquire)); + } + } + + async fn close_then_release( + &mut self, + close: impl Future>, + ) -> Result<(), E> { + let result = close.await; + self.release(); + result + } +} + +fn decision_after_resource_close( + decision: Option, + close_failed: bool, +) -> Option { + if close_failed + && matches!( + decision, + Some(RetryDecision::Immediate | RetryDecision::Delayed(_)) + ) + { + Some(RetryDecision::Immediate) + } else { + decision + } +} + +fn mark_idle(idle_since_millis: &AtomicU64) { + let now = Timestamp::now_utc().to_millis(); + let _ = idle_since_millis.fetch_update(Ordering::Release, Ordering::Acquire, |previous| { + Some(now.max(previous.saturating_add(1))) + }); +} + async fn take_pending_interrupt( signal: &Mutex, ) -> Option { @@ -1168,6 +1536,13 @@ impl Invocation<'_, Ctx> { let component_metadata = self.store.data().component_metadata().metadata.clone(); + let invocation_for_lowering = invocation.clone(); + let lowered = lower_invocation( + invocation_for_lowering, + &component_metadata, + self.parent.parsed_agent_id.as_ref(), + )?; + Self::extend_invocation_context( &mut invocation_context, &idempotency_key, @@ -1191,13 +1566,6 @@ impl Invocation<'_, Ctx> { .await; } - let invocation_for_lowering = invocation.clone(); - let lowered = lower_invocation( - invocation_for_lowering, - &component_metadata, - self.parent.parsed_agent_id.as_ref(), - )?; - Ok::<_, WorkerExecutorError>((lowered, local_span_ids, inherited_span_ids)) } .instrument(span!(Level::INFO, "prepare_invocation_context")) @@ -1294,6 +1662,13 @@ impl Invocation<'_, Ctx> { self.parent.agent_mode(), )), }; + let invalid_request = matches!( + &trap_type, + Some(TrapType::Error { + error: AgentError::InvalidRequest(_), + .. + }) + ); let decision = match trap_type { Some(trap_type) => { self.store @@ -1304,7 +1679,11 @@ impl Invocation<'_, Ctx> { None => RetryDecision::None, }; - failed_agent_invocation_outcome(self.parent.agent_mode(), decision) + if invalid_request && self.parent.agent_mode() == AgentMode::Durable { + CommandOutcome::Continue + } else { + failed_agent_invocation_outcome(self.parent.agent_mode(), decision) + } } /// Try to perform the save-snapshot step of a manual update on the worker @@ -1828,7 +2207,8 @@ fn snapshot_action_at( #[cfg(test)] mod tests { use super::{ - CommandOutcome, PeriodicSnapshotAction, failed_agent_invocation_outcome, + CommandOutcome, ConcurrentAgentPermitState, PeriodicSnapshotAction, + decision_after_resource_close, failed_agent_invocation_outcome, periodic_snapshot_failure_outcome, snapshot_action_at, snapshot_baseline_timestamp, successful_agent_invocation_outcome, }; @@ -1839,9 +2219,97 @@ mod tests { use golem_common::model::oplog::AgentError; use golem_common::model::{OplogIndex, Timestamp}; use golem_service_base::error::worker_executor::WorkerExecutorError; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use test_r::test; + struct TestPermit { + held: Arc, + drops: Arc, + held_while_dropping: Arc, + } + + impl Drop for TestPermit { + fn drop(&mut self) { + self.held_while_dropping + .store(self.held.load(Ordering::Acquire), Ordering::Release); + self.drops.fetch_add(1, Ordering::AcqRel); + } + } + + #[test] + fn permit_state_stays_conservative_through_release_and_reacquisition() { + let held = Arc::new(AtomicBool::new(false)); + let drops = Arc::new(AtomicUsize::new(0)); + let held_while_dropping = Arc::new(AtomicBool::new(false)); + let mut state = ConcurrentAgentPermitState::new(None, held.clone()); + + state.install(TestPermit { + held: held.clone(), + drops: drops.clone(), + held_while_dropping: held_while_dropping.clone(), + }); + assert!(held.load(Ordering::Acquire)); + assert!(state.is_some()); + + state.release(); + assert!(state.is_none()); + assert!(!held.load(Ordering::Acquire)); + assert!(held_while_dropping.load(Ordering::Acquire)); + assert_eq!(drops.load(Ordering::Acquire), 1); + + state.install(TestPermit { + held: held.clone(), + drops: drops.clone(), + held_while_dropping, + }); + assert!(held.load(Ordering::Acquire)); + state.release(); + assert_eq!(drops.load(Ordering::Acquire), 2); + } + + #[test] + fn close_failure_skips_delayed_sleep_and_reconstructs_immediately() { + assert_eq!( + decision_after_resource_close( + Some(RetryDecision::Delayed(Duration::from_secs(30))), + true, + ), + Some(RetryDecision::Immediate) + ); + assert_eq!( + decision_after_resource_close(Some(RetryDecision::Immediate), true), + Some(RetryDecision::Immediate) + ); + let timestamp = Timestamp::from(1_000); + assert_eq!( + decision_after_resource_close(Some(RetryDecision::TryStop(timestamp)), true), + Some(RetryDecision::TryStop(timestamp)) + ); + } + + #[test] + async fn close_failure_still_releases_the_permit() { + let held = Arc::new(AtomicBool::new(false)); + let drops = Arc::new(AtomicUsize::new(0)); + let mut state = ConcurrentAgentPermitState::new(None, held.clone()); + state.install(TestPermit { + held: held.clone(), + drops: drops.clone(), + held_while_dropping: Arc::new(AtomicBool::new(false)), + }); + + let result = state + .close_then_release(async { Err::<(), _>("authoritative usage failed") }) + .await; + + assert_eq!(result, Err("authoritative usage failed")); + assert!(state.is_none()); + assert!(!held.load(Ordering::Acquire)); + assert_eq!(drops.load(Ordering::Acquire), 1); + } + #[test] fn periodic_snapshot_uses_creation_time_until_the_first_snapshot() { let created_at = Timestamp::from(1_000); diff --git a/golem-worker-executor/src/worker/mod.rs b/golem-worker-executor/src/worker/mod.rs index 59f484158d..27121cfd3e 100644 --- a/golem-worker-executor/src/worker/mod.rs +++ b/golem-worker-executor/src/worker/mod.rs @@ -26,11 +26,10 @@ use self::agent_config::{ ensure_required_agent_secrets_are_configured, parse_worker_creation_agent_config, }; use crate::durable_host::{agent_effective_surface_from_component_metadata, recover_stderr_logs}; -use crate::metrics::storage::record_filesystem_pool_released; use crate::metrics::workers::AdmissionPhase; use crate::model::{AgentConfig, ExecutionStatus, LookupResult, ReadFileResult, TrapType}; use crate::services::active_workers::{ - FilesystemStoragePermit, MemoryGrant, RegisteredConcurrentAccount, WorkerComponentCharge, + MemoryGrant, RegisteredConcurrentAccount, WorkerComponentCharge, }; use crate::services::card_interest::CardInterestIndex; use crate::services::events::{Event, EventsSubscription}; @@ -38,6 +37,7 @@ use crate::services::golem_config::SnapshotPolicy; use crate::services::linear_memory::{LinearMemoryTracker, SHARED_LINEAR_MEMORY_ERROR}; use crate::services::oplog::plugin::ForwardingOplog; use crate::services::oplog::{CommitLevel, Oplog, OplogOps, downcast_oplog}; +use crate::services::resource_limits::AtomicResourceEntry; use crate::services::worker::GetWorkerMetadataResult; use crate::services::worker_event::{WorkerEventService, WorkerEventServiceDefault}; use crate::services::{ @@ -49,10 +49,9 @@ use crate::services::{ HasWasmtimeEngine, HasWebSocketConnectionPool, HasWorkerEnumerationService, HasWorkerForkService, HasWorkerProxy, HasWorkerService, UsesAllDeps, }; -use crate::worker::invocation_loop::InvocationLoop; +use crate::worker::invocation_loop::{ConcurrentAgentPermitState, InvocationLoop}; use crate::worker::status::calculate_last_known_status_with_checkpoint; use crate::workerctx::WorkerCtx; -use anyhow::anyhow; use futures::FutureExt; use futures::channel::oneshot; use golem_common::base_model::agent::CachePolicy; @@ -84,9 +83,7 @@ use golem_common::one_shot::OneShotEvent; use golem_common::read_only_lock; use golem_common::related_span; use golem_common::tracing::TraceOrigin; -use golem_service_base::error::worker_executor::{ - GolemSpecificWasmTrap, InterruptKind, WorkerExecutorError, -}; +use golem_service_base::error::worker_executor::{InterruptKind, WorkerExecutorError}; use golem_service_base::model::GetFileSystemNodeResult; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -269,6 +266,57 @@ pub(super) struct PendingMemoryGrowth { job_queued: AtomicBool, } +#[derive(Default)] +struct StartupAttemptTracker { + state: StdMutex, +} + +#[derive(Default)] +struct StartupAttemptState { + pending: Option, + failure: Option, +} + +impl StartupAttemptTracker { + fn begin(&self, existing: Option) -> Uuid { + let mut state = self.state.lock().unwrap(); + let attempt = existing.or(state.pending).unwrap_or_else(Uuid::new_v4); + state.pending = Some(attempt); + state.failure = None; + attempt + } + + fn pending(&self) -> Option { + self.state.lock().unwrap().pending + } + + fn current(&self) -> Result, WorkerExecutorError> { + let state = self.state.lock().unwrap(); + match (state.pending, state.failure.as_ref()) { + (Some(attempt), _) => Ok(Some(attempt)), + (None, Some(error)) => Err(error.clone()), + (None, None) => Ok(None), + } + } + + fn complete(&self, attempt: Uuid, result: &Result<(), WorkerExecutorError>) -> bool { + let mut state = self.state.lock().unwrap(); + if state.pending != Some(attempt) { + return false; + } + state.failure = result.as_ref().err().cloned(); + state.pending = None; + true + } + + fn complete_success_if_active(&self, attempt: Uuid, active_attempt: Option) -> bool { + if active_attempt != Some(attempt) { + return false; + } + self.complete(attempt, &Ok(())) + } +} + /// Represents worker that may be running or suspended. /// /// It is responsible for receiving incoming worker invocations in a non-blocking way, @@ -303,6 +351,7 @@ pub struct Worker { invocation_results: Arc>>, ephemeral_invocation: StdMutex, initial_worker_metadata: AgentMetadata, + resource_entry: Arc, registered_concurrent_account: RegisteredConcurrentAccount, /// The published worker status. Read lock-free from any context; written only by the /// worker-state actor's status task (and during construction, before the actor exists). @@ -325,7 +374,9 @@ pub struct Worker { // IMPORTANT: Every external operation must acquire the instance lock, even briefly, to confirm the worker isn’t deleting. instance: Arc>, + startup_attempt: StartupAttemptTracker, linear_memory_grant: StdMutex>>>, + filesystem_runtime: StdMutex>, /// Lifecycle request shared across resident worker generations. A terminal request is retained /// until the worker stops so permit reacquisition cannot lose it between `RunningWorker`s. interrupt_signal: Arc>, @@ -334,14 +385,10 @@ pub struct Worker { last_resume_request: Mutex, pub(crate) snapshot_recovery_disabled: AtomicBool, - /// Bytes that triggered the last `NodeOutOfFilesystemStorage` trap. Set by - /// `acquire_filesystem_space` on failure so `WaitingWorker::new` can request - /// at least that many bytes from the blocking eviction path, ensuring - /// enough idle workers are evicted to satisfy the pending write. - desired_extra_filesystem_storage: AtomicU64, startup_linear_memory_bytes: AtomicU64, memory_growth: StdMutex>, memory_limit_interrupt_queued: AtomicBool, + filesystem_limit_interrupt: Mutex>, /// Snapshot of the active component, refreshed by `create_instance`. /// Used by the read-only cache lookup without taking the wasm `Store` @@ -680,7 +727,7 @@ impl Worker { .await?; let registered_concurrent_account = deps .active_workers() - .register_account_concurrency(owner_account_id, resource_entry) + .register_account_concurrency(owner_account_id, Arc::clone(&resource_entry)) .await; let read_only_cache_cfg = &deps.config().read_only_cache; @@ -748,10 +795,13 @@ impl Worker { EphemeralInvocationState::Available }), instance, + startup_attempt: StartupAttemptTracker::default(), linear_memory_grant: StdMutex::new(None), + filesystem_runtime: StdMutex::new(None), interrupt_signal: Arc::new(async_lock::Mutex::new(WorkerInterruptState::default())), execution_status, initial_worker_metadata, + resource_entry, registered_concurrent_account, last_known_status: current_status, metrics_status, @@ -764,10 +814,10 @@ impl Worker { status_checkpointer, last_resume_request: Mutex::new(Timestamp::now_utc()), snapshot_recovery_disabled: AtomicBool::new(false), - desired_extra_filesystem_storage: AtomicU64::new(0), startup_linear_memory_bytes: AtomicU64::new(0), memory_growth: StdMutex::new(Arc::new(PendingMemoryGrowth::default())), memory_limit_interrupt_queued: AtomicBool::new(false), + filesystem_limit_interrupt: Mutex::new(None), current_component, read_only_cache, read_only_cache_epoch: Arc::new(AtomicU64::new(0)), @@ -819,14 +869,17 @@ impl Worker { &self.snapshot_policy } - pub async fn start_if_needed(this: Arc>) -> Result { - Self::start_if_needed_internal(this, 0).await + pub async fn start_if_needed( + this: Arc>, + ) -> Result, WorkerExecutorError> { + Self::start_if_needed_internal(this, 0, None).await } async fn start_if_needed_internal( this: Arc>, oom_retry_count: u32, - ) -> Result { + existing_start_attempt: Option, + ) -> Result, WorkerExecutorError> { { *this.last_resume_request.lock().await = Timestamp::now_utc(); } @@ -840,17 +893,36 @@ impl Worker { Err(err.clone()) } WorkerInstance::Unloaded { .. } => { - this.mark_as_loading(); + let start_attempt = + existing_start_attempt.or_else(|| this.startup_attempt.pending()); + let memory_requirement = match this.memory_requirement().await { + Ok(memory_requirement) => memory_requirement, + Err(error) => { + *instance_guard = WorkerInstance::Unloaded { + startup_failure: Some(error.clone()), + }; + this.fail_pending_invocations(error.clone()).await; + drop(instance_guard); + if let Some(start_attempt) = start_attempt { + this.complete_startup(start_attempt, Err(error.clone())); + } + return Err(error); + } + }; + let start_attempt = this.startup_attempt.begin(start_attempt); + this.mark_as_loading(start_attempt); crate::metrics::workers::inc_worker_waiting_for_memory(); *instance_guard = WorkerInstance::WaitingForPermit(WaitingWorker::new( this.clone(), - this.memory_requirement().await?, - this.filesystem_storage_requirement().await?, + memory_requirement, oom_retry_count, + start_attempt, )); - Ok(true) + Ok(Some(start_attempt)) } - WorkerInstance::WaitingForPermit(_) | WorkerInstance::Running(_) => Ok(false), + WorkerInstance::CleanupFailed(error) => Err(error.clone()), + WorkerInstance::WaitingForPermit(waiting) => Ok(Some(waiting.start_attempt)), + WorkerInstance::Running(_) => this.startup_attempt.current(), WorkerInstance::Deleting => Err(WorkerExecutorError::invalid_request( "Worker is being deleted", )), @@ -906,7 +978,7 @@ impl Worker { } WorkerInstance::WaitingForPermit(_) => None, WorkerInstance::Stopping(_) => None, - WorkerInstance::Unloaded { .. } => None, + WorkerInstance::Unloaded { .. } | WorkerInstance::CleanupFailed(_) => None, WorkerInstance::Deleting => None, }; @@ -932,7 +1004,10 @@ impl Worker { let error = WorkerExecutorError::invalid_request("Worker is being deleted"); self.stop_internal(false, Some(error), FinalWorkerState::Deleting) .await; - Ok(()) + match &*self.instance.lock().await { + WorkerInstance::CleanupFailed(error) => Err(error.clone()), + _ => Ok(()), + } } pub fn event_service(&self) -> Arc { @@ -946,7 +1021,8 @@ impl Worker { ) } - fn mark_as_loading(&self) { + fn mark_as_loading(&self, start_attempt: Uuid) { + self.startup_attempt.begin(Some(start_attempt)); let mut execution_status = self.execution_status.write().unwrap(); *execution_status = ExecutionStatus::Loading { agent_mode: execution_status.agent_mode(), @@ -954,6 +1030,65 @@ impl Worker { }; } + fn publish_startup_result(&self, start_attempt: Uuid, result: Result<(), WorkerExecutorError>) { + if !self.startup_attempt.complete(start_attempt, &result) { + return; + } + self.publish_completed_startup_result(start_attempt, result); + } + + fn publish_completed_startup_result( + &self, + start_attempt: Uuid, + result: Result<(), WorkerExecutorError>, + ) { + self.events().publish(Event::WorkerLoaded { + agent_id: self.agent_id(), + start_attempt, + result, + }); + } + + pub(crate) fn complete_startup( + &self, + start_attempt: Uuid, + result: Result<(), WorkerExecutorError>, + ) { + self.publish_startup_result(start_attempt, result); + } + + pub(crate) async fn complete_startup_success(&self, start_attempt: Uuid) -> bool { + let instance_guard = self.instance.lock().await; + let active_attempt = match &*instance_guard { + WorkerInstance::Running(running) => Some(running.start_attempt), + _ => None, + }; + let is_active = active_attempt == Some(start_attempt); + let result = if is_active { + Ok(()) + } else { + Err(WorkerExecutorError::unknown( + "Worker stopped before startup completed", + )) + }; + let completed = match &result { + Ok(()) => self + .startup_attempt + .complete_success_if_active(start_attempt, active_attempt), + Err(_) => self.startup_attempt.complete(start_attempt, &result), + }; + drop(instance_guard); + + if completed { + self.publish_completed_startup_result(start_attempt, result); + } + is_active + } + + pub(crate) fn pending_startup_attempt(&self) -> Option { + self.startup_attempt.pending() + } + pub fn get_initial_worker_metadata(&self) -> AgentMetadata { self.initial_worker_metadata.clone() } @@ -1004,13 +1139,17 @@ impl Worker { interrupt_kind: InterruptKind, reacquire_permits: bool, ) -> Option> { - let instance_guard = self.lock_non_stopping_worker().await; if !self .queue_interrupt(interrupt_kind, reacquire_permits) .await { return None; } + self.notify_queued_interrupt(interrupt_kind).await + } + + async fn notify_queued_interrupt(&self, interrupt_kind: InterruptKind) -> Option> { + let instance_guard = self.lock_non_stopping_worker().await; if let WorkerInstance::Running(running) = &*instance_guard { let _ = running.sender.send(WorkerCommand::WorkAvailable); } @@ -1084,6 +1223,7 @@ impl Worker { "Explicit resume is not supported for uninitialized workers", )) } + WorkerInstance::CleanupFailed(error) => Err(error.clone()), WorkerInstance::Deleting => Err(WorkerExecutorError::invalid_request( "Explicit resume is not supported for deleting workers", )), @@ -1548,6 +1688,16 @@ impl Worker { } } + async fn notify_filesystem_limit_interrupt_if_current(&self, interrupt_kind: InterruptKind) { + let InterruptKind::Suspend(timestamp) = interrupt_kind else { + return; + }; + let pending = self.filesystem_limit_interrupt.lock().await; + if *pending == Some(timestamp) { + self.notify_queued_interrupt(interrupt_kind).await; + } + } + fn require_idempotency_key( invocation: &AgentInvocation, ) -> Result { @@ -1876,16 +2026,9 @@ impl Worker { ) } - /// Gets the storage requirement of the worker based on the last known status. - /// Used by `WaitingWorker::new` to pre-acquire storage semaphore permits. - pub async fn filesystem_storage_requirement(&self) -> Result { - let metadata = self.get_latest_worker_metadata().await; - Ok(metadata.last_known_status.current_filesystem_storage_usage) - } - /// Returns true if the worker is running, but it is not performing any invocations at the moment /// (ExecutionStatus::Suspended) and has no pending work that should keep the - /// loaded worker resident while memory and filesystem pressure is low. + /// loaded worker resident while memory pressure is low. /// /// These workers can be stopped to free up available worker memory. pub async fn is_currently_idle_but_running(&self) -> bool { @@ -1905,6 +2048,13 @@ impl Worker { ); false } + WorkerInstance::CleanupFailed(_) => { + debug!( + "Worker {} has failed filesystem cleanup, cannot be used to free up memory", + self.owned_agent_id + ); + false + } // TODO: this probably wants to cooperate with memory free up WorkerInstance::Stopping(_) => { debug!( @@ -1930,9 +2080,12 @@ impl Worker { let has_queued_internal_work = !running.queue.read().await.is_empty(); let has_resume_replay = running.resume_replay_pending.load(Ordering::Acquire); let has_interrupt = running.interrupt_signal.lock().await.has_interrupt(); + let has_filesystem_effects = self.has_active_filesystem_effects(); + let has_concurrent_agent_permit = + running.concurrent_agent_permit_held.load(Ordering::Acquire); debug!( - "Worker {} idle check: waiting_for_command={waiting_for_command} has_pending_invocations={has_pending_invocations} has_queued_internal_work={has_queued_internal_work} has_resume_replay={has_resume_replay} has_interrupt={has_interrupt}", + "Worker {} idle check: waiting_for_command={waiting_for_command} has_pending_invocations={has_pending_invocations} has_queued_internal_work={has_queued_internal_work} has_resume_replay={has_resume_replay} has_interrupt={has_interrupt} has_filesystem_effects={has_filesystem_effects} has_concurrent_agent_permit={has_concurrent_agent_permit}", self.owned_agent_id ); @@ -1941,6 +2094,16 @@ impl Worker { && !has_queued_internal_work && !has_resume_replay && !has_interrupt + && !has_filesystem_effects + && !has_concurrent_agent_permit + } + + fn has_active_filesystem_effects(&self) -> bool { + self.filesystem_runtime + .lock() + .unwrap() + .as_ref() + .is_some_and(|runtime| runtime.has_active_effects()) } /// Returns `true` iff this worker currently has a loaded wasmtime instance @@ -1955,8 +2118,8 @@ impl Worker { matches!(&*self.instance.lock().await, WorkerInstance::Running(_)) } - /// Classifies the worker for eviction ordering under memory/filesystem - /// pressure. Returns `None` if the worker is not evictable. + /// Classifies the worker for eviction ordering under memory pressure. + /// Returns `None` if the worker is not evictable. /// /// - `LoadedIdle`: resident in memory, not executing, no durable pending work. /// Evicted first. @@ -1971,13 +2134,19 @@ impl Worker { let has_queued_internal_work = !running.queue.read().await.is_empty(); let has_resume_replay = running.resume_replay_pending.load(Ordering::Acquire); let has_interrupt = running.interrupt_signal.lock().await.has_interrupt(); + let has_filesystem_effects = self.has_active_filesystem_effects(); + let has_concurrent_agent_permit = + running.concurrent_agent_permit_held.load(Ordering::Acquire); // Non-evictable if actively executing or has non-durable in-memory work - if !waiting_for_command - || has_queued_internal_work - || has_resume_replay - || has_interrupt - { + if !running_worker_can_be_evicted( + waiting_for_command, + has_queued_internal_work, + has_resume_replay, + has_interrupt, + has_filesystem_effects, + has_concurrent_agent_permit, + ) { return None; } @@ -1997,6 +2166,16 @@ impl Worker { /// Re-checks the eviction classification under the instance lock to avoid /// races. Returns `true` if the worker was actually stopped. pub async fn stop_if_evictable(&self, target_class: EvictionClass) -> bool { + self.stop_if_evictable_with_outcome(target_class, None) + .await + != EvictionStopOutcome::Ineligible + } + + pub(crate) async fn stop_if_evictable_with_outcome( + &self, + target_class: EvictionClass, + expected_eligibility: Option, + ) -> EvictionStopOutcome { let mut instance_guard = self.lock_non_stopping_worker().await; let should_stop = match &*instance_guard { WorkerInstance::Running(running) => { @@ -2004,14 +2183,21 @@ impl Worker { let has_queued_internal_work = !running.queue.read().await.is_empty(); let has_resume_replay = running.resume_replay_pending.load(Ordering::Acquire); let has_interrupt = running.interrupt_signal.lock().await.has_interrupt(); - - if !waiting_for_command - || has_queued_internal_work - || has_resume_replay - || has_interrupt - { + let has_filesystem_effects = self.has_active_filesystem_effects(); + let has_concurrent_agent_permit = + running.concurrent_agent_permit_held.load(Ordering::Acquire); + + if !running_worker_can_be_evicted( + waiting_for_command, + has_queued_internal_work, + has_resume_replay, + has_interrupt, + has_filesystem_effects, + has_concurrent_agent_permit, + ) { false } else { + let eligibility = self.current_filesystem_pressure_eligibility(running); let has_pending_invocations = !self.pending_invocations().await.is_empty(); let current_class = if has_pending_invocations { EvictionClass::WarmRunnable @@ -2019,6 +2205,14 @@ impl Worker { EvictionClass::LoadedIdle }; current_class.eviction_priority() <= target_class.eviction_priority() + && expected_eligibility.is_none_or(|expected| expected == eligibility) + && expected_eligibility.is_none_or(|_| { + self.filesystem_runtime + .lock() + .unwrap() + .as_ref() + .is_none_or(|runtime| runtime.seal_if_no_active_effects()) + }) } } _ => false, @@ -2037,10 +2231,13 @@ impl Worker { .await; drop(instance_guard); self.handle_stop_result(stop_result).await; - true + match &*self.instance.lock().await { + WorkerInstance::CleanupFailed(_) => EvictionStopOutcome::CleanupFailed, + _ => EvictionStopOutcome::Unloaded, + } } else { drop(instance_guard); - false + EvictionStopOutcome::Ineligible } } @@ -2049,6 +2246,42 @@ impl Worker { self.execution_status.read().unwrap().timestamp() } + pub(crate) async fn filesystem_pressure_eligibility( + &self, + ) -> Option { + match &*self.instance.lock().await { + WorkerInstance::Running(running) => { + Some(self.current_filesystem_pressure_eligibility(running)) + } + _ => None, + } + } + + fn current_filesystem_pressure_eligibility( + &self, + running: &RunningWorker, + ) -> FilesystemPressureEligibility { + let idle_since = running.idle_since_millis.load(Ordering::Acquire); + let last_effect_completion = self + .filesystem_runtime + .lock() + .unwrap() + .as_ref() + .map_or(0, |runtime| runtime.last_effect_completion_millis()); + FilesystemPressureEligibility { + idle_since, + last_effect_completion, + } + } + + pub(crate) fn filesystem_pressure_eligible_since( + eligibility: FilesystemPressureEligibility, + ) -> u64 { + eligibility + .idle_since + .max(eligibility.last_effect_completion) + } + /// Records a committed guest `memory.grow` without blocking the store callback. pub fn request_memory_grow(self: &Arc, delta: u64) { let growth = self.memory_growth.lock().unwrap(); @@ -2130,6 +2363,33 @@ impl Worker { } } + pub(crate) async fn update_filesystem_limit_interrupt(self: &Arc, exceeded: bool) { + if exceeded { + let mut pending = self.filesystem_limit_interrupt.lock().await; + if pending.is_some() { + return; + } + let timestamp = Timestamp::now_utc(); + let interrupt_kind = InterruptKind::Suspend(timestamp); + if self.queue_interrupt(interrupt_kind, false).await { + *pending = Some(timestamp); + self.state_actor + .filesystem_limit_exceeded(self.clone(), interrupt_kind); + } + } else if let Some(timestamp) = self.filesystem_limit_interrupt.lock().await.take() { + *self.last_resume_request.lock().await = Timestamp::now_utc(); + self.interrupt_signal.lock().await.cancel_suspend(timestamp); + } + } + + async fn request_filesystem_invalidation(self: &Arc) { + self.state_actor.filesystem_invalidated(self.clone()).await; + } + + pub(crate) async fn filesystem_retry_permitted(&self) -> bool { + !self.interrupt_signal.lock().await.has_interrupt() + } + pub(crate) fn linear_memory_grant(&self) -> Arc> { self.linear_memory_grant .lock() @@ -2148,107 +2408,6 @@ impl Worker { self.startup_linear_memory_bytes.load(Ordering::Acquire) } - /// Return `freed_bytes` to the storage semaphore pool. - /// Called from `DurableWorkerCtx::release_filesystem_space` when a file is - /// deleted or truncated. Should only be called from the invocation loop. - /// - /// The permits are returned by splitting them off `RunningWorker.filesystem_storage_permit` - /// and dropping the split portion. This correctly reduces the permit count held - /// by the `RunningWorker`, preventing double-return when it later drops. - pub async fn release_filesystem_storage_space(&self, freed_bytes: u64) { - let permits_to_release = - crate::services::active_workers::bytes_to_filesystem_storage_permits(freed_bytes); - if permits_to_release == 0 { - return; - } - if let WorkerInstance::Running(running) = &mut *self.instance.lock().await - && let Some(ref mut permit) = running.filesystem_storage_permit - { - // Split off `permits_to_release` permits and drop them. - // Dropping the split permit returns its permits to the semaphore - // automatically — no separate add_permits needed. - let n = permits_to_release as usize; - let actual_n = n.min(permit.num_permits()); - let to_drop = permit.split(actual_n); - let released_bytes = - crate::services::active_workers::filesystem_storage_permits_to_bytes( - actual_n as u32, - ); - record_filesystem_pool_released(released_bytes); - drop(to_drop); // returns permits to the semaphore - } - } - - /// Acquire storage semaphore permits for a write operation. - /// Called from `DurableWorkerCtx::acquire_filesystem_space` in live mode only. - /// Returns `NodeOutOfFilesystemStorage` if the executor pool is exhausted. - /// - /// Should only be called from the invocation loop. - pub async fn acquire_filesystem_storage_space(&self, new_bytes: u64) -> anyhow::Result<()> { - match &mut *self.instance.lock().await { - WorkerInstance::Running(running) => { - if let Some(permit) = self - .active_workers() - .try_acquire_filesystem_storage(new_bytes) - .await - { - running.merge_extra_filesystem_storage_permits(permit); - // Success — clear any pending desired_extra_filesystem_storage. - self.desired_extra_filesystem_storage - .store(0, Ordering::Relaxed); - Ok(()) - } else { - // Record the requested size so WaitingWorker can evict enough - // idle workers to satisfy this write on the next restart. - self.desired_extra_filesystem_storage - .store(new_bytes, Ordering::Relaxed); - Err(anyhow!(GolemSpecificWasmTrap::NodeOutOfFilesystemStorage)) - } - } - // Worker is stopping/unloaded — no-op; the current invocation will - // fail anyway and permits will be re-acquired on restart. - _ => Ok(()), - } - } - - /// Acquire storage semaphore permits for the total size of all initial - /// component files. Called once from `DurableWorkerCtx::create` after - /// `prepare_filesystem` has loaded the files. Merges the acquired permits - /// into the running worker's `filesystem_storage_permit` so they are released - /// automatically when the worker stops. - /// - /// Uses the non-blocking priority path (`try_acquire_storage`). If the - /// semaphore pool is full, idle workers are evicted by the semaphore's own - /// logic; the permit is returned as `None` and the caller should propagate - /// a retriable `NodeOutOfFilesystemStorage` error. - /// - /// Should only be called from the invocation loop. - pub async fn acquire_initial_filesystem_storage( - &self, - total_bytes: u64, - ) -> Result<(), GolemSpecificWasmTrap> { - if total_bytes == 0 { - return Ok(()); - } - match &mut *self.instance.lock().await { - WorkerInstance::Running(running) => { - if let Some(permit) = self - .active_workers() - .try_acquire_filesystem_storage(total_bytes) - .await - { - running.merge_extra_filesystem_storage_permits(permit); - Ok(()) - } else { - Err(GolemSpecificWasmTrap::NodeOutOfFilesystemStorage) - } - } - // Worker stopped between create and acquire — no-op, permits will be - // re-acquired on next startup from AgentStatusRecord. - _ => Ok(()), - } - } - /// Bumps the read-only cache epoch, lazily invalidating all cached entries /// (the epoch is part of the cache key). Called from /// `DurableWorkerCtx::on_agent_invocation_success` immediately after a @@ -2926,7 +3085,11 @@ impl Worker { fail_pending_invocations: Option, final_state: FinalWorkerState, ) { + let startup_error = fail_pending_invocations.clone().unwrap_or_else(|| { + WorkerExecutorError::unknown("Worker stopped before startup completed") + }); let mut instance_guard = self.instance.lock().await; + let startup_attempt = self.startup_attempt.pending(); let stop_result = self .stop_internal_locked( @@ -2941,6 +3104,9 @@ impl Worker { drop(instance_guard); self.handle_stop_result(stop_result).await; + if !called_from_invocation_loop && let Some(startup_attempt) = startup_attempt { + self.complete_startup(startup_attempt, Err(startup_error)); + } } async fn stop_internal_locked( @@ -2968,15 +3134,29 @@ impl Worker { **instance_guard = final_state.into_instance(); StopResult::Stopped } + WorkerInstance::CleanupFailed(error) => { + if let Some(ref pending_error) = fail_pending_invocations { + self.fail_pending_invocations(pending_error.clone()).await; + } + **instance_guard = WorkerInstance::CleanupFailed(error); + StopResult::Stopped + } WorkerInstance::WaitingForPermit(_) => { if let Some(ref error) = fail_pending_invocations { self.fail_pending_invocations(error.clone()).await; } crate::metrics::workers::dec_worker_waiting_for_memory(); **instance_guard = final_state.into_instance(); - if let WorkerInstance::Unloaded { startup_failure } = &**instance_guard { - self.resolve_pending_readiness_awaiters_on_stop(startup_failure.as_ref()) - .await; + match &**instance_guard { + WorkerInstance::Unloaded { startup_failure } => { + self.resolve_pending_readiness_awaiters_on_stop(startup_failure.as_ref()) + .await; + } + WorkerInstance::CleanupFailed(error) => { + self.resolve_pending_readiness_awaiters_on_stop(Some(error)) + .await; + } + _ => {} } StopResult::Stopped } @@ -2985,17 +3165,19 @@ impl Worker { // Should we return an error here? StopResult::Stopped } - WorkerInstance::Stopping(_) if called_from_invocation_loop => { - **instance_guard = previous_instance_state; + WorkerInstance::Stopping(mut stopping) if called_from_invocation_loop => { + if let Some(ref error) = fail_pending_invocations { + self.fail_pending_invocations(error.clone()).await; + } + stopping.final_state = merge_final_worker_state(stopping.final_state, final_state); + **instance_guard = WorkerInstance::Stopping(stopping); StopResult::Stopped } WorkerInstance::Stopping(mut stopping) => { - // If we're stopping for deletion, upgrade the final state - if matches!(final_state, FinalWorkerState::Deleting) { - stopping.final_state = FinalWorkerState::Deleting; - if let Some(ref error) = fail_pending_invocations { - self.fail_pending_invocations(error.clone()).await; - } + let deleting = matches!(&final_state, FinalWorkerState::Deleting); + stopping.final_state = merge_final_worker_state(stopping.final_state, final_state); + if deleting && let Some(ref error) = fail_pending_invocations { + self.fail_pending_invocations(error.clone()).await; } let notify = stopping.notify.clone(); **instance_guard = WorkerInstance::Stopping(stopping); @@ -3038,9 +3220,18 @@ impl Worker { // the old generation's grant. self.release_linear_memory_grant(); **instance_guard = final_state.into_instance(); - if let WorkerInstance::Unloaded { startup_failure } = &**instance_guard { - self.resolve_pending_readiness_awaiters_on_stop(startup_failure.as_ref()) + match &**instance_guard { + WorkerInstance::Unloaded { startup_failure } => { + self.resolve_pending_readiness_awaiters_on_stop( + startup_failure.as_ref(), + ) .await; + } + WorkerInstance::CleanupFailed(error) => { + self.resolve_pending_readiness_awaiters_on_stop(Some(error)) + .await; + } + _ => {} } StopResult::Stopped } else { @@ -3105,9 +3296,16 @@ impl Worker { } other => panic!("expected Stopping, got {other:?}"), } - if let WorkerInstance::Unloaded { startup_failure } = &*instance_guard { - self.resolve_pending_readiness_awaiters_on_stop(startup_failure.as_ref()) - .await; + match &*instance_guard { + WorkerInstance::Unloaded { startup_failure } => { + self.resolve_pending_readiness_awaiters_on_stop(startup_failure.as_ref()) + .await; + } + WorkerInstance::CleanupFailed(error) => { + self.resolve_pending_readiness_awaiters_on_stop(Some(error)) + .await; + } + _ => {} } drop(instance_guard); @@ -3233,7 +3431,8 @@ impl Worker { called_from_invocation_loop: bool, delay: Option, oom_retry_count: u32, - ) -> Result { + start_attempt: Option, + ) -> Result, WorkerExecutorError> { this.stop_internal( called_from_invocation_loop, None, @@ -3245,7 +3444,7 @@ impl Worker { if let Some(delay) = delay { tokio::time::sleep(delay).await; } - Self::start_if_needed_internal(this, oom_retry_count).await + Self::start_if_needed_internal(this, oom_retry_count, start_attempt).await } async fn get_or_create_worker_metadata< @@ -3552,7 +3751,6 @@ impl Worker { this: Arc>, memory_grant: MemoryGrant, component_charge: WorkerComponentCharge, - filesystem_storage_permit: Option, concurrent_agent_permit: crate::services::active_workers::ConcurrentAgentPermit, oom_retry_count: u32, start_attempt: Uuid, @@ -3563,7 +3761,7 @@ impl Worker { WorkerInstance::WaitingForPermit(waiting_worker) if waiting_worker.start_attempt == start_attempt => { - let mut running = RunningWorker::new( + let running = RunningWorker::new( this.owned_agent_id.clone(), this.queue.clone(), this.clone(), @@ -3571,12 +3769,10 @@ impl Worker { component_charge, concurrent_agent_permit, oom_retry_count, + start_attempt, worker_trace, ) .await; - if let Some(sp) = filesystem_storage_permit { - running.merge_extra_filesystem_storage_permits(sp); - } crate::metrics::workers::dec_worker_waiting_for_memory(); crate::metrics::workers::inc_worker_memory_resident(); *instance_guard = WorkerInstance::Running(running); @@ -3713,6 +3909,7 @@ enum WorkerInstance { Unloaded { startup_failure: Option, }, + CleanupFailed(WorkerExecutorError), WaitingForPermit(WaitingWorker), Running(RunningWorker), Stopping(StoppingWorker), @@ -3735,7 +3932,8 @@ impl WorkerInstance { match self { Self::Unloaded { startup_failure: Some(err), - } => Some(err), + } + | Self::CleanupFailed(err) => Some(err), _ => None, } } @@ -3762,13 +3960,11 @@ impl WaitingWorker { pub fn new( parent: Arc>, memory_requirement: u64, - filesystem_storage_requirement: u64, oom_retry_count: u32, + start_attempt: Uuid, ) -> Self { let worker_trace = parent.trace(TraceOrigin::capture_current()); - let start_attempt = Uuid::new_v4(); - let handle = tokio::task::spawn(async move { let agent_id = parent.owned_agent_id.agent_id(); let registered_concurrent_account = parent.registered_concurrent_account.clone(); @@ -3853,71 +4049,11 @@ impl WaitingWorker { AdmissionPhase::Memory, phase_start.elapsed(), ); - // Pre-acquire storage permits for this restart. - // - // We need to acquire `filesystem_storage_requirement + desired_extra` total: - // - `filesystem_storage_requirement`: bytes to hold as the pre-acquired permit - // for replay (mirrors what the worker held before being evicted). - // The old RunningWorker already returned these bytes to the pool - // when it dropped, so the pool likely already has them — the - // blocking acquire will find them without needing to evict anyone. - // - `desired_extra`: bytes for the write that triggered NodeOutOfFilesystemStorage. - // The pool may not have these yet, so the blocking acquire will - // evict idle workers only for the missing portion. - // - // After acquiring, we release `desired_extra` back to the pool so - // it is available for the pending write to re-acquire at runtime. - // - // Example: prior writes = 3 KB, failing write needs 1 KB extra. - // Old RunningWorker drops → 3 KB returned to pool. - // acquire_bytes = 4 KB. Pool has 3 KB → 1 KB gap → evict 1 KB. - // Hold 3 KB as filesystem_storage_permit, release 1 KB → pool has 1 KB free. - // Pending write re-acquires 1 KB → succeeds. - let desired_extra = parent - .desired_extra_filesystem_storage - .load(Ordering::Relaxed); - let acquire_bytes = filesystem_storage_requirement + desired_extra; - let filesystem_storage_permit = if acquire_bytes > 0 { - let phase_start = std::time::Instant::now(); - let mut permit = parent - .active_workers() - .acquire_filesystem_storage(acquire_bytes) - .instrument(related_span!( - worker_trace.startup_origin, - Level::INFO, - "acquire_filesystem_storage", - %agent_id, - agent_type = %worker_trace.agent_type - )) - .await; - crate::metrics::workers::record_worker_admission_wait( - AdmissionPhase::FilesystemStorage, - phase_start.elapsed(), - ); - // Release the `desired_extra` portion back to the pool. - if desired_extra > 0 { - let extra_permits = - crate::services::active_workers::bytes_to_filesystem_storage_permits( - desired_extra, - ) as usize; - if let Some(extra) = permit.split(extra_permits) { - drop(extra); // returns to semaphore - } - } - if permit.num_permits() > 0 { - Some(permit) - } else { - None - } - } else { - None - }; debug!("Attempting to start worker after acquiring enough permits"); Worker::start_waiting_worker( parent, memory_grant, component_charge, - filesystem_storage_permit, concurrent_agent_permit, oom_retry_count, start_attempt, @@ -3964,17 +4100,17 @@ impl PendingWorkerInterrupt { !matches!(self.kind, InterruptKind::Restart | InterruptKind::Jump) } - /// How the invocation loop should proceed after honoring this interrupt: restart-like - /// interrupts (`Restart`, `Jump`) retry immediately, terminal ones do not retry at all, and a - /// permit-reacquisition request overrides both because the retry must go back through the - /// admission gate. + /// How the invocation loop should proceed after honoring this interrupt. A suspension retains + /// its timestamp so a newer wakeup can supersede it, while an explicit interrupt remains + /// terminal. fn retry_decision(&self) -> RetryDecision { if self.reacquire_permits { RetryDecision::ReacquirePermits } else { match self.kind { InterruptKind::Restart | InterruptKind::Jump => RetryDecision::Immediate, - InterruptKind::Interrupt(_) | InterruptKind::Suspend(_) => RetryDecision::None, + InterruptKind::Interrupt(_) => RetryDecision::None, + InterruptKind::Suspend(timestamp) => RetryDecision::TryStop(timestamp), } } } @@ -4023,6 +4159,26 @@ impl WorkerInterruptState { _ => None, } } + + fn release_terminal_claim(&mut self) { + if matches!(self, Self::TerminalClaimed) { + *self = Self::Idle; + } + } + + fn cancel_suspend(&mut self, timestamp: Timestamp) -> bool { + match self { + Self::Pending(PendingWorkerInterrupt { + kind: InterruptKind::Suspend(pending_timestamp), + .. + }) if *pending_timestamp == timestamp => { + *self = Self::Idle; + true + } + Self::TerminalClaimed => false, + _ => true, + } + } } #[derive(Debug)] @@ -4030,16 +4186,28 @@ struct RunningWorker { handle: Option>, sender: UnboundedSender, queue: Arc>>, - /// Storage semaphore permits held by this worker. `None` until storage - /// space is first acquired (at startup or on first write). Dropped - /// automatically when `RunningWorker` is dropped, returning storage - /// permits to the pool. - filesystem_storage_permit: Option, waiting_for_command: Arc, + concurrent_agent_permit_held: Arc, + idle_since_millis: Arc, interrupt_signal: Arc>, /// `ResumeReplay` is signalled directly through the command channel rather /// than the internal queue, so eviction must treat it as pending work. resume_replay_pending: Arc, + start_attempt: Uuid, +} + +pub(crate) struct CreateWorkerInstanceError { + pub(crate) error: WorkerExecutorError, + pub(crate) filesystem_cleanup_failed: bool, +} + +impl From for CreateWorkerInstanceError { + fn from(error: WorkerExecutorError) -> Self { + Self { + error, + filesystem_cleanup_failed: false, + } + } } struct LinearMemoryGrantRegistration { @@ -4074,19 +4242,6 @@ impl Drop for LinearMemoryGrantRegistration { } } -impl Drop for RunningWorker { - fn drop(&mut self) { - if let Some(ref permit) = self.filesystem_storage_permit { - let bytes = crate::services::active_workers::filesystem_storage_permits_to_bytes( - permit.num_permits() as u32, - ); - if bytes > 0 { - record_filesystem_pool_released(bytes); - } - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum LinearMemoryEnumerationError { Shared, @@ -4179,6 +4334,7 @@ impl RunningWorker { component_charge: WorkerComponentCharge, concurrent_agent_permit: crate::services::active_workers::ConcurrentAgentPermit, oom_retry_count: u32, + start_attempt: Uuid, worker_trace: WorkerTrace, ) -> Self { let (sender, receiver) = tokio::sync::mpsc::unbounded_channel(); @@ -4188,6 +4344,10 @@ impl RunningWorker { let owned_agent_id_clone = owned_agent_id.clone(); let waiting_for_command = Arc::new(AtomicBool::new(false)); let waiting_for_command_clone = waiting_for_command.clone(); + let concurrent_agent_permit_held = Arc::new(AtomicBool::new(true)); + let concurrent_agent_permit_held_clone = Arc::clone(&concurrent_agent_permit_held); + let idle_since_millis = Arc::new(AtomicU64::new(0)); + let idle_since_millis_clone = Arc::clone(&idle_since_millis); let interrupt_signal = parent.interrupt_signal.clone(); let interrupt_signal_clone = interrupt_signal.clone(); let resume_replay_pending = Arc::new(AtomicBool::new(false)); @@ -4206,7 +4366,10 @@ impl RunningWorker { interrupt_signal_clone, oom_retry_count, concurrent_agent_permit, + concurrent_agent_permit_held_clone, + idle_since_millis_clone, resume_replay_pending_clone, + start_attempt, worker_trace, ) .await; @@ -4217,23 +4380,12 @@ impl RunningWorker { handle: Some(handle), sender, queue, - filesystem_storage_permit: None, waiting_for_command, + concurrent_agent_permit_held, + idle_since_millis, interrupt_signal, resume_replay_pending, - } - } - - /// Merge additional storage permits into this worker's storage permit. If - /// the worker does not yet hold a storage permit, the given permit becomes - /// the initial one. Additional calls merge into that initial permit. - pub fn merge_extra_filesystem_storage_permits( - &mut self, - extra_permit: FilesystemStoragePermit, - ) { - match &mut self.filesystem_storage_permit { - Some(existing) => existing.merge(extra_permit), - None => self.filesystem_storage_permit = Some(extra_permit), + start_attempt, } } @@ -4243,7 +4395,14 @@ impl RunningWorker { async fn create_instance( parent: Arc>, - ) -> Result<(Instance, async_lock::Mutex>), WorkerExecutorError> { + ) -> Result< + ( + Instance, + async_lock::Mutex>, + crate::services::agent_filesystem::AgentFilesystem, + ), + CreateWorkerInstanceError, + > { let component_id = parent.owned_agent_id.component_id(); // we might have detached the worker status during the last invocation loop. Make sure it's attached and we are fully up-to-date on the oplog @@ -4316,7 +4475,7 @@ impl RunningWorker { }; if component_metadata.metadata.has_shared_linear_memory() { - return Err(shared_linear_memory_error(&parent)); + return Err(shared_linear_memory_error(&parent).into()); } // Refresh the snapshot used by the read-only cache key. The component @@ -4381,7 +4540,80 @@ impl RunningWorker { last_snapshot_index = Some(snapshot_idx); } - let context = Ctx::create( + let filesystems = parent.active_workers().agent_filesystems(); + let initial_files = parent + .parsed_agent_id + .as_ref() + .and_then(|agent_id| { + component_metadata_for_replay + .metadata + .agent_type_provision_configs() + .get(&agent_id.agent_type) + }) + .map(|config| config.files.clone()) + .unwrap_or_default(); + let filesystem = filesystems + .create_fresh(crate::services::agent_filesystem::CreateAgentFilesystem { + agent_id: parent.owned_agent_id.clone(), + initial_files, + file_loader: parent.file_loader(), + resource_limits: Some(Arc::clone(&parent.resource_entry)), + limit_exceeded: Some({ + let worker = Arc::downgrade(&parent); + Arc::new(move |exceeded| { + let worker = worker.clone(); + Box::pin(async move { + if let Some(worker) = worker.upgrade() { + worker.update_filesystem_limit_interrupt(exceeded).await; + } + }) + }) + }), + }) + .await + .map_err(|error| { + let filesystem_cleanup_failed = error.cleanup_failed(); + let error = if error.is_storage_exhaustion() { + InterruptKind::Suspend(Timestamp::now_utc()).into() + } else { + WorkerExecutorError::runtime(error.to_string()) + }; + CreateWorkerInstanceError { + error, + filesystem_cleanup_failed, + } + })?; + + *parent.filesystem_runtime.lock().unwrap() = Some(filesystem.runtime()); + + filesystem.runtime().set_invalidation_callback(Some({ + let worker = Arc::downgrade(&parent); + Arc::new(move || { + let worker = worker.clone(); + Box::pin(async move { + if let Some(worker) = worker.upgrade() { + worker.request_filesystem_invalidation().await; + } + }) + }) + })); + filesystem.runtime().set_pressure_recovery_callback(Some({ + let active_workers = Arc::downgrade(&parent.active_workers()); + Arc::new(move |operation, deadline| { + let active_workers = active_workers.clone(); + Box::pin(async move { + match active_workers.upgrade() { + Some(active_workers) => { + active_workers + .recover_filesystem_pressure(operation, deadline) + .await + } + None => false, + } + }) + }) + })); + let context = match Ctx::create( worker_metadata.created_by, OwnedAgentId::new(worker_metadata.environment_id, &worker_metadata.agent_id), parent.parsed_agent_id.clone(), @@ -4405,12 +4637,11 @@ impl RunningWorker { parent.component_service(), parent.extra_deps(), parent.config(), + filesystem.path().to_path_buf(), + filesystem.runtime(), AgentConfig::new( skipped_regions, worker_metadata.last_known_status.total_linear_memory_size, - worker_metadata - .last_known_status - .current_filesystem_storage_usage, component_version_for_replay, worker_metadata.created_by, worker_metadata.created_by_email, @@ -4431,7 +4662,13 @@ impl RunningWorker { pending_update, worker_metadata.original_phantom_id, ) - .await?; + .await + { + Ok(context) => context, + Err(error) => { + return Err(cleanup_failed_agent_filesystem(filesystem, error).await); + } + }; let engine = parent.engine(); let mut store = Store::new(&engine, context); @@ -4465,15 +4702,19 @@ impl RunningWorker { )), } }); - store + if let Err(error) = store .set_fuel(u64::MAX) - .map_err(|e| WorkerExecutorError::runtime(e.to_string()))?; + .map_err(|error| WorkerExecutorError::runtime(error.to_string())) + { + drop(store); + return Err(cleanup_failed_agent_filesystem(filesystem, error).await); + } store.limiter_async(|ctx| ctx.resource_limiter()); let linker = (*parent.linker()).clone(); // fresh linker - let instance_pre = linker.instantiate_pre(&component).map_err(|e| { + let instance_pre = match linker.instantiate_pre(&component).map_err(|e| { WorkerExecutorError::worker_creation_failed( parent.owned_agent_id.agent_id(), format!( @@ -4481,9 +4722,15 @@ impl RunningWorker { parent.owned_agent_id ), ) - })?; + }) { + Ok(instance_pre) => instance_pre, + Err(error) => { + drop(store); + return Err(cleanup_failed_agent_filesystem(filesystem, error).await); + } + }; - let instance = instance_pre + let instance = match instance_pre .instantiate_async(&mut store) .await .map_err(|e| { @@ -4502,10 +4749,19 @@ impl RunningWorker { ), ) } - })?; - Self::reconcile_linear_memories(&parent, &mut store).await?; + }) { + Ok(instance) => instance, + Err(error) => { + drop(store); + return Err(cleanup_failed_agent_filesystem(filesystem, error).await); + } + }; + if let Err(error) = Self::reconcile_linear_memories(&parent, &mut store).await { + drop(store); + return Err(cleanup_failed_agent_filesystem(filesystem, error).await); + } let store = async_lock::Mutex::new(store); - Ok((instance, store)) + Ok((instance, store, filesystem)) } async fn invocation_loop( @@ -4517,7 +4773,10 @@ impl RunningWorker { interrupt_signal: Arc>, oom_retry_count: u32, concurrent_agent_permit: crate::services::active_workers::ConcurrentAgentPermit, + concurrent_agent_permit_held: Arc, + idle_since_millis: Arc, resume_replay_pending: Arc, + start_attempt: Uuid, worker_trace: WorkerTrace, ) { let mut invocation_loop = InvocationLoop { @@ -4528,14 +4787,37 @@ impl RunningWorker { waiting_for_command, interrupt_signal, oom_retry_count, - concurrent_agent_permit: Some(concurrent_agent_permit), + permit_state: ConcurrentAgentPermitState::new( + Some(concurrent_agent_permit), + concurrent_agent_permit_held, + ), + idle_since_millis, resume_replay_pending, + start_attempt, worker_trace, }; invocation_loop.run().await; } } +async fn cleanup_failed_agent_filesystem( + filesystem: crate::services::agent_filesystem::AgentFilesystem, + startup_error: WorkerExecutorError, +) -> CreateWorkerInstanceError { + match filesystem.close_and_delete().await { + Ok(()) => startup_error.into(), + Err(cleanup_error) => { + warn!(error = %cleanup_error, "Failed to clean up filesystem after worker startup failure"); + CreateWorkerInstanceError { + error: WorkerExecutorError::runtime(format!( + "{startup_error}; additionally failed to clean up the agent filesystem: {cleanup_error}" + )), + filesystem_cleanup_failed: true, + } + } + } +} + fn shared_linear_memory_error(parent: &Arc>) -> WorkerExecutorError { WorkerExecutorError::worker_creation_failed( parent.owned_agent_id.agent_id(), @@ -4561,6 +4843,35 @@ pub enum EvictionClass { WarmRunnable, } +fn running_worker_can_be_evicted( + waiting_for_command: bool, + has_queued_internal_work: bool, + has_resume_replay: bool, + has_interrupt: bool, + has_filesystem_effects: bool, + has_concurrent_agent_permit: bool, +) -> bool { + waiting_for_command + && !has_queued_internal_work + && !has_resume_replay + && !has_interrupt + && !has_filesystem_effects + && !has_concurrent_agent_permit +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EvictionStopOutcome { + Ineligible, + Unloaded, + CleanupFailed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct FilesystemPressureEligibility { + idle_since: u64, + last_effect_completion: u64, +} + impl EvictionClass { /// Lower values are evicted first. pub fn eviction_priority(self) -> u8 { @@ -4576,6 +4887,7 @@ pub(crate) enum FinalWorkerState { Unloaded { startup_failure: Option, }, + CleanupFailed(WorkerExecutorError), Deleting, } @@ -4585,11 +4897,33 @@ impl FinalWorkerState { FinalWorkerState::Unloaded { startup_failure } => { WorkerInstance::Unloaded { startup_failure } } + FinalWorkerState::CleanupFailed(error) => WorkerInstance::CleanupFailed(error), FinalWorkerState::Deleting => WorkerInstance::Deleting, } } } +fn merge_final_worker_state( + current: FinalWorkerState, + requested: FinalWorkerState, +) -> FinalWorkerState { + match (¤t, &requested) { + (FinalWorkerState::CleanupFailed(_), _) => current, + (_, FinalWorkerState::CleanupFailed(_)) => requested, + (FinalWorkerState::Deleting, _) => current, + (_, FinalWorkerState::Deleting) => requested, + ( + FinalWorkerState::Unloaded { + startup_failure: None, + }, + FinalWorkerState::Unloaded { + startup_failure: Some(_), + }, + ) => requested, + _ => current, + } +} + #[derive(Debug)] struct StoppingWorker { notify: OneShotEvent, @@ -4762,6 +5096,131 @@ mod tests { use golem_common::model::oplog::AgentError; use test_r::test; + #[test] + fn active_filesystem_effects_exclude_an_otherwise_idle_worker() { + assert!(running_worker_can_be_evicted( + true, false, false, false, false, false + )); + assert!(!running_worker_can_be_evicted( + true, false, false, false, true, false + )); + assert!(!running_worker_can_be_evicted( + true, false, false, false, false, true + )); + } + + #[test] + fn merging_unloaded_states_preserves_startup_failure() { + let state = merge_final_worker_state( + FinalWorkerState::Unloaded { + startup_failure: None, + }, + FinalWorkerState::Unloaded { + startup_failure: Some(WorkerExecutorError::runtime("startup failed")), + }, + ); + + assert!(matches!( + state, + FinalWorkerState::Unloaded { + startup_failure: Some(_) + } + )); + } + + #[test] + fn concurrent_startup_callers_share_one_attempt() { + let tracker = Arc::new(StartupAttemptTracker::default()); + let barrier = Arc::new(std::sync::Barrier::new(16)); + let handles = (0..16) + .map(|_| { + let tracker = tracker.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + tracker.begin(None) + }) + }) + .collect::>(); + + let attempts = handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect::>(); + assert!(attempts.iter().all(|attempt| *attempt == attempts[0])); + assert_eq!(tracker.pending(), Some(attempts[0])); + } + + #[test] + fn oom_retry_preserves_startup_attempt() { + let tracker = StartupAttemptTracker::default(); + let initial = tracker.begin(None); + + let retry = tracker.begin(Some(initial)); + + assert_eq!(retry, initial); + assert_eq!(tracker.current().unwrap(), Some(initial)); + assert!(!tracker.complete(Uuid::new_v4(), &Ok(()))); + assert_eq!(tracker.pending(), Some(initial)); + assert!(tracker.complete(initial, &Ok(()))); + assert_eq!(tracker.current().unwrap(), None); + } + + #[test] + fn startup_success_requires_matching_active_attempt() { + let tracker = StartupAttemptTracker::default(); + let attempt = tracker.begin(None); + + assert!(!tracker.complete_success_if_active(attempt, None)); + assert_eq!(tracker.pending(), Some(attempt)); + assert!(!tracker.complete_success_if_active(attempt, Some(Uuid::new_v4()))); + assert_eq!(tracker.pending(), Some(attempt)); + assert!(tracker.complete_success_if_active(attempt, Some(attempt))); + assert_eq!(tracker.current().unwrap(), None); + } + + #[test] + fn stop_wins_atomic_startup_completion() { + let tracker = Arc::new(StartupAttemptTracker::default()); + let attempt = tracker.begin(None); + let active_attempt = Arc::new(StdMutex::new(Some(attempt))); + let (stop_holds_instance, wait_for_stop) = std::sync::mpsc::channel(); + let (allow_stop_to_finish, finish_stop) = std::sync::mpsc::channel(); + + let stop = { + let active_attempt = active_attempt.clone(); + std::thread::spawn(move || { + let mut active_attempt = active_attempt.lock().unwrap(); + *active_attempt = None; + stop_holds_instance.send(()).unwrap(); + finish_stop.recv().unwrap(); + }) + }; + + wait_for_stop.recv().unwrap(); + let (completion_started, wait_for_completion) = std::sync::mpsc::channel(); + let completion = { + let tracker = tracker.clone(); + let active_attempt = active_attempt.clone(); + std::thread::spawn(move || { + completion_started.send(()).unwrap(); + let active_attempt = *active_attempt.lock().unwrap(); + assert!(!tracker.complete_success_if_active(attempt, active_attempt)); + let error = WorkerExecutorError::unknown("Worker stopped before startup completed"); + assert!(tracker.complete(attempt, &Err(error.clone()))); + tracker.current() + }) + }; + + wait_for_completion.recv().unwrap(); + allow_stop_to_finish.send(()).unwrap(); + stop.join().unwrap(); + let error = WorkerExecutorError::unknown("Worker stopped before startup completed"); + assert!( + matches!(completion.join().unwrap(), Err(actual) if actual.to_string() == error.to_string()) + ); + } + #[test] fn allocated_memory_sums_unique_untouched_backings() -> anyhow::Result<()> { let engine = wasmtime::Engine::default(); @@ -4794,6 +5253,29 @@ mod tests { )); } + #[test] + fn filesystem_cleanup_failure_overrides_pending_unload() { + let cleanup_error = WorkerExecutorError::runtime("cleanup failed"); + let final_state = merge_final_worker_state( + FinalWorkerState::Unloaded { + startup_failure: None, + }, + FinalWorkerState::CleanupFailed(cleanup_error), + ); + + assert!(matches!(final_state, FinalWorkerState::CleanupFailed(_))); + } + + #[test] + fn filesystem_cleanup_failure_prevents_successful_deletion() { + let final_state = merge_final_worker_state( + FinalWorkerState::Deleting, + FinalWorkerState::CleanupFailed(WorkerExecutorError::runtime("cleanup failed")), + ); + + assert!(matches!(final_state, FinalWorkerState::CleanupFailed(_))); + } + #[test] fn ephemeral_agent_accepts_only_one_invocation_identity() { let first = IdempotencyKey::fresh(); @@ -5005,14 +5487,16 @@ mod tests { RetryDecision::Immediate ); - // Terminal interrupts do not retry. + // Explicit interrupts remain terminal. assert_eq!( decision(InterruptKind::Interrupt(Timestamp::now_utc()), false), RetryDecision::None ); + // Suspensions stop unless a newer wakeup supersedes them. + let suspend_timestamp = Timestamp::now_utc(); assert_eq!( - decision(InterruptKind::Suspend(Timestamp::now_utc()), false), - RetryDecision::None + decision(InterruptKind::Suspend(suspend_timestamp), false), + RetryDecision::TryStop(suspend_timestamp) ); // Permit reacquisition overrides the kind-based decision for every kind. @@ -5045,6 +5529,43 @@ mod tests { assert!(terminal(InterruptKind::Interrupt(Timestamp::now_utc()))); assert!(terminal(InterruptKind::Suspend(Timestamp::now_utc()))); } + + #[test] + fn terminal_interrupt_claim_is_released_after_a_worker_generation() { + let mut state = WorkerInterruptState::Idle; + assert!(state.queue(PendingWorkerInterrupt { + kind: InterruptKind::Suspend(Timestamp::now_utc()), + reacquire_permits: false, + })); + assert!(state.take().is_some()); + assert!(!state.queue(PendingWorkerInterrupt { + kind: InterruptKind::Restart, + reacquire_permits: false, + })); + + state.release_terminal_claim(); + + assert!(state.queue(PendingWorkerInterrupt { + kind: InterruptKind::Restart, + reacquire_permits: false, + })); + } + + #[test] + fn pending_filesystem_suspend_can_be_cancelled_by_timestamp() { + let timestamp = Timestamp::now_utc(); + let mut state = WorkerInterruptState::Idle; + assert!(state.queue(PendingWorkerInterrupt { + kind: InterruptKind::Suspend(timestamp), + reacquire_permits: false, + })); + + assert!(state.cancel_suspend(timestamp)); + assert!(matches!(state, WorkerInterruptState::Idle)); + + state = WorkerInterruptState::TerminalClaimed; + assert!(!state.cancel_suspend(timestamp)); + } } #[derive(Clone, Debug, Eq, PartialEq, Hash)] diff --git a/golem-worker-executor/src/worker/state_actor.rs b/golem-worker-executor/src/worker/state_actor.rs index 6520552ffa..87e0359afd 100644 --- a/golem-worker-executor/src/worker/state_actor.rs +++ b/golem-worker-executor/src/worker/state_actor.rs @@ -126,6 +126,14 @@ enum LifecycleJob { worker: Arc>, meter: AgentMemoryMeter, }, + FilesystemLimitExceeded { + worker: Arc>, + interrupt_kind: InterruptKind, + }, + FilesystemInvalidated { + worker: Arc>, + done: oneshot::Sender<()>, + }, } /// The state exclusively owned by the status task. @@ -241,6 +249,18 @@ impl WorkerStateActor { .await; } } + LifecycleJob::FilesystemLimitExceeded { + worker, + interrupt_kind, + } => { + worker + .notify_filesystem_limit_interrupt_if_current(interrupt_kind) + .await; + } + LifecycleJob::FilesystemInvalidated { worker, done } => { + worker.set_interrupting(InterruptKind::Restart).await; + let _ = done.send(()); + } } } }); @@ -324,6 +344,43 @@ impl WorkerStateActor { } } + pub fn filesystem_limit_exceeded( + &self, + worker: Arc>, + interrupt_kind: InterruptKind, + ) { + if self + .lifecycle_jobs + .send(LifecycleJob::FilesystemLimitExceeded { + worker, + interrupt_kind, + }) + .is_err() + { + panic!( + "Worker state actor for {} terminated unexpectedly", + self.owned_agent_id + ); + } + } + + pub async fn filesystem_invalidated(&self, worker: Arc>) { + let (done, done_rx) = oneshot::channel(); + if self + .lifecycle_jobs + .send(LifecycleJob::FilesystemInvalidated { worker, done }) + .is_err() + { + panic!( + "Worker state actor for {} terminated unexpectedly", + self.owned_agent_id + ); + } + done_rx + .await + .expect("Worker state actor terminated while invalidating filesystem"); + } + pub fn queue_ordered_oplog_entry( &self, worker: Arc>, diff --git a/golem-worker-executor/src/worker/status.rs b/golem-worker-executor/src/worker/status.rs index e184ddd411..95ad7d5196 100644 --- a/golem-worker-executor/src/worker/status.rs +++ b/golem-worker-executor/src/worker/status.rs @@ -310,12 +310,6 @@ pub fn update_status_with_new_entries( &new_entries, ); - let current_filesystem_storage_usage = calculate_current_filesystem_storage_usage( - last_known.current_filesystem_storage_usage, - &skipped_regions, - &new_entries, - ); - let owned_resources = collect_resources(last_known.owned_resources, &skipped_regions, &new_entries); @@ -350,7 +344,6 @@ pub fn update_status_with_new_entries( component_size, owned_resources, total_linear_memory_size, - current_filesystem_storage_usage, active_plugins, oplog_processor_checkpoints, revoked_cards, @@ -485,7 +478,6 @@ fn calculate_latest_worker_status( OplogEntry::FailedUpdate { .. } => {} OplogEntry::SuccessfulUpdate { .. } => {} OplogEntry::GrowMemory { .. } => {} - OplogEntry::FilesystemStorageUsageUpdate { .. } => {} OplogEntry::CreateResource { .. } => {} OplogEntry::DropResource { .. } => {} OplogEntry::Log { .. } => { @@ -980,38 +972,6 @@ fn calculate_total_linear_memory_size( result } -/// Accumulates `FilesystemStorageUsageUpdate` hint entries to reconstruct the current -/// storage usage at any point in the oplog. Used to populate -/// `AgentStatusRecord::current_filesystem_storage_usage` for pre-acquiring storage permits -/// when a worker restarts. -/// -/// Mirrors `calculate_total_linear_memory_size`: entries in skipped regions -/// are excluded, and `Create` resets the counter to zero (a newly created worker -/// has no written files yet). -fn calculate_current_filesystem_storage_usage( - current: u64, - skipped_regions: &DeletedRegions, - entries: &BTreeMap, -) -> u64 { - let mut result = current as i64; - for (idx, entry) in entries { - if skipped_regions.is_in_deleted_region(*idx) { - continue; - } - - match entry { - OplogEntry::Create { .. } => { - result = 0; - } - OplogEntry::FilesystemStorageUsageUpdate { delta, .. } => { - result = result.saturating_add(*delta); - } - _ => {} - } - } - result.max(0) as u64 -} - fn collect_resources( initial: HashMap, skipped_regions: &DeletedRegions, @@ -1200,8 +1160,6 @@ fn is_worker_error_retriable( AgentError::PermanentError(_) => false, AgentError::ExceededHttpCallLimit => false, AgentError::ExceededRpcCallLimit => false, - AgentError::NodeOutOfFilesystemStorage => true, - AgentError::AgentExceededFilesystemStorageLimit => false, AgentError::AgentTerminatedByQuota(_) => false, AgentError::EphemeralSleepTooLong(_) => false, AgentError::EphemeralFuelExhausted(_) => false, @@ -1284,39 +1242,6 @@ mod test { run_test_case(test_case).await; } - #[test] - async fn storage_usage_accumulated_from_deltas() { - let test_case = TestCase::builder(0) - .agent_invocation_started("a", vec![], IdempotencyKey::fresh()) - .filesystem_storage_usage_update(1024) - .filesystem_storage_usage_update(2048) - .build(); - - run_test_case(test_case).await; - } - - #[test] - async fn storage_usage_decremented_on_negative_delta() { - let test_case = TestCase::builder(0) - .agent_invocation_started("a", vec![], IdempotencyKey::fresh()) - .filesystem_storage_usage_update(1024) - .filesystem_storage_usage_update(-512) - .build(); - - run_test_case(test_case).await; - } - - #[test] - async fn storage_usage_clamped_at_zero_on_underflow() { - let test_case = TestCase::builder(0) - .agent_invocation_started("a", vec![], IdempotencyKey::fresh()) - .filesystem_storage_usage_update(100) - .filesystem_storage_usage_update(-9999) // larger than total acquired - .build(); - - run_test_case(test_case).await; - } - #[test] async fn semantic_retry_state_keeps_status_retrying_after_legacy_retry_limit() { let idempotency_key = IdempotencyKey::fresh(); @@ -2222,22 +2147,6 @@ mod test { ) } - pub fn filesystem_storage_usage_update(self, delta: i64) -> Self { - self.add( - OplogEntry::FilesystemStorageUsageUpdate { - timestamp: Timestamp::now_utc(), - delta, - }, - |mut status| { - status.current_filesystem_storage_usage = - (status.current_filesystem_storage_usage as i64) - .saturating_add(delta) - .max(0) as u64; - status - }, - ) - } - pub fn snapshot(self) -> Self { let oplog_idx = OplogIndex::from_u64(self.entries.len() as u64 + 1); let timestamp = Timestamp::now_utc().rounded(); @@ -3129,103 +3038,6 @@ mod test { assert_eq!(state.target_agent_id, None); } - fn make_fs_entry(idx: u64, delta: i64) -> (OplogIndex, OplogEntry) { - ( - OplogIndex::from_u64(idx), - OplogEntry::FilesystemStorageUsageUpdate { - timestamp: Timestamp::now_utc(), - delta, - }, - ) - } - - fn make_create_entry(idx: u64) -> (OplogIndex, OplogEntry) { - use golem_common::base_model::account::AccountId; - use golem_common::base_model::component::{ComponentId, ComponentRevision}; - use golem_common::base_model::environment::EnvironmentId; - use golem_common::model::AgentId; - let agent_id = AgentId { - component_id: ComponentId::new(), - agent_id: "w".to_string(), - }; - ( - OplogIndex::from_u64(idx), - OplogEntry::create( - agent_id, - AgentMode::Durable, - ComponentRevision::INITIAL, - vec![], - EnvironmentId::new(), - AccountId::new(), - None, - 0, - 0, - Default::default(), - vec![], - None, - Uuid::now_v7(), - ), - ) - } - - /// `FilesystemStorageUsageUpdate` entries inside a deleted (skipped) region - /// are excluded from `current_filesystem_storage_usage`. Only live entries count. - #[test] - fn filesystem_storage_usage_in_deleted_region_is_skipped() { - use golem_common::model::regions::{DeletedRegionsBuilder, OplogRegion}; - let mut builder = DeletedRegionsBuilder::default(); - // Mark indices 2..=4 as deleted. - builder.add(OplogRegion { - start: OplogIndex::from_u64(2), - end: OplogIndex::from_u64(4), - }); - let deleted = builder.build(); - - let entries: BTreeMap = BTreeMap::from([ - make_fs_entry(2, 1024), // deleted — must be skipped - make_fs_entry(3, 2048), // deleted — must be skipped - make_fs_entry(5, 512), // live - ]); - - let result = super::calculate_current_filesystem_storage_usage(0, &deleted, &entries); - assert_eq!( - result, 512, - "only the live entry outside the deleted region counts" - ); - } - - /// A `Create` entry mid-oplog resets `current_filesystem_storage_usage` to zero, - /// discarding usage accumulated before it (including the seed). - #[test] - fn filesystem_storage_usage_reset_to_zero_on_create() { - let deleted = DeletedRegions::default(); - - let entries: BTreeMap = BTreeMap::from([ - make_fs_entry(1, 1024), // before Create → should be wiped - make_create_entry(2), // resets counter to 0 - make_fs_entry(3, 512), // after Create → counts - ]); - - // Seed with prior usage to confirm Create overrides the seed too. - let result = super::calculate_current_filesystem_storage_usage(999, &deleted, &entries); - assert_eq!( - result, 512, - "Create must reset usage to 0 before accumulating post-Create deltas" - ); - } - - /// `current` seed is used as the starting value when there are no `Create` - /// entries and no deleted regions. - #[test] - fn filesystem_storage_usage_uses_seed_when_no_create() { - let deleted = DeletedRegions::default(); - - let entries: BTreeMap = BTreeMap::from([make_fs_entry(1, 512)]); - - let result = super::calculate_current_filesystem_storage_usage(1024, &deleted, &entries); - assert_eq!(result, 1536, "seed + delta"); - } - #[test] fn card_revoked_entry_is_recorded_in_status() { let card_id = golem_common::model::card::CardId::new(); diff --git a/golem-worker-executor/src/workerctx/default.rs b/golem-worker-executor/src/workerctx/default.rs index 3f2a634c2d..bbf869cac3 100644 --- a/golem-worker-executor/src/workerctx/default.rs +++ b/golem-worker-executor/src/workerctx/default.rs @@ -27,6 +27,7 @@ use crate::preview2::golem::agent::host::{ ScheduledInvocationReceipt, WasmRpc, }; use crate::services::active_workers::ActiveWorkers; +use crate::services::agent_filesystem::AgentFilesystemRuntime; use crate::services::agent_types::AgentTypesService; use crate::services::agent_webhooks::AgentWebhooksService; use crate::services::blob_store::BlobStoreService; @@ -880,6 +881,8 @@ impl WorkerCtx for Context { component_service: Arc, _extra_deps: Self::ExtraDeps, config: Arc, + filesystem_root: std::path::PathBuf, + filesystem_runtime: AgentFilesystemRuntime, worker_config: AgentConfig, execution_status: Arc>, file_loader: Arc, @@ -917,6 +920,8 @@ impl WorkerCtx for Context { component_service, account_resource_limits.clone(), config.clone(), + filesystem_root, + filesystem_runtime, worker_config.clone(), execution_status, file_loader, diff --git a/golem-worker-executor/src/workerctx/mod.rs b/golem-worker-executor/src/workerctx/mod.rs index 5403220cdc..99870992f9 100644 --- a/golem-worker-executor/src/workerctx/mod.rs +++ b/golem-worker-executor/src/workerctx/mod.rs @@ -18,6 +18,7 @@ use crate::durable_host::websocket::WebSocketConnectionPool; use crate::durable_host::{DurableWorkerCtxView, SnapshotBoundaryBlocker}; use crate::model::{AgentConfig, ExecutionStatus, LastError, ReadFileResult, TrapType}; use crate::services::active_workers::ActiveWorkers; +use crate::services::agent_filesystem::AgentFilesystemRuntime; use crate::services::agent_types::AgentTypesService; use crate::services::agent_webhooks::AgentWebhooksService; use crate::services::blob_store::BlobStoreService; @@ -158,6 +159,8 @@ pub trait WorkerCtx: component_service: Arc, extra_deps: Self::ExtraDeps, config: Arc, + filesystem_root: std::path::PathBuf, + filesystem_runtime: AgentFilesystemRuntime, worker_config: AgentConfig, execution_status: Arc>, file_loader: Arc, diff --git a/golem-worker-executor/tests/api.rs b/golem-worker-executor/tests/api.rs index 6d8964fbf1..5b679af67e 100644 --- a/golem-worker-executor/tests/api.rs +++ b/golem-worker-executor/tests/api.rs @@ -2060,7 +2060,6 @@ async fn long_running_poll_loop_http_failures_are_retried( deps, &context, None, - None, Some(RetryConfig { max_attempts: 30, min_delay: Duration::from_millis(100), diff --git a/golem-worker-executor/tests/lib.rs b/golem-worker-executor/tests/lib.rs index 046114f2d8..fdbcabd021 100644 --- a/golem-worker-executor/tests/lib.rs +++ b/golem-worker-executor/tests/lib.rs @@ -52,7 +52,6 @@ pub mod retry_policies; pub mod revert; pub mod rpc; pub mod scalability; -pub mod storage_quota; pub mod tool_discovery; pub mod transactions; pub mod wasi; @@ -113,7 +112,6 @@ tag_suite!(ignite_service, ignite_service); tag_suite!(rdbms_service, rdbms_service); tag_suite!(resource_limits, group1); tag_suite!(oplog_metrics, group1); -tag_suite!(storage_quota, group1); tag_suite!(tool_discovery, group1); sequential_suite!(key_value_storage); diff --git a/golem-worker-executor/tests/scalability.rs b/golem-worker-executor/tests/scalability.rs index d4331e719c..b6c9dec0cd 100644 --- a/golem-worker-executor/tests/scalability.rs +++ b/golem-worker-executor/tests/scalability.rs @@ -311,7 +311,6 @@ async fn initial_large_memory_allocation( None, None, None, - None, ) .await?; let component = executor @@ -373,7 +372,6 @@ async fn dynamic_large_memory_allocation( None, None, None, - None, ) .await?; let component = executor @@ -431,17 +429,8 @@ async fn interrupt_wins_over_dynamic_memory_permit_reacquisition( (MEMORY_LIMIT as f64 * MemoryConfig::default().worker_memory_ratio) as u64; let context = TestContext::new(last_unique_id); - let executor = start_customized( - deps, - &context, - Some(MEMORY_LIMIT), - None, - None, - None, - None, - None, - ) - .await?; + let executor = + start_customized(deps, &context, Some(MEMORY_LIMIT), None, None, None, None).await?; let component = executor .component_dep(&context.default_environment_id, large_dynamic_memory) .store() @@ -578,7 +567,6 @@ async fn eviction_prefers_idle_workers_over_warm_runnable( None, None, None, - None, ) .await?; diff --git a/golem-worker-executor/tests/storage_quota.rs b/golem-worker-executor/tests/storage_quota.rs deleted file mode 100644 index 4e464794d1..0000000000 --- a/golem-worker-executor/tests/storage_quota.rs +++ /dev/null @@ -1,1966 +0,0 @@ -// Copyright 2024-2026 Golem Cloud -// -// Licensed under the Golem Source License v1.1 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://license.golem.cloud/LICENSE -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::Tracing; -use golem_common::model::component::{AgentFilePermissions, CanonicalFilePath}; -use golem_common::schema::SchemaValue; -use golem_common::schema::schema_value::ResultValuePayload; -use golem_common::{agent_id, data_value}; -use golem_test_framework::dsl::TestDsl; -use golem_test_framework::model::IFSEntry; -use golem_worker_executor_test_utils::{ - LastUniqueId, PrecompiledComponent, TestContext, WorkerExecutorTestDependencies, - start_with_agent_storage_quota, start_with_executor_storage_pool, -}; -use std::path::PathBuf; -use test_r::{inherit_test_dep, test, timeout}; - -inherit_test_dep!(WorkerExecutorTestDependencies); -inherit_test_dep!(LastUniqueId); -inherit_test_dep!(Tracing); -inherit_test_dep!( - #[tagged_as("host_api_tests")] - PrecompiledComponent -); - -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_write_within_limit_succeeds( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start_with_agent_storage_quota(deps, &context, 1024 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "agent-quota-ok-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile.txt", "hello world"), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_write_exceeding_limit_fails( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 5-byte limit — "hello world" (11 bytes) exceeds it. - let executor = start_with_agent_storage_quota(deps, &context, 5).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "agent-quota-exceeded-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - let result = executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile.txt", "hello world"), - ) - .await; - - assert!( - result.is_err(), - "expected write to fail when agent quota is exceeded" - ); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected AgentExceededFilesystemStorageLimit, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_exceeded_limit_is_not_retried( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start_with_agent_storage_quota(deps, &context, 5).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "agent-quota-no-retry-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - let first = executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile.txt", "hello world"), - ) - .await; - assert!( - first.is_err(), - "expected first write to fail with AgentExceededFilesystemStorageLimit" - ); - - let second = executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile2.txt", "hello world 2"), - ) - .await; - assert!( - second.is_err(), - "expected second write to also fail — agent quota is permanent" - ); - - let err_str = format!("{second:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") - || err_str.contains("storage") - || err_str.contains("already exists") - || err_str.contains("AlreadyExists"), - "expected a relevant error on second attempt, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_freed_after_file_deletion( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // Exactly 11 bytes — fits "hello world" once. A second write of the same - // size must succeed after deletion returns the quota. - let executor = start_with_agent_storage_quota(deps, &context, 11).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "agent-quota-release-delete-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile.txt", "hello world"), - ) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "delete_file", - data_value!("/testfile.txt"), - ) - .await?; - - // Write a second file with distinct content within the 11-byte quota. - // "hi world!!" is 10 bytes - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile2.txt", "hi world!!"), - ) - .await?; - - let content = executor - .invoke_and_await_agent( - &component, - &agent_id, - "read_file", - data_value!("/testfile2.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - content, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String("hi world!!".to_string()))) - }) - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// Rewriting the same file with the same content via `write_file` should not -/// consume additional quota, because the resulting file size does not grow. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_overwrite_same_file_should_not_double_charge( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // Exactly 11 bytes: enough for one "hello world" payload. Overwrite of the - // same file with the same payload must also succeed. - let executor = start_with_agent_storage_quota(deps, &context, 11).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "agent-quota-overwrite-same-file-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/same.txt", "hello world"), - ) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/same.txt", "hello world"), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// Stream-based writes on a file-backed output stream must be subject to -/// storage quota. Writing more bytes than the per-agent quota allows must fail with -/// `AgentExceededFilesystemStorageLimit`. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_stream_write_exceeding_limit_fails( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 5-byte quota — writing 1024 bytes via stream exceeds it. - let executor = start_with_agent_storage_quota(deps, &context, 5).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "stream-write-quota-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - let result = executor - .invoke_and_await_agent( - &component, - &agent_id, - "stream_to_file", - data_value!("/stream.bin", 1024u64), - ) - .await; - - assert!( - result.is_err(), - "expected stream_to_file to fail when quota exceeded" - ); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected AgentExceededFilesystemStorageLimit, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// Stream-based writes within the per-agent quota succeed. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_stream_write_within_limit_succeeds( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 1 MB quota — writing 1024 bytes via stream is within it. - let executor = start_with_agent_storage_quota(deps, &context, 1024 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "stream-write-quota-ok-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "stream_to_file", - data_value!("/stream.bin", 1024u64), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// A failed stream-write attempt that exceeds stream permit (`check_write`) -/// must roll back any pre-reserved executor pool allocation so another worker -/// can still allocate and write. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn executor_pool_stream_write_failed_attempt_does_not_leak_pool_permits( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 2 MiB executor pool so the failing worker can reserve 2 MiB first. - let executor = start_with_executor_storage_pool(deps, &context, 2 * 1024 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let failing_agent_id = agent_id!("FileSystem", "stream-write-failed-no-leak-a-1"); - let failing_worker_id = executor - .start_agent(&component.id, failing_agent_id.clone()) - .await?; - - let failed = executor - .invoke_and_await_agent( - &component, - &failing_agent_id, - "stream_to_file", - data_value!("/big.bin", 2 * 1024 * 1024u64), - ) - .await; - assert!(failed.is_err(), "expected oversized stream_to_file to fail"); - - // A second worker should still be able to allocate and write if the first - // worker's failed attempt released pool permits. - let succeeding_agent_id = agent_id!("FileSystem", "stream-write-failed-no-leak-b-1"); - let succeeding_worker_id = executor - .start_agent(&component.id, succeeding_agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &succeeding_agent_id, - "stream_to_file", - data_value!("/small.bin", 1024u64), - ) - .await?; - - executor - .check_oplog_is_queryable(&failing_worker_id) - .await?; - executor - .check_oplog_is_queryable(&succeeding_worker_id) - .await?; - - Ok(()) -} - -/// Overwriting the same file with `stream_to_file` must not double-charge -/// quota when the file does not grow. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_stream_overwrite_same_file_should_not_double_charge( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start_with_agent_storage_quota(deps, &context, 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "stream-overwrite-same-file-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "stream_to_file", - data_value!("/same-stream.bin", 1024u64), - ) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "stream_to_file", - data_value!("/same-stream.bin", 1024u64), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// Stream writes on stdout must NOT charge storage quota — console streams -/// are exempt. Even with a very tight per-agent quota (1 byte), a large -/// stream write to stdout must succeed. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_stream_to_stdout_does_not_charge_quota( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 1-byte per-agent quota — file writes would immediately fail. - // Stdout writes must succeed regardless. - let executor = start_with_agent_storage_quota(deps, &context, 1).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "stream-stdout-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "stream_to_stdout", - data_value!(1024u64), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// `blocking_stream_and_flush_to_file` on a file-backed output stream must be -/// subject to storage quota. Exceeding the limit must fail. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_blocking_stream_and_flush_exceeding_limit_fails( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 5-byte quota — writing 1024 bytes via stream exceeds it. - let executor = start_with_agent_storage_quota(deps, &context, 5).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "blocking-stream-write-quota-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - let result = executor - .invoke_and_await_agent( - &component, - &agent_id, - "blocking_stream_and_flush_to_file", - data_value!("/stream.bin", 1024u64), - ) - .await; - - assert!( - result.is_err(), - "expected blocking_stream_and_flush_to_file to fail when quota exceeded" - ); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected AgentExceededFilesystemStorageLimit, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// `blocking_stream_and_flush_to_file` on a file-backed stream within the -/// per-agent quota succeeds. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_blocking_stream_and_flush_within_limit_succeeds( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start_with_agent_storage_quota(deps, &context, 1024 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "blocking-stream-write-quota-ok-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "blocking_stream_and_flush_to_file", - data_value!("/stream.bin", 1024u64), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// `set_file_size` growing a file beyond the per-agent quota must fail with -/// `AgentExceededFilesystemStorageLimit`. Only the delta (new_size − current_size) is -/// charged, not the full new size. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_set_size_grow_beyond_limit_fails( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 11-byte quota. Write 11 bytes first (exhausts quota). Then set_size to - // 12 — grows by 1 byte — must fail because the delta (1 byte) would exceed - // the remaining quota (0 bytes). - let executor = start_with_agent_storage_quota(deps, &context, 11).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "set-size-grow-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - // Write 11 bytes — exhausts the quota. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/file.txt", "hello world"), - ) - .await?; - - // Grow by 1 byte — must fail (quota exhausted). - let result = executor - .invoke_and_await_agent( - &component, - &agent_id, - "set_file_size", - data_value!("/file.txt", 12u64), - ) - .await; - assert!( - result.is_err(), - "expected set_size grow to fail: quota exhausted" - ); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected AgentExceededFilesystemStorageLimit, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// `set_file_size` shrinking a file releases the freed bytes back to the -/// per-agent quota, allowing a subsequent write of equal size to succeed. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_set_size_shrink_releases_quota( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 11-byte quota. Write 11 bytes (exhausts quota). Shrink to 5 bytes — - // releases 6 bytes. A subsequent write of 6 bytes must now succeed. - let executor = start_with_agent_storage_quota(deps, &context, 11).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "set-size-shrink-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - // Write 11 bytes — exhausts quota. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/file.txt", "hello world"), - ) - .await?; - - // Shrink to 5 bytes — frees 6 bytes back to quota. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "set_file_size", - data_value!("/file.txt", 5u64), - ) - .await?; - - // Write 6 bytes to a new file — must succeed (6 bytes freed by shrink). - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/file2.txt", "hello!"), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// `pwrite_file` (direct `descriptor::write`) is subject to per-agent quota. -/// Writing beyond the quota must fail with `AgentExceededFilesystemStorageLimit`. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_pwrite_beyond_limit_fails( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 5-byte quota. pwrite of 11 bytes ("hello world") exceeds it. - let executor = start_with_agent_storage_quota(deps, &context, 5).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "pwrite-quota-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - let result = executor - .invoke_and_await_agent( - &component, - &agent_id, - "pwrite_file", - data_value!("/file.txt", 0u64, "hello world"), - ) - .await; - assert!(result.is_err(), "expected pwrite to fail: quota exceeded"); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected AgentExceededFilesystemStorageLimit, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// `pwrite_file` (direct `descriptor::write`) within the per-agent quota succeeds. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_pwrite_within_limit_succeeds( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start_with_agent_storage_quota(deps, &context, 1024 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "pwrite-ok-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "pwrite_file", - data_value!("/file.txt", 0u64, "hello world"), - ) - .await?; - - let content = executor - .invoke_and_await_agent(&component, &agent_id, "read_file", data_value!("/file.txt")) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - - assert_eq!( - content, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String("hello world".to_string()))) - }) - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// Overwriting the same byte range via direct `descriptor::write` (`pwrite_file`) -/// must not consume additional quota when file size does not grow. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_pwrite_overwrite_same_range_should_not_double_charge( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // Exactly 11 bytes: enough for one "hello world" payload. - let executor = start_with_agent_storage_quota(deps, &context, 11).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "agent-quota-pwrite-overwrite-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "pwrite_file", - data_value!("/pwrite.txt", 0u64, "hello world"), - ) - .await?; - - // Same offset and same payload: logical size remains 11 bytes. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "pwrite_file", - data_value!("/pwrite.txt", 0u64, "hello world"), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// Storage quota is cumulative across write paths. Writing via `write_file` -/// and then via `stream_to_file` both count against the same per-agent -/// quota. If their combined sizes exceed the limit, the second write fails. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_cumulative_across_write_paths( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 15-byte quota. First write: 11 bytes ("hello world") via write_file. - // Second write: 8 bytes via stream_to_file — combined 19 bytes > 15 → fails. - let executor = start_with_agent_storage_quota(deps, &context, 15).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "agent-quota-cumulative-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - // First write: 11 bytes — succeeds, 4 bytes remaining. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/file1.txt", "hello world"), - ) - .await?; - - // Second write: 8 bytes via stream API — would total 19 bytes > 15 → fails. - let result = executor - .invoke_and_await_agent( - &component, - &agent_id, - "stream_to_file", - data_value!("/stream.bin", 8u64), - ) - .await; - - assert!( - result.is_err(), - "expected stream_to_file to fail: combined writes exceed quota" - ); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected AgentExceededFilesystemStorageLimit, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// Account-quota level lifecycle: write → delete → suspend → restart → write → verify quota. -/// -/// Same sequence as `executor_pool_storage_usage_survives_restart` but using -/// the per-agent plan limit. Verifies that `current_filesystem_storage_usage` stays -/// accurate at the account-quota layer across the suspend/restart cycle. -/// -/// Quota = 22 bytes. Each 11-byte "hello world" write is counted exactly. -/// -/// 1. Write file-1 (11 bytes) → usage: 11, remaining: 11. -/// 2. Write file-2 (11 bytes) → quota exhausted: usage: 22, remaining: 0. -/// 3. Delete file-1 → usage: 11, remaining: 11. -/// 4. Interrupt → worker unloaded from memory. -/// 5. Re-invoke → restart reconstructs current_filesystem_storage_usage = 11 bytes. -/// 6. Write file-3 (11 bytes) → succeeds (11 bytes remaining in quota). -/// 7. Write file-4 (11 bytes) → fails with `AgentExceededFilesystemStorageLimit` -/// (quota exhausted: file-2 + file-3 = 22 bytes). -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_storage_usage_survives_restart( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 32-byte quota: fits exactly two 16-byte writes ("unique-content-N"). - let executor = start_with_agent_storage_quota(deps, &context, 32).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent = agent_id!("FileSystem", "agent-lifecycle-1"); - let worker = executor.start_agent(&component.id, agent.clone()).await?; - - executor - .invoke_and_await_agent( - &component, - &agent, - "write_file", - data_value!("/file-1.txt", "unique-content-1"), - ) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent, - "write_file", - data_value!("/file-2.txt", "unique-content-2"), - ) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent, - "delete_file", - data_value!("/file-1.txt"), - ) - .await?; - - executor.interrupt(&worker).await?; - - // Restart reconstructs current_filesystem_storage_usage = 16 bytes (+16 +16 -16). - // Verify file-2 has its distinct content — confirms the right file survived. - // file-1 was deleted; reading file-1 would return an error, not file-2's content. - let content = executor - .invoke_and_await_agent(&component, &agent, "read_file", data_value!("/file-2.txt")) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - content, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String( - "unique-content-2".to_string() - ))) - }) - ); - - executor - .invoke_and_await_agent( - &component, - &agent, - "write_file", - data_value!("/file-3.txt", "hello world"), - ) - .await?; - - let result = executor - .invoke_and_await_agent( - &component, - &agent, - "write_file", - data_value!("/file-4.txt", "hello world"), - ) - .await; - assert!( - result.is_err(), - "expected write to fail: quota exhausted (file-2 + file-3 = 22 bytes)" - ); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected AgentExceededFilesystemStorageLimit, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker).await?; - - Ok(()) -} - -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn executor_pool_write_within_capacity_succeeds( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start_with_executor_storage_pool(deps, &context, 1024 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "exec-pool-ok-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile.txt", "hello world"), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn executor_pool_freed_after_file_deletion( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 1 KB pool. "hello world" (11 bytes) rounds up to 1 KB = 1 permit, - // exhausting the pool. A second write of the same size must succeed after - // deletion returns the permit. - let executor = start_with_executor_storage_pool(deps, &context, 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "exec-pool-release-delete-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile.txt", "hello world"), - ) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "delete_file", - data_value!("/testfile.txt"), - ) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile2.txt", "hello world"), - ) - .await?; - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// A failed executor-pool acquire (`NodeOutOfFilesystemStorage`) must not leave a -/// phantom `FilesystemStorageUsageUpdate` in the oplog. -/// -/// `FilesystemStorageUsageUpdate` is written to the oplog only AFTER `acquire_filesystem_space` -/// succeeds. If a transient failure during the `ReacquirePermits` retry loop -/// wrote a phantom entry, `current_filesystem_storage_usage` would be inflated on restart -/// and the pool pre-acquire would consume more than the actual files on disk. -/// -/// Pool = 3 KB (no agent quota so only `NodeOutOfFilesystemStorage` can fire). -/// -/// 1. Write file1 (1 KB) → usage: 1 KB, pool: 2 KB free. -/// 2. Write file2 (2 KB) → usage: 3 KB, pool: 0 KB. -/// May transiently hit NodeOutOfFilesystemStorage and retry — no phantom must be written. -/// 3. Delete file1 → usage: 2 KB, pool: 1 KB free. -/// 4. Interrupt → RunningWorker drops → pool: 3 KB free. -/// 5. Restart → pre-acquires `current_filesystem_storage_usage = 2 KB` from oplog → pool: 1 KB free. -/// 6. Write file3 (1 KB) → must succeed. -/// If phantom entries inflated usage to > 2 KB, pre-acquire would leave < 1 KB -/// free and file3 would fail. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn executor_pool_failed_acquire_leaves_no_phantom_oplog_entry( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 3 KB pool, no per-agent quota. Only NodeOutOfFilesystemStorage (retriable) can fire. - let executor = start_with_executor_storage_pool(deps, &context, 3 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "exec-ordering-1"); - let worker = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - let content_1kb = "A".repeat(1024); - let content_2kb = "B".repeat(2 * 1024); - let content_file3 = "C".repeat(1024); - - // Step 1: write 1 KB → pool: 2 KB free, usage: 1 KB. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/file1.txt", content_1kb.as_str()), - ) - .await?; - - // Step 2: write 2 KB → pool: 0 KB, usage: 3 KB. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/file2.txt", content_2kb.as_str()), - ) - .await?; - - // Step 3: delete file1 → usage drops to 2 KB, pool: 1 KB free. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "delete_file", - data_value!("/file1.txt"), - ) - .await?; - - // Step 4: interrupt → RunningWorker drops, pool: 3 KB free. - executor.interrupt(&worker).await?; - - // Step 5+6: restart pre-acquires 2 KB (file2 only, no phantom from transient - // failures) → pool: 1 KB free. Write file3 (1 KB) → must succeed. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/file3.txt", content_file3.as_str()), - ) - .await?; - - // Verify file2 and file3 content - let read2 = executor - .invoke_and_await_agent( - &component, - &agent_id, - "read_file", - data_value!("/file2.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value"))?; - assert_eq!( - read2, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String(content_2kb))) - }) - ); - - let read3 = executor - .invoke_and_await_agent( - &component, - &agent_id, - "read_file", - data_value!("/file3.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value"))?; - assert_eq!( - read3, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String(content_file3))) - }) - ); - - executor.check_oplog_is_queryable(&worker).await?; - - Ok(()) -} - -/// When the executor storage pool is full, a worker restart pre-acquires -/// storage via the blocking path which evicts the oldest idle worker, freeing -/// its permits so the restarting worker can proceed. -/// -/// Flow (1 KB pool, each 11-byte write rounds up to 1 KB = 1 permit): -/// 1. Worker A writes 1 KB, then is interrupted → permit released. -/// 2. Worker B writes 1 KB (pool free after A's interrupt), then is interrupted. -/// 3. Worker A re-invoked → restarts with filesystem_storage_requirement = 1 KB -/// → blocking acquire_storage succeeds → A holds the 1 KB permit, now idle. -/// 4. Worker B re-invoked → restarts with filesystem_storage_requirement = 1 KB -/// → pool is 0 (A holds it) → blocking acquire_storage calls -/// try_free_up_storage → evicts idle Worker A → 1 KB freed -/// → Worker B acquires the permit and its invocation succeeds. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn executor_pool_idle_worker_evicted_when_pool_full( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 4 KB pool = 4 permits. Each worker writes a 2 KB file (2 permits). - // Using >1 KB content exercises multi-permit eviction — verifies that - // try_free_up_storage frees enough bytes (not just 1 permit minimum). - let executor = start_with_executor_storage_pool(deps, &context, 4 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_a = agent_id!("FileSystem", "eviction-a-1"); - let agent_b = agent_id!("FileSystem", "eviction-b-1"); - - // 2 KB content strings (> 1 KB so each write consumes 2 permits). - let content_a = "A".repeat(2048); - let content_b = "B".repeat(2048); - - // Step 1: Worker A writes 2 KB, then is interrupted to release its 2 permits. - let worker_a = executor.start_agent(&component.id, agent_a.clone()).await?; - executor - .invoke_and_await_agent( - &component, - &agent_a, - "write_file", - data_value!("/file-a.txt", content_a.as_str()), - ) - .await?; - executor.interrupt(&worker_a).await?; - - // Step 2: Worker B writes 2 KB (pool is free after A's interrupt), then - // is interrupted so its 2 permits are released. - let worker_b = executor.start_agent(&component.id, agent_b.clone()).await?; - executor - .invoke_and_await_agent( - &component, - &agent_b, - "write_file", - data_value!("/file-b.txt", content_b.as_str()), - ) - .await?; - executor.interrupt(&worker_b).await?; - - // Step 3: Re-invoke Worker A. Restarts with filesystem_storage_requirement = 2 KB. - // Pool has 4 KB free → blocking acquire_storage acquires 2 permits. - // Worker A reads file and becomes idle, holding 2 permits. - let read_a = executor - .invoke_and_await_agent( - &component, - &agent_a, - "read_file", - data_value!("/file-a.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - read_a, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String(content_a.clone()))) - }) - ); - - // Step 4: Re-invoke Worker B. Restarts with filesystem_storage_requirement = 2 KB. - // Pool has 2 KB free (Worker A holds 2) — not enough for B's 2 KB requirement. - // Blocking acquire_storage calls try_free_up_storage → evicts idle Worker A - // (freeing 2 permits) → Worker B acquires its 2 permits and reads successfully. - let read_b = executor - .invoke_and_await_agent( - &component, - &agent_b, - "read_file", - data_value!("/file-b.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - read_b, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String(content_b.clone()))) - }) - ); - - executor.check_oplog_is_queryable(&worker_a).await?; - executor.check_oplog_is_queryable(&worker_b).await?; - - Ok(()) -} - -/// When the executor pool is exhausted by an idle worker, a second worker's -/// first-time write triggers eviction of the idle worker via `desired_extra_filesystem_storage`, -/// freeing enough permits for the write to succeed. -/// -/// Worker A writes 2 KB and goes idle (holding 2 permits). Worker B tries to -/// write 2 KB — pool is exhausted → `NodeOutOfFilesystemStorage` → `ReacquirePermits`. -/// `desired_extra_filesystem_storage` is set to 2 KB so the blocking `acquire_storage` in -/// the restart path requests 2 KB from `try_free_up_storage`, which evicts idle -/// Worker A (freeing 2 permits). Worker B then succeeds. -/// -/// Uses >1 KB content to verify that `desired_extra_filesystem_storage` correctly drives -/// multi-permit eviction rather than just the 1-permit minimum. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn executor_pool_idle_worker_evicted_on_first_write_node_out_of_filesystem_storage( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 2 KB pool = 2 permits. Worker A exhausts it with a 2 KB write. - // Worker B must evict A to write its own 2 KB file. - let executor = start_with_executor_storage_pool(deps, &context, 2 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_a = agent_id!("FileSystem", "exec-pool-exhausted-a-1"); - let agent_b = agent_id!("FileSystem", "exec-pool-exhausted-b-1"); - - // 2 KB content (> 1 KB so each write consumes 2 permits, not just 1). - let content_a = "A".repeat(2048); - let content_b = "B".repeat(2048); - - // Worker A writes 2 KB and goes idle — pool exhausted (2/2 permits held). - let worker_a = executor.start_agent(&component.id, agent_a.clone()).await?; - executor - .invoke_and_await_agent( - &component, - &agent_a, - "write_file", - data_value!("/file-a.txt", content_a.as_str()), - ) - .await?; - - // Worker B writes 2 KB. Pool is full → NodeOutOfFilesystemStorage → desired_extra_filesystem_storage - // set to 2 KB → ReacquirePermits → blocking acquire_storage requests 2 KB - // → try_free_up_storage evicts idle Worker A (freeing 2 permits) - // → Worker B acquires 2 permits and its write succeeds. - let worker_b = executor.start_agent(&component.id, agent_b.clone()).await?; - executor - .invoke_and_await_agent( - &component, - &agent_b, - "write_file", - data_value!("/file-b.txt", content_b.as_str()), - ) - .await?; - - // Verify Worker B's file has the correct unique content. - let content = executor - .invoke_and_await_agent( - &component, - &agent_b, - "read_file", - data_value!("/file-b.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - content, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String(content_b))) - }) - ); - - executor.check_oplog_is_queryable(&worker_a).await?; - executor.check_oplog_is_queryable(&worker_b).await?; - - Ok(()) -} - -/// Verifies that eviction only acquires the gap — i.e. only evicts as many -/// idle workers as needed to cover the missing portion of the requested bytes, -/// not the full amount. -/// -/// Pool = 6 KB. Setup: -/// Worker A writes 2 KB → pool: 4 KB free. -/// Worker B writes 1 KB → pool: 3 KB free. -/// Worker C writes 3 KB → pool: 0 KB free. C goes idle. -/// -/// C then tries to write an extra 2 KB. Pool is exhausted → NodeOutOfFilesystemStorage. -/// `desired_extra_filesystem_storage = 2 KB`. On `ReacquirePermits` restart: -/// - Old C RunningWorker drops → 3 KB returned to pool (pool: 3 KB free). -/// - `acquire_bytes = filesystem_storage_requirement(3KB) + desired_extra(2KB) = 5 KB`. -/// - Pool has 3 KB → gap = 2 KB → eviction targets 2 KB → evicts idle Worker A -/// (holding 2 KB) → pool: 3 + 2 = 5 KB free. -/// - Acquires 5 KB, releases 2 KB (desired_extra) → pool: 2 KB free. -/// - C holds 3 KB as filesystem_storage_permit (its existing files). -/// - C's pending 2 KB write re-acquires → succeeds. -/// - Worker B (1 KB) is NOT evicted — only the minimum gap was cleared. -/// -/// After the test: B and C are both running with correct file contents. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn executor_pool_only_gap_evicted_not_full_amount( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 6 KB pool = 6 permits. - let executor = start_with_executor_storage_pool(deps, &context, 6 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_a = agent_id!("FileSystem", "gap-evict-a-1"); - let agent_b = agent_id!("FileSystem", "gap-evict-b-1"); - let agent_c = agent_id!("FileSystem", "gap-evict-c-1"); - - // Content strings sized to consume specific permit counts. - let content_a = "A".repeat(2 * 1024); // 2 KB → 2 permits - let content_b = "B".repeat(1024); // 1 KB → 1 permit - let content_c1 = "C".repeat(3 * 1024); // 3 KB → 3 permits - let content_c2 = "D".repeat(2 * 1024); // 2 KB → the extra write that fails - - // Worker A: write 2 KB → pool: 4 KB free. - let worker_a = executor.start_agent(&component.id, agent_a.clone()).await?; - executor - .invoke_and_await_agent( - &component, - &agent_a, - "write_file", - data_value!("/file-a.txt", content_a.as_str()), - ) - .await?; - - // Worker B: write 1 KB → pool: 3 KB free. - let worker_b = executor.start_agent(&component.id, agent_b.clone()).await?; - executor - .invoke_and_await_agent( - &component, - &agent_b, - "write_file", - data_value!("/file-b.txt", content_b.as_str()), - ) - .await?; - - // Worker C: write 3 KB → pool: 0 KB free. C goes idle. - let worker_c = executor.start_agent(&component.id, agent_c.clone()).await?; - executor - .invoke_and_await_agent( - &component, - &agent_c, - "write_file", - data_value!("/file-c1.txt", content_c1.as_str()), - ) - .await?; - - // Worker C tries to write 2 KB more. Pool is exhausted → - // NodeOutOfFilesystemStorage → desired_extra_filesystem_storage = 2 KB → ReacquirePermits. - // C's old RunningWorker drops returning 3 KB → pool: 3 KB. - // acquire_bytes = 3+2 = 5 KB → gap = 2 KB → evicts idle Worker A (2 KB). - // Pool after eviction: 5 KB → acquire 5 KB → release 2 KB → pool: 2 KB free. - // C's pending 2 KB write re-acquires → succeeds. - // Worker B (1 KB) must NOT be evicted. - executor - .invoke_and_await_agent( - &component, - &agent_c, - "write_file", - data_value!("/file-c2.txt", content_c2.as_str()), - ) - .await?; - - // Verify Worker B was NOT evicted — it should still be idle in memory. - // We check its status is Idle (loaded, not suspended/failed) before reading, - // proving eviction stopped after freeing only the gap (2 KB from A), - // not B's 1 KB as well. - let metadata_b = executor.get_worker_metadata(&worker_b).await?; - assert_eq!( - metadata_b.status, - golem_common::model::AgentStatus::Idle, - "Worker B must remain Idle (not evicted) — only Worker A's 2 KB should have been evicted" - ); - - // Verify Worker B's file content is intact. - let read_b = executor - .invoke_and_await_agent( - &component, - &agent_b, - "read_file", - data_value!("/file-b.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - read_b, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String(content_b))) - }) - ); - - // Verify Worker C's first file — confirms prior state survived the eviction/restart. - let read_c1 = executor - .invoke_and_await_agent( - &component, - &agent_c, - "read_file", - data_value!("/file-c1.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - read_c1, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String(content_c1))) - }) - ); - - // Verify Worker C's second file — the write that triggered eviction. - let read_c2 = executor - .invoke_and_await_agent( - &component, - &agent_c, - "read_file", - data_value!("/file-c2.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - read_c2, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String(content_c2))) - }) - ); - - executor.check_oplog_is_queryable(&worker_a).await?; - executor.check_oplog_is_queryable(&worker_b).await?; - executor.check_oplog_is_queryable(&worker_c).await?; - - Ok(()) -} - -/// Full lifecycle: write → delete → interrupt → restart → verify pool accounting. -/// -/// Verifies that `current_filesystem_storage_usage` is reconstructed correctly from the oplog -/// across an interrupt/restart cycle, and that the executor semaphore reflects the -/// reconstructed value accurately. -/// -/// Pool = 2 KB (2 permits). Each 11-byte "hello world" write rounds up to 1 KB. -/// -/// Worker A: -/// 1. Writes file-1 (1 KB) → pool: 1 KB used. -/// 2. Writes file-2 (1 KB) → pool: 2 KB used, exhausted. -/// 3. Deletes file-1 → pool: 1 KB used. FilesystemStorageUsageUpdate(-1KB) written to oplog. -/// 4. Interrupted → RunningWorker drops → 1 KB permit returned → pool: 1 KB free. -/// -/// Worker B (different agent, same pool): -/// 5. Writes 1 KB → should succeed because pool has 1 KB free. -/// If A's current_filesystem_storage_usage was wrong (e.g. 2 KB instead of 1 KB), A would -/// have consumed 2 KB on restart pre-acquire, leaving 0 KB free, and B would fail. -/// -/// Worker A re-invoked: -/// 6. Restart reconstructs current_filesystem_storage_usage = 1 KB from oplog (+1+1-1). -/// Pre-acquires 1 KB from the pool. Pool: 0 KB free. -/// Reads file-2 to confirm durable state is intact. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn executor_pool_storage_usage_survives_restart( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 2 KB pool. - let executor = start_with_executor_storage_pool(deps, &context, 2 * 1024).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_a = agent_id!("FileSystem", "lifecycle-a-1"); - let agent_b = agent_id!("FileSystem", "lifecycle-b-1"); - let worker_a = executor.start_agent(&component.id, agent_a.clone()).await?; - - // Step 1: Worker A writes file-1 → 1 KB used, 1 KB free. - executor - .invoke_and_await_agent( - &component, - &agent_a, - "write_file", - data_value!("/file-1.txt", "content-a-file1"), - ) - .await?; - - // Step 2: Worker A writes file-2 → pool exhausted (2 KB used). - executor - .invoke_and_await_agent( - &component, - &agent_a, - "write_file", - data_value!("/file-2.txt", "content-a-file2"), - ) - .await?; - - // Step 3: Worker A deletes file-1 → 1 KB freed. FilesystemStorageUsageUpdate(-1 KB) → oplog. - executor - .invoke_and_await_agent( - &component, - &agent_a, - "delete_file", - data_value!("/file-1.txt"), - ) - .await?; - - // Step 4: Interrupt Worker A → RunningWorker drops → 1 KB permit returned. - // Pool now: 1 KB free. - executor.interrupt(&worker_a).await?; - - // Step 5: Worker B writes 1 KB. Pool has 1 KB free → must succeed. - // If Worker A's current_filesystem_storage_usage was incorrectly reconstructed as 2 KB - // (missing the delete delta), its restart would pre-acquire 2 KB leaving 0 KB - // free, and Worker B's write would fail. - let worker_b = executor.start_agent(&component.id, agent_b.clone()).await?; - executor - .invoke_and_await_agent( - &component, - &agent_b, - "write_file", - data_value!("/file-b.txt", "content-b-file1"), - ) - .await?; - - // Verify Worker B's file has the correct unique content. - let content_b = executor - .invoke_and_await_agent( - &component, - &agent_b, - "read_file", - data_value!("/file-b.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - content_b, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String("content-b-file1".to_string()))) - }) - ); - - // Step 6: Re-invoke Worker A. Restart reconstructs current_filesystem_storage_usage = 1 KB. - // Reads file-2 — confirms durable state survived the interrupt with correct content. - // file-1 was deleted; file-2 must still have its distinct original content. - let content_a = executor - .invoke_and_await_agent( - &component, - &agent_a, - "read_file", - data_value!("/file-2.txt"), - ) - .await? - .into_return_value() - .ok_or_else(|| anyhow::anyhow!("expected return value from read_file"))?; - assert_eq!( - content_a, - SchemaValue::Result(ResultValuePayload::Ok { - value: Some(Box::new(SchemaValue::String("content-a-file2".to_string()))) - }) - ); - - executor.check_oplog_is_queryable(&worker_a).await?; - executor.check_oplog_is_queryable(&worker_b).await?; - - Ok(()) -} - -/// `blocking_splice` from an input stream to a file-backed output stream must -/// be subject to storage quota. After writing an 11-byte file that nearly -/// exhausts the 20-byte quota, splicing those 11 bytes into a second file -/// (total 22 bytes) must fail with `AgentExceededFilesystemStorageLimit`. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_blocking_splice_exceeding_limit_fails( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - // 20-byte quota — "hello world" (11 bytes) fits once but not twice. - let executor = start_with_agent_storage_quota(deps, &context, 20).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "blocking-splice-quota-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - // Write the source file — consumes 11 of 20 bytes. - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/src.txt", "hello world"), - ) - .await?; - - // Splice source into a new destination — needs 11 more bytes (total 22), - // which exceeds the 20-byte quota. - let result = executor - .invoke_and_await_agent( - &component, - &agent_id, - "blocking_splice", - data_value!("/src.txt", "/dst.txt"), - ) - .await; - - assert!( - result.is_err(), - "expected blocking_splice to fail when total bytes exceed agent quota" - ); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected AgentExceededFilesystemStorageLimit, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -/// Same as `agent_quota_blocking_splice_exceeding_limit_fails` but exercises -/// the non-blocking `output-stream.splice` path. -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn agent_quota_splice_exceeding_limit_fails( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start_with_agent_storage_quota(deps, &context, 20).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "splice-quota-1"); - let worker_id = executor - .start_agent(&component.id, agent_id.clone()) - .await?; - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/src.txt", "hello world"), - ) - .await?; - - let result = executor - .invoke_and_await_agent( - &component, - &agent_id, - "splice", - data_value!("/src.txt", "/dst.txt"), - ) - .await; - - assert!( - result.is_err(), - "expected splice to fail when total bytes exceed agent quota" - ); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected AgentExceededFilesystemStorageLimit, got: {err_str}" - ); - - executor.check_oplog_is_queryable(&worker_id).await?; - - Ok(()) -} - -#[test] -#[tracing::instrument] -#[timeout("2m")] -async fn provisioned_read_write_file_counts_toward_agent_quota( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - _tracing: &Tracing, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, -) -> anyhow::Result<()> { - let context = TestContext::new(last_unique_id); - let executor = start_with_agent_storage_quota(deps, &context, 3).await?; - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .with_files( - "FileSystem", - &[IFSEntry { - source_path: PathBuf::from("initial-file-system/files/baz.txt"), - target_path: CanonicalFilePath::from_abs_str("/bar/baz.txt").unwrap(), - permissions: AgentFilePermissions::ReadWrite, - }], - ) - .store() - .await?; - - let result = executor - .start_agent( - &component.id, - agent_id!("FileSystem", "provisioned-file-over-quota"), - ) - .await; - - let error = result.expect_err( - "4-byte provisioned read-write file must prevent creation under a 3-byte quota", - ); - let message = error.to_string(); - assert!( - message.contains( - "Provisioned read-write files require 4 bytes, exceeding the per-agent disk limit of 3 bytes" - ), - "unexpected worker creation error: {message}" - ); - Ok(()) -} diff --git a/golem-worker-executor/tests/wasi.rs b/golem-worker-executor/tests/wasi.rs index dcdf65f903..e305e01cb7 100644 --- a/golem-worker-executor/tests/wasi.rs +++ b/golem-worker-executor/tests/wasi.rs @@ -20,22 +20,31 @@ use axum::{BoxError, Router}; use bytes::Bytes; use futures::stream; use golem_common::agent_id; +use golem_common::model::OwnedAgentId; use golem_common::model::component::{AgentFilePermissions, CanonicalFilePath}; use golem_common::model::worker::{ - AgentConfigEntryDto, AgentFileSystemNode, AgentFileSystemNodeKind, + AgentConfigEntryDto, AgentFileSystemNode, AgentFileSystemNodeKind, RevertToOplogIndex, + RevertWorkerTarget, }; -use golem_common::model::{AgentStatus, IdempotencyKey, RetryConfig}; +use golem_common::model::{AgentStatus, IdempotencyKey, OplogIndex, RetryConfig}; use golem_common::schema::SchemaValue; use golem_common::schema::schema_value::ResultValuePayload; -use golem_test_framework::dsl::{TestDsl, drain_connection, stderr_events, stdout_events}; -use golem_test_framework::model::IFSEntry; -use golem_worker_executor::metrics::storage::{ - STORAGE_BYTES_WRITTEN_TOTAL, STORAGE_TYPE_FILESYSTEM, +use golem_test_framework::dsl::{ + TestDsl, count_agent_invocation_pair_since, drain_connection, stderr_events, stdout_events, }; +use golem_test_framework::model::IFSEntry; +use golem_worker_executor::services::golem_config::SnapshotPolicy; +#[cfg(target_os = "linux")] +use golem_worker_executor_test_utils::start_with_agent_storage_quota_on_managed_xfs; use golem_worker_executor_test_utils::{ - LastUniqueId, PrecompiledComponent, TestContext, TestExecutorOverrides, + LastUniqueId, PrecompiledComponent, TestContext, TestExecutorOverrides, TestWorkerExecutor, WorkerExecutorTestDependencies, start, start_with_overrides, }; +#[cfg(target_os = "linux")] +use golem_worker_executor_test_utils::{ + start_with_agent_storage_and_object_quota_on_managed_xfs, + start_with_mutable_agent_storage_quota_on_managed_xfs, +}; use http::{HeaderMap, StatusCode}; use pretty_assertions::assert_eq; use std::collections::{BTreeMap, HashMap}; @@ -80,6 +89,56 @@ fn sorted_config_entries(value: SchemaValue) -> SchemaValue { SchemaValue::List { elements } } +fn schema_string_list(result: SchemaValue) -> Vec { + let SchemaValue::List { elements } = result else { + panic!("expected list, got {result:?}") + }; + elements + .into_iter() + .map(|element| match element { + SchemaValue::String(entry) => entry, + other => panic!("expected string, got {other:?}"), + }) + .collect() +} + +async fn assert_reconstructed_writable_file( + executor: &TestWorkerExecutor, + component: &golem_common::base_model::component::ComponentDto, + agent_id: &golem_common::model::agent::ParsedAgentId, +) -> anyhow::Result<()> { + let result = executor + .invoke_and_await_agent( + component, + agent_id, + "inspect_writable", + golem_common::data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value"))?; + assert_eq!( + schema_string_list(result), + vec![ + "p2_read=p3-to-p2".to_string(), + "p3_read=p3-to-p2".to_string(), + ] + ); + Ok(()) +} + +fn full_replay_config(config: &mut golem_worker_executor::services::golem_config::GolemConfig) { + config.oplog.default_snapshotting = SnapshotPolicy::Disabled; + config.oplog.oplog_processor_snapshotting = SnapshotPolicy::Disabled; +} + +#[cfg(target_os = "linux")] +fn managed_xfs_test_root() -> PathBuf { + std::env::var_os("GOLEM_MANAGED_XFS_TEST_ROOT") + .map(PathBuf::from) + .expect("GOLEM_MANAGED_XFS_TEST_ROOT must name the mounted XFS test root") +} + #[test] #[tracing::instrument] async fn write_stdout( @@ -298,198 +357,1457 @@ async fn file_write_read_delete( .invoke_and_await_agent( &component, &agent_id, - "run_file_write_read_delete", - data_value!(), + "run_file_write_read_delete", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value"))?; + + executor.check_oplog_is_queryable(&worker_id).await?; + + assert_eq!( + result, + SchemaValue::Record { + fields: vec![ + SchemaValue::Option { inner: None }, + SchemaValue::Option { + inner: Some(Box::new(SchemaValue::String("hello world".to_string()))) + }, + SchemaValue::Option { inner: None } + ] + } + ); + + Ok(()) +} + +#[test] +#[tracing::instrument] +async fn initial_file_read_write( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + use golem_common::{agent_id, data_value}; + + let context = TestContext::new(last_unique_id); + let executor = start(deps, &context).await?; + + let component = executor + .component_dep(&context.default_environment_id, initial_file_system) + .with_files( + "FileReadWrite", + &[ + IFSEntry { + source_path: PathBuf::from("initial-file-system/files/foo.txt"), + target_path: CanonicalFilePath::from_abs_str("/foo.txt").unwrap(), + permissions: AgentFilePermissions::ReadOnly, + }, + IFSEntry { + source_path: PathBuf::from("initial-file-system/files/baz.txt"), + target_path: CanonicalFilePath::from_abs_str("/bar/baz.txt").unwrap(), + permissions: AgentFilePermissions::ReadWrite, + }, + ], + ) + .store() + .await?; + + let mut env = HashMap::new(); + env.insert("RUST_BACKTRACE".to_string(), "full".to_string()); + let agent_id = agent_id!("FileReadWrite", "initial-file-read-write-1"); + let worker_id = executor + .start_agent_with(&component.id, agent_id.clone(), env, Vec::new()) + .await?; + + let result = executor + .invoke_and_await_agent(&component, &agent_id, "run", data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value"))?; + + executor.check_oplog_is_queryable(&worker_id).await?; + + assert_eq!( + result, + SchemaValue::Tuple { + elements: vec![ + SchemaValue::Option { + inner: Some(Box::new(SchemaValue::String("foo\n".to_string()))) + }, + SchemaValue::Option { inner: None }, + SchemaValue::Option { inner: None }, + SchemaValue::Option { + inner: Some(Box::new(SchemaValue::String("baz\n".to_string()))) + }, + SchemaValue::Option { + inner: Some(Box::new(SchemaValue::String("hello world".to_string()))) + }, + ] + } + ); + + Ok(()) +} + +#[test] +#[tracing::instrument] +async fn initial_file_p3_parity( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + initial_file_p3_parity_with_backend(last_unique_id, deps, initial_file_system, None).await +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[tracing::instrument] +async fn initial_file_p2_p3_parity_on_managed_xfs( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + let root = managed_xfs_test_root(); + initial_file_p3_parity_with_backend(last_unique_id, deps, initial_file_system, Some(root)).await +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[tracing::instrument] +async fn p2_p3_quota_exhaustion_on_managed_xfs( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + use golem_common::{agent_id, data_value}; + + let root = managed_xfs_test_root(); + let context = TestContext::new(last_unique_id); + let executor = + start_with_agent_storage_quota_on_managed_xfs(deps, &context, 1024 * 1024, root).await?; + let component = executor + .component_dep(&context.default_environment_id, initial_file_system) + .store() + .await?; + let p2_agent = agent_id!("P3FileSystem", "managed-quota-p2"); + + let result = executor + .invoke_and_await_agent(&component, &p2_agent, "run_p2_quota_surface", data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value"))?; + let SchemaValue::List { elements } = result else { + panic!("expected list, got {result:?}") + }; + assert_eq!( + elements, + vec![ + SchemaValue::String("p2_wrote_before_limit=true".to_string()), + SchemaValue::String("p2_growth_denied=true".to_string()), + ] + ); + + let p3_within_quota_agent = agent_id!("P3FileSystem", "managed-quota-p3-within-limit"); + let p3_within_quota = executor + .invoke_and_await_agent( + &component, + &p3_within_quota_agent, + "run_p3_with_quota", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value"))?; + assert_eq!(p3_within_quota, SchemaValue::Bool(true)); + + let p2_exhaustion_agent = agent_id!("P3FileSystem", "managed-quota-p2-exhaustion"); + let p2_exhaustion_worker = executor + .start_agent(&component.id, p2_exhaustion_agent.clone()) + .await?; + let p2_exhaustion = schema_string_list( + executor + .invoke_and_await_agent( + &component, + &p2_exhaustion_agent, + "exhaust_p2_quota", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected P2 exhaustion result"))?, + ); + assert_eq!( + p2_exhaustion.first().map(String::as_str), + Some("completion=err:quota"), + "P2 growth must fail specifically because the project quota is exhausted" + ); + assert_eq!( + p2_exhaustion.get(1).map(String::as_str), + Some("prefix-persisted=true") + ); + let persisted_p2_prefix = schema_string_list( + executor + .invoke_and_await_agent( + &component, + &p2_exhaustion_agent, + "inspect_p2_exhaustion", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected persisted P2 failure prefix"))?, + ); + let p2_size = persisted_p2_prefix + .first() + .and_then(|size| size.strip_prefix("size=")) + .and_then(|size| size.parse::().ok()) + .expect("P2 exhaustion inspection must report a numeric size"); + assert!((4096..=1024 * 1024).contains(&p2_size)); + assert_eq!( + &persisted_p2_prefix[1..], + ["prefix-complete=true", "suffix-bytes=0"] + ); + executor.simulated_crash(&p2_exhaustion_worker).await?; + let reconstructed_p2_prefix = executor + .invoke_and_await_agent( + &component, + &p2_exhaustion_agent, + "inspect_p2_exhaustion", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected reconstructed P2 failure prefix"))?; + assert_eq!( + schema_string_list(reconstructed_p2_prefix), + persisted_p2_prefix + ); + + let p3_exhaustion_agent = agent_id!("P3FileSystem", "managed-quota-p3-exhaustion"); + let p3_exhaustion_worker = executor + .start_agent(&component.id, p3_exhaustion_agent.clone()) + .await?; + let p3_exhaustion = executor + .invoke_and_await_agent( + &component, + &p3_exhaustion_agent, + "exhaust_p3_quota", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected P3 exhaustion result"))?; + let p3_exhaustion = schema_string_list(p3_exhaustion); + assert_eq!( + p3_exhaustion.first().map(String::as_str), + Some("completion=err:quota"), + "P3 growth must fail specifically because the project quota is exhausted" + ); + let unwritten_bytes = p3_exhaustion + .get(2) + .and_then(|value| value.strip_prefix("unwritten-bytes=")) + .ok_or_else(|| anyhow!("P3 stream input result was not reported: {p3_exhaustion:?}"))? + .parse::()?; + assert_eq!( + p3_exhaustion.get(1).map(String::as_str), + Some("prefix-persisted=true"), + "P3 stream must acknowledge data persisted before quota denial" + ); + assert!( + unwritten_bytes > 0, + "P3 quota failure must return the input suffix that was not persisted" + ); + let persisted_prefix = executor + .invoke_and_await_agent( + &component, + &p3_exhaustion_agent, + "inspect_p3_exhaustion", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected persisted P3 failure prefix"))?; + let persisted_prefix = schema_string_list(persisted_prefix); + assert_eq!( + persisted_prefix.get(1).map(String::as_str), + Some("prefix-complete=true") + ); + executor.simulated_crash(&p3_exhaustion_worker).await?; + let reconstructed_prefix = executor + .invoke_and_await_agent( + &component, + &p3_exhaustion_agent, + "inspect_p3_exhaustion", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected reconstructed P3 failure prefix"))?; + assert_eq!(schema_string_list(reconstructed_prefix), persisted_prefix); + + for (agent, method, expected) in [ + ( + agent_id!("P3FileSystem", "managed-quota-p2-matrix"), + "run_p2_quota_matrix", + vec![ + "direct=true", + "positioned-stream=true", + "append=true", + "sparse-resize=true", + "overwrite=true", + "truncate=true", + "splice=true", + "hard-link=true", + "first-unlink=true", + "open-unlinked=true", + "rename-replace=true", + "grew=true", + "growth-denied=true", + ], + ), + ( + agent_id!("P3FileSystem", "managed-quota-p3-matrix"), + "run_p3_quota_matrix", + vec![ + "positioned-stream=true", + "append=true", + "sparse-resize=true", + "overwrite=true", + "truncate=true", + "hard-link=true", + "first-unlink=true", + "open-unlinked=true", + "rename-replace=true", + "growth-denied=true", + ], + ), + ] { + let result = executor + .invoke_and_await_agent(&component, &agent, method, data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value from {method}"))?; + assert_eq!(schema_string_list(result), expected); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn fill_filesystem_leaving(path: &std::path::Path, reserve_bytes: u64) -> anyhow::Result<()> { + use std::io::Write; + + let mut filler = std::fs::File::create(path)?; + let block = vec![0x7f; 1024 * 1024]; + let mut length = 0u64; + loop { + match filler.write(&block) { + Ok(0) => break, + Ok(written) => length += written as u64, + Err(error) if error.raw_os_error() == Some(28) => break, + Err(error) => return Err(error.into()), + } + } + filler.sync_all()?; + filler.set_len(length.saturating_sub(reserve_bytes))?; + filler.sync_all()?; + Ok(()) +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[timeout("2m")] +#[tracing::instrument] +async fn p2_p3_mid_effect_enospc_reconstructs_on_unmanaged_filesystem( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + use golem_common::{agent_id, data_value}; + + let mount_root = managed_xfs_test_root(); + let runtime_root = mount_root.join("unmanaged-mid-effect-runtime"); + let filler_path = mount_root.join("unmanaged-mid-effect-filler"); + let context = TestContext::new(last_unique_id); + let executor = start_with_overrides( + deps, + &context, + TestExecutorOverrides { + configure: Some(Arc::new(move |config| { + config.filesystem_storage.deterministic_root_dir = Some(runtime_root.clone()); + full_replay_config(config); + })), + ..TestExecutorOverrides::default() + }, + ) + .await?; + let component = executor + .component_dep(&context.default_environment_id, initial_file_system) + .store() + .await?; + + for (agent_id, exhaust_method, inspect_method) in [ + ( + agent_id!("P3FileSystem", "unmanaged-enospc-p2"), + "exhaust_p2_quota", + "inspect_p2_exhaustion", + ), + ( + agent_id!("P3FileSystem", "unmanaged-enospc-p3"), + "exhaust_p3_quota", + "inspect_p3_exhaustion", + ), + ] { + let worker_id = executor + .start_agent(&component.id, agent_id.clone()) + .await?; + fill_filesystem_leaving(&filler_path, 256 * 1024)?; + let failure = schema_string_list( + executor + .invoke_and_await_agent(&component, &agent_id, exhaust_method, data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected {exhaust_method} result"))?, + ); + assert_eq!( + failure.get(1).map(String::as_str), + Some("prefix-persisted=true") + ); + std::fs::remove_file(&filler_path)?; + + let persisted = schema_string_list( + executor + .invoke_and_await_agent(&component, &agent_id, inspect_method, data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected {inspect_method} result"))?, + ); + assert_eq!( + persisted.get(1).map(String::as_str), + Some("prefix-complete=true") + ); + assert!( + persisted + .first() + .and_then(|entry| entry.strip_prefix("size=")) + .and_then(|size| size.parse::().ok()) + .is_some_and(|size| (4096..2 * 1024 * 1024 + 4096).contains(&size)), + "{exhaust_method} did not retain only its completed prefix: {persisted:?}" + ); + + executor.simulated_crash(&worker_id).await?; + let reconstructed = executor + .invoke_and_await_agent(&component, &agent_id, inspect_method, data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected reconstructed {inspect_method} result"))?; + assert_eq!(schema_string_list(reconstructed), persisted); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[tracing::instrument] +async fn p2_p3_object_quota_on_managed_xfs( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + use golem_common::{agent_id, data_value}; + + let root = managed_xfs_test_root(); + let context = TestContext::new(last_unique_id); + let executor = start_with_agent_storage_and_object_quota_on_managed_xfs( + deps, + &context, + 16 * 1024 * 1024, + 32, + root, + ) + .await?; + let component = executor + .component_dep(&context.default_environment_id, initial_file_system) + .store() + .await?; + let expected = vec![ + "hard-link-same-inode=true", + "object-denied=true", + "directory-denied=true", + "symlink-denied=true", + "denied-while-open=true", + ]; + for (agent, method, completion_method) in [ + ( + agent_id!("P3FileSystem", "managed-object-quota-p2"), + "run_p2_object_quota", + "complete_p2_object_quota_release", + ), + ( + agent_id!("P3FileSystem", "managed-object-quota-p3"), + "run_p3_object_quota", + "complete_p3_object_quota_release", + ), + ] { + let result = executor + .invoke_and_await_agent(&component, &agent, method, data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value from {method}"))?; + assert_eq!(schema_string_list(result), expected); + let admitted_after_close = executor + .invoke_and_await_agent(&component, &agent, completion_method, data_value!()) + .await? + .into_return_value(); + assert_eq!(admitted_after_close, Some(SchemaValue::Bool(true))); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[timeout("2m")] +#[tracing::instrument] +async fn filesystem_downgrade_blocks_guest_until_limit_recovers( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + use golem_common::{agent_id, data_value}; + + let root = managed_xfs_test_root(); + let context = TestContext::new(last_unique_id); + let (executor, quota) = start_with_mutable_agent_storage_quota_on_managed_xfs( + deps, + &context, + 1024 * 1024, + root.clone(), + ) + .await?; + let component = executor + .component_dep(&context.default_environment_id, initial_file_system) + .with_files( + "P3FileSystem", + &[ + IFSEntry { + source_path: PathBuf::from("initial-file-system/files/foo.txt"), + target_path: CanonicalFilePath::from_abs_str("/foo.txt").unwrap(), + permissions: AgentFilePermissions::ReadOnly, + }, + IFSEntry { + source_path: PathBuf::from("initial-file-system/files/foo.txt"), + target_path: CanonicalFilePath::from_abs_str("/foo-copy.txt").unwrap(), + permissions: AgentFilePermissions::ReadOnly, + }, + IFSEntry { + source_path: PathBuf::from("initial-file-system/files/baz.txt"), + target_path: CanonicalFilePath::from_abs_str("/bar/baz.txt").unwrap(), + permissions: AgentFilePermissions::ReadWrite, + }, + ], + ) + .store() + .await?; + let agent = agent_id!("P3FileSystem", "managed-quota-downgrade"); + let worker_id = executor.start_agent(&component.id, agent.clone()).await?; + let initial_result = executor + .invoke_and_await_agent( + &component, + &agent, + "confirm_invocation_started", + data_value!(), + ) + .await?; + assert_eq!( + initial_result.into_return_value(), + Some(SchemaValue::String("executed".to_string())) + ); + let runtime_path = root + .join(context.default_environment_id.to_string()) + .join(component.id.to_string()) + .join(worker_id.agent_name_encoded()); + assert!(runtime_path.exists()); + + quota.set_limit(4096).await?; + executor + .wait_for_status(&worker_id, AgentStatus::Suspended, Duration::from_secs(10)) + .await?; + tokio::time::timeout(Duration::from_secs(30), async { + while runtime_path.exists() { + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("over-limit runtime filesystem was not deleted"); + + let before_blocked_invocation = executor.oplog_max_index(&worker_id).await?; + let blocked_key = IdempotencyKey::fresh(); + executor + .invoke_agent_with_key( + &component, + &agent, + &blocked_key, + "confirm_invocation_started", + data_value!(), + ) + .await?; + let blocked = tokio::time::timeout( + Duration::from_secs(1), + executor.invoke_and_await_agent_with_key( + &component, + &agent, + &blocked_key, + "confirm_invocation_started", + data_value!(), + ), + ) + .await; + assert!( + !matches!(blocked, Ok(Ok(_))), + "pending invocation completed while reconstruction exceeded its installed quota" + ); + let blocked_oplog = executor.get_oplog(&worker_id, OplogIndex::INITIAL).await?; + assert_eq!( + count_agent_invocation_pair_since(&blocked_oplog, before_blocked_invocation), + (0, 0), + "over-limit reconstruction must not start the pending guest invocation" + ); + + quota.set_limit(1024 * 1024).await?; + executor.resume(&worker_id, false).await?; + let recovered = tokio::time::timeout( + Duration::from_secs(20), + executor.invoke_and_await_agent_with_key( + &component, + &agent, + &blocked_key, + "confirm_invocation_started", + data_value!(), + ), + ) + .await + .expect("pending invocation did not recover after raising the limit")?; + assert_eq!( + recovered.into_return_value(), + Some(SchemaValue::String("executed".to_string())) + ); + executor + .wait_for_status(&worker_id, AgentStatus::Idle, Duration::from_secs(10)) + .await?; + assert!(runtime_path.exists()); + let recovered_oplog = executor.get_oplog(&worker_id, OplogIndex::INITIAL).await?; + assert_eq!( + count_agent_invocation_pair_since(&recovered_oplog, before_blocked_invocation), + (1, 1) + ); + Ok(()) +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[timeout("2m")] +#[tracing::instrument] +async fn managed_xfs_resource_billing_survives_idle_and_replay( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + use golem_common::{agent_id, data_value}; + + let context = TestContext::new(last_unique_id); + let (executor, billing) = start_with_mutable_agent_storage_quota_on_managed_xfs( + deps, + &context, + 1024 * 1024, + managed_xfs_test_root(), + ) + .await?; + let component = executor + .component_dep(&context.default_environment_id, initial_file_system) + .store() + .await?; + let agent = agent_id!("P3FileSystem", "managed-xfs-resource-billing"); + let worker_id = executor.start_agent(&component.id, agent.clone()).await?; + + let mutation = executor + .invoke_and_await_agent(&component, &agent, "run_writable", data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected run_writable return value"))?; + assert_eq!( + schema_string_list(mutation), + vec![ + "p2_write_p3_read=p2-to-p3".to_string(), + "p3_write_p2_read=p3-to-p2".to_string(), + ] + ); + executor + .wait_for_status(&worker_id, AgentStatus::Idle, Duration::from_secs(10)) + .await?; + + let after_mutation = billing.flush_durable_storage_byte_seconds(); + let mut after_active_window = after_mutation; + for _ in 0..32 { + assert_reconstructed_writable_file(&executor, &component, &agent).await?; + executor + .wait_for_status(&worker_id, AgentStatus::Idle, Duration::from_secs(10)) + .await?; + after_active_window = billing.flush_durable_storage_byte_seconds(); + if after_active_window > after_mutation { + break; + } + } + assert!( + after_active_window > after_mutation, + "authoritative managed-XFS allocation remained zero-billed across active windows" + ); + + let mut idle_start = billing.flush_durable_storage_byte_seconds(); + let mut stable_samples = 0; + while stable_samples < 3 { + tokio::time::sleep(Duration::from_millis(100)).await; + let current = billing.flush_durable_storage_byte_seconds(); + if current == idle_start { + stable_samples += 1; + } else { + idle_start = current; + stable_samples = 0; + } + } + tokio::time::sleep(Duration::from_secs(1)).await; + let idle_end = billing.flush_durable_storage_byte_seconds(); + assert_eq!( + idle_end, idle_start, + "loaded-idle managed-XFS storage continued billing" + ); + + executor.simulated_crash(&worker_id).await?; + assert_reconstructed_writable_file(&executor, &component, &agent).await?; + executor + .wait_for_status(&worker_id, AgentStatus::Idle, Duration::from_secs(10)) + .await?; + let after_replay = billing.flush_durable_storage_byte_seconds(); + assert!( + after_replay > idle_end, + "replay and its following invocation produced no managed-XFS storage billing" + ); + + let before_delete = billing.flush_durable_storage_byte_seconds(); + executor.delete_worker(&worker_id).await?; + let after_delete = billing.flush_durable_storage_byte_seconds(); + assert!(after_delete >= before_delete); + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + billing.flush_durable_storage_byte_seconds(), + after_delete, + "deleted managed-XFS storage continued billing" + ); + Ok(()) +} + +async fn initial_file_p3_parity_with_backend( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + initial_file_system: &PrecompiledComponent, + managed_xfs_root: Option, +) -> anyhow::Result<()> { + use golem_common::{agent_id, data_value}; + + let context = TestContext::new(last_unique_id); + let executor = match managed_xfs_root { + Some(root) => { + start_with_overrides( + deps, + &context, + TestExecutorOverrides { + configure: Some(Arc::new(move |config| { + config.filesystem_storage.managed_xfs_root_dir = Some(root.clone()); + })), + ..TestExecutorOverrides::default() + }, + ) + .await? + } + None => start(deps, &context).await?, + }; + + let component = executor + .component_dep(&context.default_environment_id, initial_file_system) + .with_files( + "P3FileSystem", + &[ + IFSEntry { + source_path: PathBuf::from("initial-file-system/files/foo.txt"), + target_path: CanonicalFilePath::from_abs_str("/foo.txt").unwrap(), + permissions: AgentFilePermissions::ReadOnly, + }, + IFSEntry { + source_path: PathBuf::from("initial-file-system/files/foo.txt"), + target_path: CanonicalFilePath::from_abs_str("/foo-copy.txt").unwrap(), + permissions: AgentFilePermissions::ReadOnly, + }, + IFSEntry { + source_path: PathBuf::from("initial-file-system/files/baz.txt"), + target_path: CanonicalFilePath::from_abs_str("/bar/baz.txt").unwrap(), + permissions: AgentFilePermissions::ReadWrite, + }, + ], + ) + .store() + .await?; + + let agent_id = agent_id!("P3FileSystem", "initial-file-p3-parity-1"); + let worker_id = executor + .start_agent(&component.id, agent_id.clone()) + .await?; + + let abandoned_completion = executor + .invoke_and_await_agent( + &component, + &agent_id, + "abandon_p3_write_completion", + data_value!(), + ) + .await? + .into_return_value(); + assert_eq!(abandoned_completion, Some(SchemaValue::Bool(true))); + + let expected = vec![ + "ro_flags_p2_write=false".to_string(), + "ro_flags_p3_write=false".to_string(), + "ro_hash_parity=true".to_string(), + "ro_hash_p3_deterministic=true".to_string(), + "ro_hash_at_parity=true".to_string(), + "ro_set_times_p2=err:not-permitted".to_string(), + "ro_set_times_p3=err:not-permitted".to_string(), + "ro_set_times_at_p2=err:not-permitted".to_string(), + "ro_set_times_at_p3=err:not-permitted".to_string(), + "ro_rename_at_p2=err:not-permitted".to_string(), + "ro_rename_at_p3=err:not-permitted".to_string(), + "ro_symlink_at_p2=err:not-permitted".to_string(), + "ro_symlink_at_p3=err:not-permitted".to_string(), + "ro_unlink_file_at_p2=err:not-permitted".to_string(), + "ro_unlink_file_at_p3=err:not-permitted".to_string(), + "ro_parent_open_write_p2=err:not-permitted".to_string(), + "ro_parent_open_write_p3=err:not-permitted".to_string(), + "ro_parent_unlink_p2=err:not-permitted".to_string(), + "ro_parent_unlink_p3=err:not-permitted".to_string(), + "ro_parent_rename_p2=err:not-permitted".to_string(), + "ro_parent_rename_p3=err:not-permitted".to_string(), + "ro_parent_link_p2=err:not-permitted".to_string(), + "ro_parent_link_p3=err:not-permitted".to_string(), + "ro_alias_create_p2=ok".to_string(), + "ro_alias_open_write_p2=err:not-permitted".to_string(), + "ro_alias_unlink_p2=ok".to_string(), + "ro_alias_create_p3=ok".to_string(), + "ro_alias_open_write_p3=err:not-permitted".to_string(), + "ro_alias_unlink_p3=ok".to_string(), + "rw_flags_p2_write=true".to_string(), + "rw_flags_p3_write=true".to_string(), + "rw_hash_parity=true".to_string(), + "rw_set_times_p2=ok".to_string(), + "rw_set_times_p3=ok".to_string(), + "p2_write_p3_read=p2-to-p3".to_string(), + "p3_write_p2_read=p3-to-p2".to_string(), + ]; + + let result = executor + .invoke_and_await_agent(&component, &agent_id, "run", data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value"))?; + + executor.check_oplog_is_queryable(&worker_id).await?; + + assert_eq!(schema_string_list(result), expected); + + // Crash the worker so the next invocation replays the recorded oplog first, + // verifying the P3 metadata-hash -> durable stat call sequence is replay-stable + // and that replay reconstructed the mutable initial file without help from + // the verification invocation. + executor.simulated_crash(&worker_id).await?; + + let result_after_crash = executor + .invoke_and_await_agent(&component, &agent_id, "inspect_run", data_value!()) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected return value"))?; + + executor.check_oplog_is_queryable(&worker_id).await?; + + assert_eq!( + schema_string_list(result_after_crash), + vec![ + "p2_read=p3-to-p2".to_string(), + "p3_read=p3-to-p2".to_string(), + ] + ); + + Ok(()) +} + +#[test] +#[timeout("2m")] +#[tracing::instrument] +async fn filesystem_mutation_histories_reconstruct_from_full_replay( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + filesystem_mutation_histories_reconstruct_with_backend( + last_unique_id, + deps, + initial_file_system, + None, + ) + .await +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[timeout("2m")] +#[tracing::instrument] +async fn filesystem_mutation_histories_reconstruct_on_managed_xfs( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + filesystem_mutation_histories_reconstruct_with_backend( + last_unique_id, + deps, + initial_file_system, + Some(managed_xfs_test_root()), + ) + .await +} + +async fn filesystem_mutation_histories_reconstruct_with_backend( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + initial_file_system: &PrecompiledComponent, + managed_xfs_root: Option, +) -> anyhow::Result<()> { + use golem_common::{agent_id, data_value}; + + let context = TestContext::new(last_unique_id); + let executor = start_with_overrides( + deps, + &context, + TestExecutorOverrides { + configure: Some(Arc::new(move |config| { + if let Some(root) = &managed_xfs_root { + config.filesystem_storage.managed_xfs_root_dir = Some(root.clone()); + } + full_replay_config(config); + })), + ..TestExecutorOverrides::default() + }, + ) + .await?; + let component = executor + .component_dep(&context.default_environment_id, initial_file_system) + .store() + .await?; + let agent_id = agent_id!("P3FileSystem", "filesystem-reconstruction-matrix"); + let worker_id = executor + .start_agent(&component.id, agent_id.clone()) + .await?; + + let expected = schema_string_list( + executor + .invoke_and_await_agent( + &component, + &agent_id, + "run_reconstruction_matrix", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected reconstruction matrix"))?, + ); + assert_eq!( + expected, + vec![ + "p2-resize=abcdef", + "p2-times=946684800:0", + "p2-append=p2-append", + "p2-directory=true:removed=true", + "p2-splice=splice-data", + "p2-hard=hard-p2:2", + "p2-hard-link=hard-p2:2", + "p2-symlink=replay-p2-hard.bin:hard-p2", + "p2-replacement=new-p2", + "p2-open-unlinked-absent=true", + "p3-resize=uvwxyz", + "p3-times=978307200:0", + "p3-append=p3-append", + "p3-directory=true:removed=true", + "p3-hard=hard-p3:2", + "p3-hard-link=hard-p3:2", + "p3-symlink=replay-p3-hard.bin:hard-p3", + "p3-replacement=new-p3", + "p3-open-unlinked-absent=true", + ] + ); + + executor.simulated_crash(&worker_id).await?; + let reconstructed = executor + .invoke_and_await_agent( + &component, + &agent_id, + "inspect_reconstruction_matrix", + data_value!(), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected reconstructed matrix"))?; + assert_eq!(schema_string_list(reconstructed), expected); + executor.check_oplog_is_queryable(&worker_id).await?; + Ok(()) +} + +#[test] +#[timeout("2m")] +#[tracing::instrument] +async fn filesystem_reconstruction_stops_at_exact_revert_target( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + filesystem_reconstruction_stops_at_exact_revert_target_with_backend( + last_unique_id, + deps, + initial_file_system, + None, + ) + .await +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[timeout("2m")] +#[tracing::instrument] +async fn filesystem_reconstruction_stops_at_exact_revert_target_on_managed_xfs( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + filesystem_reconstruction_stops_at_exact_revert_target_with_backend( + last_unique_id, + deps, + initial_file_system, + Some(managed_xfs_test_root()), + ) + .await +} + +async fn filesystem_reconstruction_stops_at_exact_revert_target_with_backend( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + initial_file_system: &PrecompiledComponent, + managed_xfs_root: Option, +) -> anyhow::Result<()> { + use golem_common::{agent_id, data_value}; + + let context = TestContext::new(last_unique_id); + let executor = start_with_overrides( + deps, + &context, + TestExecutorOverrides { + configure: Some(Arc::new(move |config| { + if let Some(root) = &managed_xfs_root { + config.filesystem_storage.managed_xfs_root_dir = Some(root.clone()); + } + full_replay_config(config); + })), + ..TestExecutorOverrides::default() + }, + ) + .await?; + let component = executor + .component_dep(&context.default_environment_id, initial_file_system) + .store() + .await?; + let agent_id = agent_id!("P3FileSystem", "filesystem-exact-replay-target"); + let worker_id = executor + .start_agent(&component.id, agent_id.clone()) + .await?; + + executor + .invoke_and_await_agent( + &component, + &agent_id, + "write_replay_target", + data_value!("first"), + ) + .await?; + let first_target = executor.oplog_max_index(&worker_id).await?; + executor + .invoke_and_await_agent( + &component, + &agent_id, + "write_replay_target", + data_value!("second"), + ) + .await?; + + executor + .revert( + &worker_id, + RevertWorkerTarget::RevertToOplogIndex(RevertToOplogIndex { + last_oplog_index: first_target, + }), + ) + .await?; + let result = executor + .invoke_and_await_agent( + &component, + &agent_id, + "inspect_path", + data_value!("replay-target.txt"), ) .await? .into_return_value() - .ok_or_else(|| anyhow!("expected return value"))?; - - executor.check_oplog_is_queryable(&worker_id).await?; - + .ok_or_else(|| anyhow!("expected replay target contents"))?; assert_eq!( - result, - SchemaValue::Record { - fields: vec![ - SchemaValue::Option { inner: None }, - SchemaValue::Option { - inner: Some(Box::new(SchemaValue::String("hello world".to_string()))) - }, - SchemaValue::Option { inner: None } - ] - } + schema_string_list(result), + vec!["p2_read=first", "p3_read=first"] ); - Ok(()) } #[test] +#[timeout("2m")] #[tracing::instrument] -async fn initial_file_read_write( +async fn filesystem_reconstruction_uses_updated_initial_files( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + filesystem_reconstruction_uses_updated_initial_files_with_backend( + last_unique_id, + deps, + initial_file_system, + None, + ) + .await +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[timeout("2m")] +#[tracing::instrument] +async fn filesystem_reconstruction_uses_updated_initial_files_on_managed_xfs( last_unique_id: &LastUniqueId, deps: &WorkerExecutorTestDependencies, #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, _tracing: &Tracing, +) -> anyhow::Result<()> { + filesystem_reconstruction_uses_updated_initial_files_with_backend( + last_unique_id, + deps, + initial_file_system, + Some(managed_xfs_test_root()), + ) + .await +} + +async fn filesystem_reconstruction_uses_updated_initial_files_with_backend( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + initial_file_system: &PrecompiledComponent, + managed_xfs_root: Option, ) -> anyhow::Result<()> { use golem_common::{agent_id, data_value}; let context = TestContext::new(last_unique_id); - let executor = start(deps, &context).await?; - + let executor = start_with_overrides( + deps, + &context, + TestExecutorOverrides { + configure: Some(Arc::new(move |config| { + if let Some(root) = &managed_xfs_root { + config.filesystem_storage.managed_xfs_root_dir = Some(root.clone()); + } + full_replay_config(config); + })), + ..TestExecutorOverrides::default() + }, + ) + .await?; let component = executor .component_dep(&context.default_environment_id, initial_file_system) .with_files( - "FileReadWrite", - &[ - IFSEntry { - source_path: PathBuf::from("initial-file-system/files/foo.txt"), - target_path: CanonicalFilePath::from_abs_str("/foo.txt").unwrap(), - permissions: AgentFilePermissions::ReadOnly, - }, - IFSEntry { - source_path: PathBuf::from("initial-file-system/files/baz.txt"), - target_path: CanonicalFilePath::from_abs_str("/bar/baz.txt").unwrap(), - permissions: AgentFilePermissions::ReadWrite, - }, - ], + "P3FileSystem", + &[IFSEntry { + source_path: PathBuf::from("initial-file-system/files/baz.txt"), + target_path: CanonicalFilePath::from_abs_str("/versioned.txt").unwrap(), + permissions: AgentFilePermissions::ReadOnly, + }], ) .store() .await?; - - let mut env = HashMap::new(); - env.insert("RUST_BACKTRACE".to_string(), "full".to_string()); - let agent_id = agent_id!("FileReadWrite", "initial-file-read-write-1"); + let agent_id = agent_id!("P3FileSystem", "filesystem-updated-initial-files"); let worker_id = executor - .start_agent_with(&component.id, agent_id.clone(), env, Vec::new()) + .start_agent(&component.id, agent_id.clone()) .await?; + assert_eq!( + executor + .get_file_contents(&worker_id, "/versioned.txt") + .await?, + Bytes::from_static(b"baz\n") + ); - let result = executor - .invoke_and_await_agent(&component, &agent_id, "run", data_value!()) + let updated = executor + .update_component_with_files( + &component.id, + "P3FileSystem", + &initial_file_system.wasm_name, + vec![IFSEntry { + source_path: PathBuf::from("initial-file-system/files/foo.txt"), + target_path: CanonicalFilePath::from_abs_str("/versioned.txt").unwrap(), + permissions: AgentFilePermissions::ReadOnly, + }], + ) + .await?; + executor + .auto_update_worker(&worker_id, updated.revision, false) + .await?; + + let updated_result = executor + .invoke_and_await_agent( + &component, + &agent_id, + "inspect_path", + data_value!("versioned.txt"), + ) .await? .into_return_value() - .ok_or_else(|| anyhow!("expected return value"))?; - - executor.check_oplog_is_queryable(&worker_id).await?; - + .ok_or_else(|| anyhow!("expected updated initial file"))?; assert_eq!( - result, - SchemaValue::Tuple { - elements: vec![ - SchemaValue::Option { - inner: Some(Box::new(SchemaValue::String("foo\n".to_string()))) - }, - SchemaValue::Option { inner: None }, - SchemaValue::Option { inner: None }, - SchemaValue::Option { - inner: Some(Box::new(SchemaValue::String("baz\n".to_string()))) - }, - SchemaValue::Option { - inner: Some(Box::new(SchemaValue::String("hello world".to_string()))) - }, - ] - } + schema_string_list(updated_result), + vec!["p2_read=foo\n", "p3_read=foo\n"] ); + executor.simulated_crash(&worker_id).await?; + let reconstructed = executor + .invoke_and_await_agent( + &component, + &agent_id, + "inspect_path", + data_value!("versioned.txt"), + ) + .await? + .into_return_value() + .ok_or_else(|| anyhow!("expected reconstructed updated initial file"))?; + assert_eq!( + schema_string_list(reconstructed), + vec!["p2_read=foo\n", "p3_read=foo\n"] + ); Ok(()) } #[test] +#[timeout("2m")] #[tracing::instrument] -async fn initial_file_p3_parity( +async fn filesystem_full_replay_survives_lifecycle_transitions( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, + _tracing: &Tracing, +) -> anyhow::Result<()> { + filesystem_full_replay_survives_lifecycle_transitions_with_backend( + last_unique_id, + deps, + initial_file_system, + None, + ) + .await +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "requires the privileged managed XFS test runner"] +#[timeout("2m")] +#[tracing::instrument] +async fn filesystem_full_replay_survives_managed_xfs_lifecycle_transitions( last_unique_id: &LastUniqueId, deps: &WorkerExecutorTestDependencies, #[tagged_as("initial_file_system")] initial_file_system: &PrecompiledComponent, _tracing: &Tracing, ) -> anyhow::Result<()> { + filesystem_full_replay_survives_lifecycle_transitions_with_backend( + last_unique_id, + deps, + initial_file_system, + Some(managed_xfs_test_root()), + ) + .await +} + +async fn filesystem_full_replay_survives_lifecycle_transitions_with_backend( + last_unique_id: &LastUniqueId, + deps: &WorkerExecutorTestDependencies, + initial_file_system: &PrecompiledComponent, + managed_xfs_root: Option, +) -> anyhow::Result<()> { + use golem_api_grpc::proto::golem::shardmanager::ShardId; + use golem_api_grpc::proto::golem::workerexecutor::v1::{ + AssignShardsRequest, RevokeShardsRequest, assign_shards_response, revoke_shards_response, + }; use golem_common::{agent_id, data_value}; let context = TestContext::new(last_unique_id); - let executor = start(deps, &context).await?; - + let first_root = tempfile::tempdir()?; + let second_root = tempfile::tempdir()?; + let managed = managed_xfs_root.is_some(); + let (first_backend_root, second_backend_root) = match managed_xfs_root { + Some(root) => { + let first = root.join("lifecycle-first-executor"); + let second = root.join("lifecycle-second-executor"); + std::fs::create_dir_all(&first)?; + std::fs::create_dir_all(&second)?; + (first, second) + } + None => ( + first_root.path().to_path_buf(), + second_root.path().to_path_buf(), + ), + }; + let root = first_backend_root; + let executor = start_with_overrides( + deps, + &context, + TestExecutorOverrides { + configure: Some(Arc::new(move |config| { + if managed { + config.filesystem_storage.managed_xfs_root_dir = Some(root.clone()); + } else { + config.filesystem_storage.deterministic_root_dir = Some(root.clone()); + } + full_replay_config(config); + })), + ..TestExecutorOverrides::default() + }, + ) + .await?; let component = executor .component_dep(&context.default_environment_id, initial_file_system) - .with_files( - "P3FileSystem", - &[ - IFSEntry { - source_path: PathBuf::from("initial-file-system/files/foo.txt"), - target_path: CanonicalFilePath::from_abs_str("/foo.txt").unwrap(), - permissions: AgentFilePermissions::ReadOnly, - }, - IFSEntry { - source_path: PathBuf::from("initial-file-system/files/baz.txt"), - target_path: CanonicalFilePath::from_abs_str("/bar/baz.txt").unwrap(), - permissions: AgentFilePermissions::ReadWrite, - }, - ], - ) .store() .await?; - - let agent_id = agent_id!("P3FileSystem", "initial-file-p3-parity-1"); + let agent_id = agent_id!("P3FileSystem", "filesystem-full-replay-lifecycles"); let worker_id = executor .start_agent(&component.id, agent_id.clone()) .await?; - - let expected = vec![ - "ro_flags_p2_write=false".to_string(), - "ro_flags_p3_write=false".to_string(), - "ro_hash_parity=true".to_string(), - "ro_hash_p3_deterministic=true".to_string(), - "ro_hash_at_parity=true".to_string(), - "ro_set_times_p2=err:not-permitted".to_string(), - "ro_set_times_p3=err:not-permitted".to_string(), - "ro_set_times_at_p2=err:not-permitted".to_string(), - "ro_set_times_at_p3=err:not-permitted".to_string(), - "ro_rename_at_p2=err:not-permitted".to_string(), - "ro_rename_at_p3=err:not-permitted".to_string(), - "ro_symlink_at_p2=err:not-permitted".to_string(), - "ro_symlink_at_p3=err:not-permitted".to_string(), - "ro_unlink_file_at_p2=err:not-permitted".to_string(), - "ro_unlink_file_at_p3=err:not-permitted".to_string(), - "rw_flags_p2_write=true".to_string(), - "rw_flags_p3_write=true".to_string(), - "rw_hash_parity=true".to_string(), - "rw_set_times_p2=ok".to_string(), - "rw_set_times_p3=ok".to_string(), - ]; - - fn as_entries(result: SchemaValue) -> Vec { - let SchemaValue::List { elements } = result else { - panic!("expected list, got {result:?}") - }; - elements - .into_iter() - .map(|element| match element { - SchemaValue::String(entry) => entry, - other => panic!("expected string, got {other:?}"), - }) - .collect::>() - } + let owned_agent_id = OwnedAgentId::new(context.default_environment_id, &worker_id); let result = executor - .invoke_and_await_agent(&component, &agent_id, "run", data_value!()) + .invoke_and_await_agent(&component, &agent_id, "run_writable", data_value!()) .await? .into_return_value() .ok_or_else(|| anyhow!("expected return value"))?; - + assert_eq!( + schema_string_list(result), + vec![ + "p2_write_p3_read=p2-to-p3".to_string(), + "p3_write_p2_read=p3-to-p2".to_string(), + ] + ); executor.check_oplog_is_queryable(&worker_id).await?; - assert_eq!(as_entries(result), expected); - - // Crash the worker so the next invocation replays the recorded oplog first, - // verifying the P3 metadata-hash -> durable stat call sequence is replay-stable - executor.simulated_crash(&worker_id).await?; - - let result_after_crash = executor - .invoke_and_await_agent(&component, &agent_id, "run", data_value!()) + executor + .wait_for_status(&worker_id, AgentStatus::Idle, Duration::from_secs(10)) + .await?; + tokio::time::timeout(Duration::from_secs(10), async { + while executor.worker_eviction_class(&owned_agent_id).await + != Some(golem_worker_executor::worker::EvictionClass::LoadedIdle) + { + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .map_err(|_| anyhow!("worker did not become loaded-idle after its billing window closed"))?; + assert!(executor.stop_worker_if_idle(&owned_agent_id).await?); + assert!(!executor.worker_is_loaded(&owned_agent_id).await); + assert_reconstructed_writable_file(&executor, &component, &agent_id).await?; + + let shard = ShardId { value: 0 }; + let mut client = executor.client.clone(); + let revoked = client + .revoke_shards(RevokeShardsRequest { + shard_ids: vec![shard], + }) .await? - .into_return_value() - .ok_or_else(|| anyhow!("expected return value"))?; - - executor.check_oplog_is_queryable(&worker_id).await?; + .into_inner(); + assert!(matches!( + revoked.result, + Some(revoke_shards_response::Result::Success(_)) + )); + tokio::time::timeout(Duration::from_secs(5), async { + while executor.worker_is_loaded(&owned_agent_id).await { + tokio::task::yield_now().await; + } + }) + .await + .map_err(|_| anyhow!("worker remained loaded after its shard was revoked"))?; + assert!(!executor.worker_is_loaded(&owned_agent_id).await); + let assigned = client + .assign_shards(AssignShardsRequest { + shard_ids: vec![shard], + }) + .await? + .into_inner(); + assert!(matches!( + assigned.result, + Some(assign_shards_response::Result::Success(_)) + )); + assert_reconstructed_writable_file(&executor, &component, &agent_id).await?; + + drop(client); + drop(executor); - assert_eq!(as_entries(result_after_crash), expected); + let root = second_backend_root; + let relocated = start_with_overrides( + deps, + &context, + TestExecutorOverrides { + configure: Some(Arc::new(move |config| { + if managed { + config.filesystem_storage.managed_xfs_root_dir = Some(root.clone()); + } else { + config.filesystem_storage.deterministic_root_dir = Some(root.clone()); + } + full_replay_config(config); + })), + ..TestExecutorOverrides::default() + }, + ) + .await?; + assert_reconstructed_writable_file(&relocated, &component, &agent_id).await?; + relocated.check_oplog_is_queryable(&worker_id).await?; Ok(()) } @@ -4727,54 +6045,6 @@ async fn http_timeout_and_restart( Ok(()) } -#[test] -#[tracing::instrument] -async fn filesystem_write_increments_storage_bytes_written_metric( - last_unique_id: &LastUniqueId, - deps: &WorkerExecutorTestDependencies, - #[tagged_as("host_api_tests")] host_api_tests: &PrecompiledComponent, - _tracing: &Tracing, -) -> anyhow::Result<()> { - use golem_common::{agent_id, data_value}; - - let context = TestContext::new(last_unique_id); - let executor = start(deps, &context).await?; - - let component = executor - .component_dep(&context.default_environment_id, host_api_tests) - .store() - .await?; - let agent_id = agent_id!("FileSystem", "fs-metrics-test-1"); - let account_id = context.account_id.to_string(); - let environment_id = context.default_environment_id.to_string(); - - let bytes_before = STORAGE_BYTES_WRITTEN_TOTAL - .with_label_values(&[STORAGE_TYPE_FILESYSTEM, &account_id, &environment_id]) - .get(); - - executor - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/fs-metrics-test.txt", "hello from metrics test"), - ) - .await?; - - drop(executor); - - let bytes_after = STORAGE_BYTES_WRITTEN_TOTAL - .with_label_values(&[STORAGE_TYPE_FILESYSTEM, &account_id, &environment_id]) - .get(); - - assert!( - bytes_after > bytes_before, - "filesystem bytes_written should have increased: before={bytes_before}, after={bytes_after}" - ); - - Ok(()) -} - /// Reproducer for oplog mismatch bug with STREAMING HTTP responses. /// /// Uses streaming_http_read which reads a chunked HTTP response body diff --git a/integration-tests/lima/managed-filesystem.yaml b/integration-tests/lima/managed-filesystem.yaml new file mode 100644 index 0000000000..1e08df06d1 --- /dev/null +++ b/integration-tests/lima/managed-filesystem.yaml @@ -0,0 +1,38 @@ +minimumLimaVersion: 2.0.0 + +base: + - template:_images/ubuntu + +cpus: 8 +memory: 12GiB +disk: 80GiB + +containerd: + system: false + user: false + +provision: + - mode: system + script: | + #!/bin/bash + set -eux -o pipefail + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y build-essential clang cmake curl git libssl-dev pkg-config protobuf-compiler python3 redis-server util-linux xfsprogs + - mode: user + script: | + #!/bin/bash + set -eux -o pipefail + if ! command -v cargo >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + fi + . "${HOME}/.cargo/env" + rustup target add wasm32-wasip1 wasm32-wasip2 + +probes: + - mode: readiness + description: XFS and Rust tooling is installed + script: | + #!/bin/bash + test -x /usr/sbin/mkfs.xfs + test -x "${HOME}/.cargo/bin/cargo" diff --git a/integration-tests/scripts/managed-filesystem/run-lima.sh b/integration-tests/scripts/managed-filesystem/run-lima.sh new file mode 100755 index 0000000000..ddb47657b2 --- /dev/null +++ b/integration-tests/scripts/managed-filesystem/run-lima.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +instance=golem-managed-xfs-v1 +template=${repo_root}/integration-tests/lima/managed-filesystem.yaml + +if ! command -v limactl >/dev/null 2>&1; then + echo "missing required command: limactl" >&2 + exit 1 +fi + +exists=false +while read -r name; do + if [[ ${name} == "${instance}" ]]; then + exists=true + fi +done < <(limactl list --format '{{.Name}}') + +if [[ ${exists} == false ]]; then + limactl start --name "${instance}" --mount-only "${repo_root}:w" "${template}" +else + limactl start "${instance}" +fi + +guest_home=$(limactl shell "${instance}" -- printenv HOME) +runs=${GOLEM_MANAGED_XFS_RUNS:-1} +if [[ ! ${runs} =~ ^[1-9][0-9]*$ ]]; then + echo "GOLEM_MANAGED_XFS_RUNS must be a positive integer" >&2 + exit 1 +fi + +for ((run = 1; run <= runs; run++)); do + echo "Managed XFS Lima run ${run}/${runs}" + limactl shell "${instance}" -- \ + sudo GOLEM_REPO_ROOT="${repo_root}" \ + GOLEM_MANAGED_XFS_TARGET_DIR="${GOLEM_MANAGED_XFS_TARGET_DIR:-${guest_home}/.cache/golem-managed-xfs-target}" \ + GOLEM_MANAGED_XFS_CLI_TARGET_DIR="${guest_home}/.cache/golem-managed-xfs-cli-target" \ + GOLEM_MANAGED_XFS_CLEAN="${GOLEM_MANAGED_XFS_CLEAN:-0}" \ + GOLEM_MANAGED_XFS_MIN_FREE_GIB="${GOLEM_MANAGED_XFS_MIN_FREE_GIB:-15}" \ + GOLEM_MANAGED_XFS_VALIDATE_CACHE_ONLY="${GOLEM_MANAGED_XFS_VALIDATE_CACHE_ONLY:-0}" \ + GOLEM_MANAGED_XFS_REUSE_TEST_BINARIES="${GOLEM_MANAGED_XFS_REUSE_TEST_BINARIES:-0}" \ + GOLEM_MANAGED_XFS_CARGO_TEST_R="${GOLEM_MANAGED_XFS_CARGO_TEST_R:-cargo-test-r}" \ + CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-4}" \ + "${repo_root}/integration-tests/scripts/managed-filesystem/run-loopback-xfs.sh" "$@" +done diff --git a/integration-tests/scripts/managed-filesystem/run-loopback-xfs.sh b/integration-tests/scripts/managed-filesystem/run-loopback-xfs.sh new file mode 100755 index 0000000000..ddeb90722c --- /dev/null +++ b/integration-tests/scripts/managed-filesystem/run-loopback-xfs.sh @@ -0,0 +1,345 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${EUID} -ne 0 ]]; then + echo "run-loopback-xfs.sh must run as root inside a privileged container or VM" >&2 + exit 1 +fi + +for command in flock losetup mkfs.xfs mount python3 timeout truncate; do + if ! command -v "${command}" >/dev/null 2>&1; then + echo "missing required command: ${command}" >&2 + exit 1 + fi +done + +exec 9>/tmp/golem-managed-xfs.lock +if ! flock --wait 10 9; then + echo "another managed XFS test run is still active" >&2 + exit 1 +fi + +repo_root=${GOLEM_REPO_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)} +target_dir=${GOLEM_MANAGED_XFS_TARGET_DIR:-${CARGO_TARGET_DIR:-/var/tmp/golem-managed-xfs-target}} +cli_target_dir=${GOLEM_MANAGED_XFS_CLI_TARGET_DIR:-${repo_root}/target/managed-xfs-cli-linux} +clean_build=${GOLEM_MANAGED_XFS_CLEAN:-0} +if [[ ${clean_build} != 0 && ${clean_build} != 1 ]]; then + echo "GOLEM_MANAGED_XFS_CLEAN must be 0 or 1" >&2 + exit 1 +fi +minimum_free_gib=${GOLEM_MANAGED_XFS_MIN_FREE_GIB:-15} +if [[ ! ${minimum_free_gib} =~ ^[0-9]+$ ]]; then + echo "GOLEM_MANAGED_XFS_MIN_FREE_GIB must be a non-negative integer" >&2 + exit 1 +fi +validate_cache_only=${GOLEM_MANAGED_XFS_VALIDATE_CACHE_ONLY:-0} +if [[ ${validate_cache_only} != 0 && ${validate_cache_only} != 1 ]]; then + echo "GOLEM_MANAGED_XFS_VALIDATE_CACHE_ONLY must be 0 or 1" >&2 + exit 1 +fi +reuse_test_binaries=${GOLEM_MANAGED_XFS_REUSE_TEST_BINARIES:-0} +if [[ ${reuse_test_binaries} != 0 && ${reuse_test_binaries} != 1 ]]; then + echo "GOLEM_MANAGED_XFS_REUSE_TEST_BINARIES must be 0 or 1" >&2 + exit 1 +fi +cargo_test_r=${GOLEM_MANAGED_XFS_CARGO_TEST_R:-cargo-test-r} +export CARGO_TARGET_DIR=${target_dir} +export CARGO_INCREMENTAL=0 +export CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS:-4} +export CARGO_PROFILE_DEV_DEBUG=0 +export CARGO_PROFILE_DEV_INCREMENTAL=false +export CARGO_PROFILE_TEST_DEBUG=0 +export CARGO_PROFILE_TEST_INCREMENTAL=false + +run_test() { + if [[ -n ${SUDO_USER:-} ]]; then + user_home=$(eval echo "~${SUDO_USER}") + sudo --user "${SUDO_USER}" --set-home \ + --preserve-env=GOLEM_MANAGED_XFS_TEST_ROOT,GOLEM_MANAGED_XFS_TARGET_DIR,GOLEM_MANAGED_XFS_CLEAN,GOLEM_MANAGED_XFS_MIN_FREE_GIB,CARGO_TARGET_DIR,CARGO_BUILD_JOBS,CARGO_INCREMENTAL,CARGO_PROFILE_DEV_DEBUG,CARGO_PROFILE_DEV_INCREMENTAL,CARGO_PROFILE_TEST_DEBUG,CARGO_PROFILE_TEST_INCREMENTAL \ + env PATH="${user_home}/.cargo/bin:/usr/local/bin:/usr/bin:/bin" "$@" + else + "$@" + fi +} + +run_vm_cargo() { + if [[ -n ${SUDO_USER:-} ]]; then + user_home=$(eval echo "~${SUDO_USER}") + sudo --user "${SUDO_USER}" --set-home \ + --preserve-env=GOLEM_MANAGED_XFS_TEST_ROOT,GOLEM_MANAGED_XFS_TARGET_DIR,GOLEM_MANAGED_XFS_CLEAN,GOLEM_MANAGED_XFS_MIN_FREE_GIB,CARGO_TARGET_DIR,CARGO_BUILD_JOBS,CARGO_INCREMENTAL,CARGO_PROFILE_DEV_DEBUG,CARGO_PROFILE_DEV_INCREMENTAL,CARGO_PROFILE_TEST_DEBUG,CARGO_PROFILE_TEST_INCREMENTAL \ + env PATH="${user_home}/.cargo/bin:/usr/local/bin:/usr/bin:/bin" "$@" + else + "$@" + fi +} + +clean_cargo_cache_if_needed() { + local cache_dir=$1 + local cache_name=$2 + local runner=$3 + local available_kib= + local minimum_free_kib=$((minimum_free_gib * 1024 * 1024)) + local write_probe=${cache_dir}/.golem-managed-xfs-write-probe.$$ + + if ! "${runner}" mkdir -p "${cache_dir}" || + ! "${runner}" test -w "${cache_dir}" || + ! "${runner}" touch "${write_probe}" || + ! "${runner}" rm "${write_probe}"; then + echo "${cache_name} is not writable by its build user: ${cache_dir}" >&2 + echo "Remove or repair that cache directory as its owner, then rerun this script." >&2 + exit 1 + fi + while read -r _ _ _ available _; do + if [[ ${available} =~ ^[0-9]+$ ]]; then + available_kib=${available} + fi + done < <(df -Pk "${cache_dir}") + if [[ -z ${available_kib} ]]; then + echo "failed to determine free space for ${cache_dir}" >&2 + exit 1 + fi + if [[ ${clean_build} == 1 || ${available_kib} -lt ${minimum_free_kib} ]]; then + if [[ ${clean_build} == 0 ]]; then + echo "Cleaning ${cache_name}: less than ${minimum_free_gib} GiB is available" >&2 + fi + "${runner}" cargo clean --target-dir "${cache_dir}" + fi +} + +clean_cargo_cache_if_needed "${target_dir}" "managed XFS Cargo cache" run_vm_cargo + +if [[ ${validate_cache_only} == 1 ]]; then + clean_cargo_cache_if_needed "${cli_target_dir}" "managed XFS CLI Cargo cache" run_test + run_vm_cargo test -w "${target_dir}" + run_test test -w "${cli_target_dir}" + exit +fi + +work_dir=$(mktemp -d /tmp/golem-managed-xfs.XXXXXX) +chmod 0755 "${work_dir}" +image=${work_dir}/filesystem.img +mount_point=${work_dir}/mount +loop_device= + +cleanup() { + set +e + sync + if mountpoint -q "${mount_point}"; then + umount "${mount_point}" + fi + if [[ -n ${loop_device} ]]; then + losetup --detach "${loop_device}" + fi + rm -rf "${work_dir}" +} +trap cleanup EXIT INT TERM + +truncate --size 1G "${image}" +loop_device=$(losetup --find --show "${image}") +mkfs.xfs -f -m reflink=1 -n ftype=1 "${loop_device}" +mkdir -p "${mount_point}" +mount -t xfs -o prjquota "${loop_device}" "${mount_point}" +mkdir -p "${mount_point}/agents" +chmod 0777 "${mount_point}/agents" +if [[ -n ${SUDO_UID:-} && -n ${SUDO_GID:-} ]]; then + chown "${SUDO_UID}:${SUDO_GID}" "${mount_point}/agents" +fi + +export GOLEM_MANAGED_XFS_TEST_ROOT=${mount_point}/agents +cd "${repo_root}" + +build_test_binaries() { + echo "Building managed XFS test binaries with ${CARGO_BUILD_JOBS:-default} Cargo jobs" >&2 + local build_messages=${work_dir}/cargo-test-artifacts.jsonl + local selected_executables=${work_dir}/cargo-test-executables.tsv + local lib_decoy=${target_dir}/debug/deps/golem_worker_executor-stale-decoy + local integration_decoy=${target_dir}/debug/deps/integration-stale-decoy + run_vm_cargo mkdir -p "${target_dir}/debug/deps" + run_vm_cargo touch -t 203801010000 "${lib_decoy}" "${integration_decoy}" + run_vm_cargo chmod 0755 "${lib_decoy}" "${integration_decoy}" + local build_command=( + timeout --kill-after=30s 20m cargo test + -p golem-worker-executor + --lib + --test integration + --no-run + --message-format=json-render-diagnostics + ) + if ! run_vm_cargo "${build_command[@]}" > "${build_messages}"; then + echo "Initial clean build failed; retrying once with completed dependencies" >&2 + if ! run_vm_cargo "${build_command[@]}" > "${build_messages}"; then + run_vm_cargo rm -f "${lib_decoy}" "${integration_decoy}" + return 1 + fi + fi + + lib_test_binary= + integration_test_binary= + if ! run_vm_cargo python3 \ + "${repo_root}/integration-tests/scripts/managed-filesystem/select-cargo-test-executables.py" \ + "${build_messages}" "${repo_root}" > "${selected_executables}"; then + run_vm_cargo rm -f "${lib_decoy}" "${integration_decoy}" + return 1 + fi + local target executable + while IFS=$'\t' read -r target executable; do + case ${target} in + lib) lib_test_binary=${executable} ;; + integration) integration_test_binary=${executable} ;; + *) + echo "unexpected Cargo test target selected: ${target}" >&2 + run_vm_cargo rm -f "${lib_decoy}" "${integration_decoy}" + return 1 + ;; + esac + done < "${selected_executables}" + run_vm_cargo rm -f "${lib_decoy}" "${integration_decoy}" + if [[ -z ${lib_test_binary} || -z ${integration_test_binary} ]]; then + echo "cargo did not emit both managed XFS test executables" >&2 + return 1 + fi + if [[ ${lib_test_binary} == "${lib_decoy}" || ${integration_test_binary} == "${integration_decoy}" ]]; then + echo "Cargo artifact selection accepted a stale decoy executable" >&2 + return 1 + fi +} + +run_privileged_test() { + local target=$1 + local selected=$2 + local filter=$3 + + if [[ ${reuse_test_binaries} == 1 ]]; then + local target_args=() + if [[ ${target} == integration ]]; then + target_args=(--test integration) + else + target_args=(--lib) + fi + timeout --kill-after=30s 5m \ + "${cargo_test_r}" run --package golem-worker-executor "${target_args[@]}" \ + "${filter}" -- --exact --include-ignored --nocapture --report-time + return + fi + + local capable_binary=${work_dir}/${target}-capable + cp "${selected}" "${capable_binary}" + chmod 0755 "${capable_binary}" + if [[ ${target} == integration ]]; then + ( + cd "${repo_root}/golem-worker-executor" + timeout --kill-after=30s 5m \ + "${capable_binary}" "${filter}" --exact --include-ignored --nocapture --report-time + ) + else + timeout --kill-after=30s 5m \ + "${capable_binary}" "${filter}" --exact --include-ignored --nocapture --report-time + fi +} + +if [[ $# -gt 0 ]]; then + run_test "$@" + exit +fi + +initial_file_wasm=test-components/it_initial_file_system_release.wasm +rebuild_initial_file_wasm=false +if [[ ! -f ${initial_file_wasm} ]]; then + rebuild_initial_file_wasm=true +else + shopt -s globstar nullglob + for source in Cargo.toml \ + Cargo.lock \ + test-components/initial-file-system/Cargo.toml \ + test-components/initial-file-system/Cargo.lock \ + test-components/initial-file-system/golem.yaml \ + test-components/golem-test-components-common.yaml \ + sdks/rust/golem-rust/Cargo.toml \ + sdks/rust/golem-rust/src/**/*.rs \ + sdks/rust/golem-rust/wit/**/*.wit \ + sdks/rust/golem-rust-macro/Cargo.toml \ + sdks/rust/golem-rust-macro/src/**/*.rs \ + test-components/initial-file-system/src/**/*.rs; do + if [[ ${source} -nt ${initial_file_wasm} ]]; then + rebuild_initial_file_wasm=true + break + fi + done +fi +if [[ ${rebuild_initial_file_wasm} == true ]]; then + clean_cargo_cache_if_needed "${cli_target_dir}" "managed XFS CLI Cargo cache" run_test + run_test env CARGO_TARGET_DIR="${cli_target_dir}" cargo build -p golem-cli + ( + cd test-components/initial-file-system + run_test env CARGO_TARGET_DIR="${cli_target_dir}" \ + "${cli_target_dir}/debug/golem-cli" --preset release build --yes --skip-check + run_test env CARGO_TARGET_DIR="${cli_target_dir}" \ + "${cli_target_dir}/debug/golem-cli" --preset release exec copy + ) +fi + +lib_test_binary= +integration_test_binary= +if [[ ${reuse_test_binaries} == 0 ]]; then + build_test_binaries +fi + +run_privileged_test \ + lib \ + "${lib_test_binary}" \ + services::agent_filesystem::tests::managed_xfs_owns_observes_and_cleans_project_filesystem + +run_privileged_test \ + lib \ + "${lib_test_binary}" \ + services::agent_filesystem::tests::managed_xfs_allocated_bytes_flow_through_resource_billing + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::initial_file_p2_p3_parity_on_managed_xfs + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::managed_xfs_resource_billing_survives_idle_and_replay + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::filesystem_full_replay_survives_managed_xfs_lifecycle_transitions + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::filesystem_mutation_histories_reconstruct_on_managed_xfs + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::filesystem_reconstruction_stops_at_exact_revert_target_on_managed_xfs + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::filesystem_reconstruction_uses_updated_initial_files_on_managed_xfs + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::p2_p3_mid_effect_enospc_reconstructs_on_unmanaged_filesystem + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::p2_p3_quota_exhaustion_on_managed_xfs + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::p2_p3_object_quota_on_managed_xfs + +run_privileged_test \ + integration \ + "${integration_test_binary}" \ + wasi::filesystem_downgrade_blocks_guest_until_limit_recovers diff --git a/integration-tests/scripts/managed-filesystem/select-cargo-test-executables.py b/integration-tests/scripts/managed-filesystem/select-cargo-test-executables.py new file mode 100644 index 0000000000..5f7bc5dfe4 --- /dev/null +++ b/integration-tests/scripts/managed-filesystem/select-cargo-test-executables.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import json +import os +import sys + + +def fail(message: str) -> None: + print(message, file=sys.stderr) + raise SystemExit(1) + + +if len(sys.argv) != 3: + fail(f"usage: {sys.argv[0]} ") + +matches = {"lib": set(), "integration": set()} +repo_root = os.path.realpath(sys.argv[2]) +expected_sources = { + "lib": os.path.join(repo_root, "golem-worker-executor", "src", "lib.rs"), + "integration": os.path.join(repo_root, "golem-worker-executor", "tests", "lib.rs"), +} +with open(sys.argv[1], encoding="utf-8") as messages: + for line_number, line in enumerate(messages, 1): + if not line.strip(): + continue + try: + message = json.loads(line) + except json.JSONDecodeError as error: + fail(f"invalid Cargo JSON message on line {line_number}: {error}") + if message.get("reason") != "compiler-artifact": + continue + if not message.get("profile", {}).get("test"): + continue + target = message.get("target", {}) + executable = message.get("executable") + if target.get("name") == "golem_worker_executor" and target.get("kind") == ["lib"]: + label = "lib" + elif target.get("name") == "integration" and target.get("kind") == ["test"]: + label = "integration" + else: + continue + if os.path.realpath(target.get("src_path", "")) != expected_sources[label]: + continue + if executable is not None: + matches[label].add(executable) + +for label, executables in matches.items(): + if len(executables) != 1: + fail(f"expected exactly one {label} test executable from Cargo, got {sorted(executables)}") + executable = executables.pop() + if not os.path.isfile(executable) or not os.access(executable, os.X_OK): + fail(f"Cargo emitted a missing or non-executable {label} test artifact: {executable}") + print(f"{label}\t{executable}") diff --git a/integration-tests/tests/capabilities.rs b/integration-tests/tests/capabilities.rs index 51a0101d60..96cedb88cb 100644 --- a/integration-tests/tests/capabilities.rs +++ b/integration-tests/tests/capabilities.rs @@ -155,6 +155,14 @@ async fn quota_token_capability_round_trips_and_is_redacted( ) .await?; + tokio::time::timeout(Duration::from_secs(60), async { + while received.load(Ordering::SeqCst) < 4 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_| anyhow!("timed out waiting for all quota-token HTTP calls"))?; + http_server.abort(); // Reconstruction on the receiver side: the split token was rebuilt into a diff --git a/integration-tests/tests/memory_billing.rs b/integration-tests/tests/memory_billing.rs index 59c3dd9537..75ed84be57 100644 --- a/integration-tests/tests/memory_billing.rs +++ b/integration-tests/tests/memory_billing.rs @@ -130,9 +130,11 @@ mod tests { user.wait_for_status(&worker, AgentStatus::Idle, Duration::from_secs(10)) .await?; tokio::time::sleep(Duration::from_secs(5)).await; + let after_permit_release = memory_gb_seconds(&deps, &user).await?; + tokio::time::sleep(Duration::from_secs(5)).await; let after_idle = memory_gb_seconds(&deps, &user).await?; assert_eq!( - after_idle, after_replay, + after_idle, after_permit_release, "loaded-idle time after permit release must not accrue memory usage" ); diff --git a/integration-tests/tests/storage_billing.rs b/integration-tests/tests/storage_billing.rs index 6aa61f1901..6b1f2a1839 100644 --- a/integration-tests/tests/storage_billing.rs +++ b/integration-tests/tests/storage_billing.rs @@ -17,10 +17,10 @@ test_r::enable!(); #[test_r::sequential] mod tests { use golem_client::api::RegistryServiceClient; + use golem_common::agent_id; use golem_common::model::account_usage::BYTE_SECONDS_PER_GB_MONTH; use golem_common::model::component::{AgentFilePermissions, CanonicalFilePath}; use golem_common::tracing::{TracingConfig, init_tracing_with_default_debug_env_filter}; - use golem_common::{agent_id, data_value}; use golem_test_framework::config::{ EnvBasedTestDependencies, EnvBasedTestDependenciesConfig, TestDependencies, }; @@ -62,12 +62,6 @@ mod tests { "GOLEM__RESOURCE_LIMITS__CONFIG__LIMIT_REFRESH_INTERVAL".to_string(), "1s".to_string(), ), - ( - "GOLEM__FILESYSTEM_STORAGE__TOTAL_WORKER_FILESYSTEM_STORAGE_BYTES".to_string(), - // The eviction test writes 8-byte files. An 8-byte pool forces the first - // agent out before the second agent's filesystem can be loaded. - "8".to_string(), - ), ]) .await; @@ -90,7 +84,6 @@ mod tests { async fn wait_for_durable_billing_to_settle( deps: &EnvBasedTestDependencies, user: &golem_test_framework::config::dsl_impl::TestUserContext, - must_exceed: Option, ) -> anyhow::Result { let deadline = tokio::time::Instant::now() + Duration::from_secs(10); let mut last = durable_byte_seconds(deps, user).await?; @@ -103,9 +96,7 @@ mod tests { last = current; unchanged_since = tokio::time::Instant::now(); } - if unchanged_since.elapsed() >= Duration::from_secs(1) - && must_exceed.is_none_or(|minimum| current > minimum) - { + if unchanged_since.elapsed() >= Duration::from_secs(1) { return Ok(current); } if tokio::time::Instant::now() >= deadline { @@ -114,16 +105,9 @@ mod tests { } } - fn assert_billing_window(actual: f64, min: f64, max: f64, phase: &str) { - assert!( - (min..=max).contains(&actual), - "unexpected storage billing during {phase}: expected {min}..={max} byte-seconds, got {actual}" - ); - } - #[test] #[timeout("2m")] - async fn provisioned_read_write_file_is_metered() -> anyhow::Result<()> { + async fn unmanaged_filesystem_storage_is_unmetered() -> anyhow::Result<()> { let deps = create_deps().await; let user = deps.user().await?; let (_, env) = user.app_and_env().await?; @@ -147,96 +131,9 @@ mod tests { let before = durable_byte_seconds(&deps, &user).await?; tokio::time::sleep(Duration::from_secs(2)).await; user.delete_worker(&worker).await?; - let after = wait_for_durable_billing_to_settle(&deps, &user, Some(before)).await?; - - // Deleting the idle worker flushes its meter before the final snapshot. Four bytes over - // two seconds should produce about eight byte-seconds without permitting overbilling. - assert_billing_window(after - before, 4.0, 12.0, "provisioned-file metering"); - Ok(()) - } - - #[test] - #[timeout("2m")] - async fn evicted_agent_stops_metering_until_reload() -> anyhow::Result<()> { - let deps = create_deps().await; - let user = deps.user().await?; - let (_, env) = user.app_and_env().await?; - let component = user - .component(&env.id, "golem_it_host_api_tests_release") - .store() - .await?; - let agent_a = agent_id!("FileSystem", "storage-billing-a"); - let agent_b = agent_id!("FileSystem", "storage-billing-b"); - - let worker_a = user.start_agent(&component.id, agent_a.clone()).await?; - user.invoke_and_await_agent( - &component, - &agent_a, - "write_file", - data_value!("/metered.txt", "12345678"), - ) - .await?; - let active_start = durable_byte_seconds(&deps, &user).await?; - tokio::time::sleep(Duration::from_secs(2)).await; - let active_end = durable_byte_seconds(&deps, &user).await?; - - let worker_b = user.start_agent(&component.id, agent_b.clone()).await?; - user.invoke_and_await_agent( - &component, - &agent_b, - "write_file", - data_value!("/temporary.txt", "abcdefgh"), - ) - .await?; - user.invoke_and_await_agent( - &component, - &agent_b, - "delete_file", - data_value!("/temporary.txt"), - ) - .await?; - - // The successful 8-byte write proves the full pool forced agent A out. Wait until any - // final meter/drop batch arrives before opening the interval that must remain quiescent. - let evicted_start = wait_for_durable_billing_to_settle(&deps, &user, None).await?; - tokio::time::sleep(Duration::from_secs(2)).await; - let evicted_end = durable_byte_seconds(&deps, &user).await?; - - user.invoke_and_await_agent( - &component, - &agent_a, - "read_file", - data_value!("/metered.txt"), - ) - .await?; - let reloaded_start = durable_byte_seconds(&deps, &user).await?; - tokio::time::sleep(Duration::from_secs(2)).await; - user.delete_worker(&worker_a).await?; - let reloaded_end = - wait_for_durable_billing_to_settle(&deps, &user, Some(reloaded_start)).await?; - user.delete_worker(&worker_b).await?; - - // Eight bytes over two seconds should produce about sixteen byte-seconds. The window - // covers asynchronous flush and batching latency while still rejecting overbilling. - assert_billing_window( - active_end - active_start, - 10.0, - 24.0, - "pre-eviction metering", - ); - // An evicted agent must stop accruing. The bound is 6.0 rather than something - // tighter because eviction settling is asynchronous: at 8 bytes, 2.0 byte-seconds - // is only a quarter-second of headroom, which a loaded CI runner can exceed - // without anything being wrong. 6.0 still sits clearly below the 10.0 floor the - // active windows assert, so an agent that kept billing after eviction fails. - assert_billing_window(evicted_end - evicted_start, 0.0, 6.0, "evicted interval"); - assert_billing_window( - reloaded_end - reloaded_start, - 10.0, - 24.0, - "post-reload metering", - ); + let after = wait_for_durable_billing_to_settle(&deps, &user).await?; + assert_eq!(after, before); Ok(()) } } diff --git a/integration-tests/tests/worker.rs b/integration-tests/tests/worker.rs index b9173ce15d..4ea0f02e2d 100644 --- a/integration-tests/tests/worker.rs +++ b/integration-tests/tests/worker.rs @@ -23,7 +23,6 @@ use futures_concurrency::future::Join; use golem_api_grpc::proto::golem::worker::{LogEvent, log_event}; use golem_client::api::RegistryServiceClient; use golem_common::model::account::{AccountRevision, AccountSetPlan}; -use golem_common::model::account_usage::SetStorageLimit; use golem_common::model::component::{AgentFilePermissions, CanonicalFilePath, ComponentId}; use golem_common::model::oplog::public_oplog_entry::AgentInvocationStartedParams; use golem_common::model::oplog::{ @@ -2359,179 +2358,6 @@ async fn deployment_invalidates_agent_resolution_cache( Ok(()) } -/// Verifies that the per-agent disk space quota from the plan is enforced -/// end-to-end through the full Golem stack. The user's plan is changed to one -/// with a 5-byte disk quota; writing "hello world" (11 bytes) must then fail -/// with `WorkerAgentExceededFilesystemStorageLimit`. -#[test] -#[tracing::instrument] -#[timeout("4m")] -async fn agent_exceeds_per_plan_disk_space_quota( - deps: &EnvBasedTestDependencies, - _tracing: &Tracing, -) -> anyhow::Result<()> { - let admin = deps.admin().await; - let admin_client = admin.registry_service_client().await; - - let user = deps.user().await?; - - // Switch the user to the low_disk_space plan (5-byte per-worker quota). - admin_client - .set_account_plan( - &user.account_id.0, - &AccountSetPlan { - current_revision: AccountRevision::INITIAL, - plan: deps.registry_service().low_disk_space_plan(), - }, - ) - .await?; - - let (_, env) = user.app_and_env().await?; - - let component = user - .component(&env.id, "golem_it_host_api_tests_release") - .name("golem-it:host-api-tests") - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "disk-quota-exceeded-1"); - user.start_agent(&component.id, agent_id.clone()).await?; - - // "hello world" is 11 bytes — exceeds the 5-byte per-agent plan limit. - let result = user - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile.txt", "hello world"), - ) - .await; - - assert!( - result.is_err(), - "expected write to fail when per-plan disk quota is exceeded" - ); - let err_str = format!("{result:?}"); - assert!( - err_str.contains("AgentExceededFilesystemStorageLimit") || err_str.contains("storage"), - "expected WorkerAgentExceededFilesystemStorageLimit from plan quota enforcement, got: {err_str}" - ); - - Ok(()) -} - -/// Verifies that an account storage override set through the registry HTTP API -/// becomes the executor's effective per-agent disk quota. -#[test] -#[tracing::instrument] -#[timeout("4m")] -async fn agent_uses_account_storage_override_for_disk_space_quota( - deps: &EnvBasedTestDependencies, - _tracing: &Tracing, -) -> anyhow::Result<()> { - let admin = deps.admin().await; - let admin_client = admin.registry_service_client().await; - let user = deps.user().await?; - - admin_client - .set_account_plan( - &user.account_id.0, - &AccountSetPlan { - current_revision: AccountRevision::INITIAL, - plan: deps.registry_service().low_disk_space_plan(), - }, - ) - .await?; - let (_, env) = user.app_and_env().await?; - let component = user - .component(&env.id, "golem_it_host_api_tests_release") - .name("golem-it:host-api-tests") - .store() - .await?; - let agent_id = agent_id!("FileSystem", "disk-quota-override-1"); - user.start_agent(&component.id, agent_id.clone()).await?; - - user.invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/initial.txt", "x"), - ) - .await?; - - deps.registry_service() - .client(&user.token) - .await - .set_account_storage_override( - &user.account_id.0, - &SetStorageLimit { - value: 12, - expires_at: None, - }, - ) - .await?; - - // Invocation fuel activity includes this account in the next one-minute usage batch, whose - // response refreshes the executor's cached limits. - user.invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/initial.txt", "x"), - ) - .await?; - sleep(Duration::from_secs(65)).await; - - let result = user - .invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile.txt", "hello world"), - ) - .await; - assert!( - result.is_ok(), - "expected 11-byte write to fit after raising disk quota to 12 bytes, got: {result:?}" - ); - - Ok(()) -} - -/// Verifies that writing within the per-plan disk quota succeeds end-to-end -/// through the full Golem stack. Uses the default plan which has a generous -/// disk quota. -#[test] -#[tracing::instrument] -#[timeout("4m")] -async fn agent_write_within_per_plan_disk_space_quota_succeeds( - deps: &EnvBasedTestDependencies, - _tracing: &Tracing, -) -> anyhow::Result<()> { - let user = deps.user().await?; - let (_, env) = user.app_and_env().await?; - - // Default plan has a very large disk quota — any normal write succeeds. - let component = user - .component(&env.id, "golem_it_host_api_tests_release") - .name("golem-it:host-api-tests") - .store() - .await?; - - let agent_id = agent_id!("FileSystem", "disk-quota-ok-1"); - user.start_agent(&component.id, agent_id.clone()).await?; - - user.invoke_and_await_agent( - &component, - &agent_id, - "write_file", - data_value!("/testfile.txt", "hello world"), - ) - .await?; - - Ok(()) -} - /// A worker making HTTP calls is suspended when the account's monthly HTTP /// call limit is exhausted. When `remaining_http_calls() == 0`, the host function /// `handle()` raises `WorkerMonthlyHttpCallBudgetExhausted` which maps to diff --git a/openapi/golem-service.yaml b/openapi/golem-service.yaml index ef34a23d51..7dd7c36f59 100644 --- a/openapi/golem-service.yaml +++ b/openapi/golem-service.yaml @@ -13601,7 +13601,6 @@ components: SuccessfulUpdate: '#/components/schemas/PublicOplogEntry_r#SuccessfulUpdateParams' FailedUpdate: '#/components/schemas/PublicOplogEntry_r#FailedUpdateParams' GrowMemory: '#/components/schemas/PublicOplogEntry_r#GrowMemoryParams' - FilesystemStorageUsageUpdate: '#/components/schemas/PublicOplogEntry_r#FilesystemStorageUsageUpdateParams' CreateResource: '#/components/schemas/PublicOplogEntry_r#CreateResourceParams' DropResource: '#/components/schemas/PublicOplogEntry_r#DropResourceParams' Log: '#/components/schemas/PublicOplogEntry_r#LogParams' @@ -13650,7 +13649,6 @@ components: - $ref: '#/components/schemas/PublicOplogEntry_r#SuccessfulUpdateParams' - $ref: '#/components/schemas/PublicOplogEntry_r#FailedUpdateParams' - $ref: '#/components/schemas/PublicOplogEntry_r#GrowMemoryParams' - - $ref: '#/components/schemas/PublicOplogEntry_r#FilesystemStorageUsageUpdateParams' - $ref: '#/components/schemas/PublicOplogEntry_r#CreateResourceParams' - $ref: '#/components/schemas/PublicOplogEntry_r#DropResourceParams' - $ref: '#/components/schemas/PublicOplogEntry_r#LogParams' @@ -13966,18 +13964,6 @@ components: required: - type - $ref: '#/components/schemas/r#FailedUpdateParams' - PublicOplogEntry_r#FilesystemStorageUsageUpdateParams: - allOf: - - type: object - properties: - type: - example: FilesystemStorageUsageUpdate - type: string - enum: - - FilesystemStorageUsageUpdate - required: - - type - - $ref: '#/components/schemas/r#FilesystemStorageUsageUpdateParams' PublicOplogEntry_r#FinishSpanParams: allOf: - type: object @@ -16737,19 +16723,6 @@ components: required: - timestamp - targetRevision - r#FilesystemStorageUsageUpdateParams: - title: r#FilesystemStorageUsageUpdateParams - type: object - properties: - timestamp: - type: string - format: date-time - delta: - type: integer - format: int64 - required: - - timestamp - - delta r#FinishSpanParams: title: r#FinishSpanParams type: object diff --git a/openapi/golem-worker-service.yaml b/openapi/golem-worker-service.yaml index a99b1ad791..9cfa433e39 100644 --- a/openapi/golem-worker-service.yaml +++ b/openapi/golem-worker-service.yaml @@ -4482,7 +4482,6 @@ components: - $ref: '#/components/schemas/PublicOplogEntry_r#SuccessfulUpdateParams' - $ref: '#/components/schemas/PublicOplogEntry_r#FailedUpdateParams' - $ref: '#/components/schemas/PublicOplogEntry_r#GrowMemoryParams' - - $ref: '#/components/schemas/PublicOplogEntry_r#FilesystemStorageUsageUpdateParams' - $ref: '#/components/schemas/PublicOplogEntry_r#CreateResourceParams' - $ref: '#/components/schemas/PublicOplogEntry_r#DropResourceParams' - $ref: '#/components/schemas/PublicOplogEntry_r#LogParams' @@ -4532,7 +4531,6 @@ components: SuccessfulUpdate: '#/components/schemas/PublicOplogEntry_r#SuccessfulUpdateParams' FailedUpdate: '#/components/schemas/PublicOplogEntry_r#FailedUpdateParams' GrowMemory: '#/components/schemas/PublicOplogEntry_r#GrowMemoryParams' - FilesystemStorageUsageUpdate: '#/components/schemas/PublicOplogEntry_r#FilesystemStorageUsageUpdateParams' CreateResource: '#/components/schemas/PublicOplogEntry_r#CreateResourceParams' DropResource: '#/components/schemas/PublicOplogEntry_r#DropResourceParams' Log: '#/components/schemas/PublicOplogEntry_r#LogParams' @@ -4848,18 +4846,6 @@ components: - FailedUpdate example: FailedUpdate - $ref: '#/components/schemas/r#FailedUpdateParams' - PublicOplogEntry_r#FilesystemStorageUsageUpdateParams: - allOf: - - type: object - required: - - type - properties: - type: - type: string - enum: - - FilesystemStorageUsageUpdate - example: FilesystemStorageUsageUpdate - - $ref: '#/components/schemas/r#FilesystemStorageUsageUpdateParams' PublicOplogEntry_r#FinishSpanParams: allOf: - type: object @@ -7619,19 +7605,6 @@ components: format: uint64 details: type: string - r#FilesystemStorageUsageUpdateParams: - type: object - title: r#FilesystemStorageUsageUpdateParams - required: - - timestamp - - delta - properties: - timestamp: - type: string - format: date-time - delta: - type: integer - format: int64 r#FinishSpanParams: type: object title: r#FinishSpanParams diff --git a/plugins/otlp-exporter.wasm b/plugins/otlp-exporter.wasm index bda3fdf87d..8342396e44 100644 Binary files a/plugins/otlp-exporter.wasm and b/plugins/otlp-exporter.wasm differ diff --git a/plugins/otlp-exporter/.agents/skills/golem-debug-agent-history/SKILL.md b/plugins/otlp-exporter/.agents/skills/golem-debug-agent-history/SKILL.md index 395ad0629d..8b35df6391 100644 --- a/plugins/otlp-exporter/.agents/skills/golem-debug-agent-history/SKILL.md +++ b/plugins/otlp-exporter/.agents/skills/golem-debug-agent-history/SKILL.md @@ -63,7 +63,6 @@ Each text entry is printed with its index (e.g. `#00042:`) followed by a labeled | `INTERRUPTED` / `EXITED` | Agent interrupted or exited | | `NOP` | No-operation marker | | `JUMP` | Oplog jump — shows from/to indices | -| `STORAGE USAGE UPDATE` | Filesystem storage usage change | ### Examples diff --git a/plugins/otlp-exporter/components-rust/otlp-exporter/src/helpers.rs b/plugins/otlp-exporter/components-rust/otlp-exporter/src/helpers.rs index c8395a7765..e1519e85a2 100644 --- a/plugins/otlp-exporter/components-rust/otlp-exporter/src/helpers.rs +++ b/plugins/otlp-exporter/components-rust/otlp-exporter/src/helpers.rs @@ -79,8 +79,6 @@ pub(crate) fn worker_error_to_string(e: &WorkerError) -> String { WorkerError::ExceededTableLimit => "exceeded table limit".to_string(), WorkerError::ExceededHttpCallLimit => "exceeded http call limit".to_string(), WorkerError::ExceededRpcCallLimit => "exceeded rpc call limit".to_string(), - WorkerError::NodeOutOfFilesystemStorage => "node out of filesystem storage".to_string(), - WorkerError::AgentExceededFilesystemStorageLimit => "agent exceeded filesystem storage limit".to_string(), WorkerError::AgentTerminatedByQuota(_) => "agent terminated by quota".to_string(), WorkerError::EphemeralSleepTooLong(_) => "ephemeral sleep too long".to_string(), WorkerError::EphemeralFuelExhausted(_) => "ephemeral fuel exhausted".to_string(), diff --git a/plugins/otlp-exporter/components-rust/otlp-exporter/src/processing.rs b/plugins/otlp-exporter/components-rust/otlp-exporter/src/processing.rs index 126371a098..246c84db62 100644 --- a/plugins/otlp-exporter/components-rust/otlp-exporter/src/processing.rs +++ b/plugins/otlp-exporter/components-rust/otlp-exporter/src/processing.rs @@ -266,12 +266,6 @@ fn worker_error_variant_name(e: &golem_rust::bindings::golem::api::oplog::Worker golem_rust::bindings::golem::api::oplog::WorkerError::ExceededRpcCallLimit => { "ExceededRpcCallLimit".to_string() } - golem_rust::bindings::golem::api::oplog::WorkerError::NodeOutOfFilesystemStorage => { - "NodeOutOfFilesystemStorage".to_string() - } - golem_rust::bindings::golem::api::oplog::WorkerError::AgentExceededFilesystemStorageLimit => { - "AgentExceededFilesystemStorageLimit".to_string() - } golem_rust::bindings::golem::api::oplog::WorkerError::AgentTerminatedByQuota(_) => { "AgentTerminatedByQuota".to_string() } diff --git a/sdks/moonbit/golem_sdk/wit/deps/golem-1.x/golem-oplog.wit b/sdks/moonbit/golem_sdk/wit/deps/golem-1.x/golem-oplog.wit index 28f7d9d514..97ef23469d 100644 --- a/sdks/moonbit/golem_sdk/wit/deps/golem-1.x/golem-oplog.wit +++ b/sdks/moonbit/golem_sdk/wit/deps/golem-1.x/golem-oplog.wit @@ -406,11 +406,6 @@ interface oplog { delta: u64 } - record filesystem-storage-usage-update-parameters { - timestamp: datetime, - delta: s64 - } - type agent-resource-id = u64; record create-resource-parameters { @@ -566,8 +561,6 @@ interface oplog { exceeded-table-limit, exceeded-http-call-limit, exceeded-rpc-call-limit, - node-out-of-filesystem-storage, - agent-exceeded-filesystem-storage-limit, agent-terminated-by-quota(agent-terminated-by-quota-error), ephemeral-sleep-too-long(ephemeral-sleep-too-long), ephemeral-fuel-exhausted(ephemeral-fuel-exhausted), @@ -786,8 +779,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(raw-create-resource-parameters), /// Dropped a resource instance @@ -895,8 +886,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(create-resource-parameters), /// Dropped a resource instance diff --git a/sdks/rust/golem-rust/wit/deps/golem-1.x/golem-oplog.wit b/sdks/rust/golem-rust/wit/deps/golem-1.x/golem-oplog.wit index 28f7d9d514..97ef23469d 100644 --- a/sdks/rust/golem-rust/wit/deps/golem-1.x/golem-oplog.wit +++ b/sdks/rust/golem-rust/wit/deps/golem-1.x/golem-oplog.wit @@ -406,11 +406,6 @@ interface oplog { delta: u64 } - record filesystem-storage-usage-update-parameters { - timestamp: datetime, - delta: s64 - } - type agent-resource-id = u64; record create-resource-parameters { @@ -566,8 +561,6 @@ interface oplog { exceeded-table-limit, exceeded-http-call-limit, exceeded-rpc-call-limit, - node-out-of-filesystem-storage, - agent-exceeded-filesystem-storage-limit, agent-terminated-by-quota(agent-terminated-by-quota-error), ephemeral-sleep-too-long(ephemeral-sleep-too-long), ephemeral-fuel-exhausted(ephemeral-fuel-exhausted), @@ -786,8 +779,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(raw-create-resource-parameters), /// Dropped a resource instance @@ -895,8 +886,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(create-resource-parameters), /// Dropped a resource instance diff --git a/sdks/scala/core/js/src/main/scala/golem/host/OplogApi.scala b/sdks/scala/core/js/src/main/scala/golem/host/OplogApi.scala index 0f7a59643f..d837e29a74 100644 --- a/sdks/scala/core/js/src/main/scala/golem/host/OplogApi.scala +++ b/sdks/scala/core/js/src/main/scala/golem/host/OplogApi.scala @@ -150,11 +150,6 @@ object OplogApi { name: String ) - final case class FilesystemStorageUsageUpdateParameters( - timestamp: ContextApi.DateTime, - delta: BigInt - ) - final case class EndAtomicRegionParameters( timestamp: ContextApi.DateTime, beginIndex: OplogIndex @@ -369,9 +364,6 @@ object OplogApi { final case class RemoveRetryPolicy(params: RemoveRetryPolicyParameters) extends OplogEntry { def timestamp: ContextApi.DateTime = params.timestamp } - final case class FilesystemStorageUsageUpdate(params: FilesystemStorageUsageUpdateParameters) extends OplogEntry { - def timestamp: ContextApi.DateTime = params.timestamp - } final case class BeginAtomicRegion(ts: ContextApi.DateTime) extends OplogEntry { def timestamp: ContextApi.DateTime = ts } @@ -494,10 +486,6 @@ object OplogApi { SetRetryPolicy(parseSetRetryPolicyParameters(v.asInstanceOf[JsSetRetryPolicyParameters])) case "remove-retry-policy" => RemoveRetryPolicy(parseRemoveRetryPolicyParameters(v.asInstanceOf[JsRemoveRetryPolicyParameters])) - case "filesystem-storage-usage-update" => - FilesystemStorageUsageUpdate( - parseFilesystemStorageUsageUpdateParameters(v.asInstanceOf[JsFilesystemStorageUsageUpdateParameters]) - ) case "begin-atomic-region" => BeginAtomicRegion(parseTimestamp(v.asInstanceOf[JsOplogTimestamp])) case "end-atomic-region" => EndAtomicRegion(parseEndAtomicRegionParameters(v.asInstanceOf[JsEndAtomicRegionParameters])) @@ -800,14 +788,6 @@ object OplogApi { name = raw.name ) - private def parseFilesystemStorageUsageUpdateParameters( - raw: JsFilesystemStorageUsageUpdateParameters - ): FilesystemStorageUsageUpdateParameters = - FilesystemStorageUsageUpdateParameters( - timestamp = parseDateTime(raw.timestamp), - delta = BigInt(raw.delta.toString) - ) - private def parseEndAtomicRegionParameters(raw: JsEndAtomicRegionParameters): EndAtomicRegionParameters = EndAtomicRegionParameters( timestamp = parseDateTime(raw.timestamp), diff --git a/sdks/scala/core/js/src/main/scala/golem/host/js/OplogTypes.scala b/sdks/scala/core/js/src/main/scala/golem/host/js/OplogTypes.scala index 1d339a734d..cf6b8c5d66 100644 --- a/sdks/scala/core/js/src/main/scala/golem/host/js/OplogTypes.scala +++ b/sdks/scala/core/js/src/main/scala/golem/host/js/OplogTypes.scala @@ -193,14 +193,6 @@ sealed trait JsRemoveRetryPolicyParameters extends js.Object { def name: String = js.native } -// --- FilesystemStorageUsageUpdateParameters --- - -@js.native -sealed trait JsFilesystemStorageUsageUpdateParameters extends js.Object { - def timestamp: JsDatetime = js.native - def delta: js.BigInt = js.native -} - // --- EndAtomicRegionParameters --- @js.native diff --git a/sdks/scala/core/js/src/test/scala/golem/host/OplogApiCompileSpec.scala b/sdks/scala/core/js/src/test/scala/golem/host/OplogApiCompileSpec.scala index edd27bd6fe..de3f3ddcc3 100644 --- a/sdks/scala/core/js/src/test/scala/golem/host/OplogApiCompileSpec.scala +++ b/sdks/scala/core/js/src/test/scala/golem/host/OplogApiCompileSpec.scala @@ -95,7 +95,6 @@ object OplogApiCompileSpec extends ZIOSpecDefault { case OplogEntry.Exited(t) => s"exited(${t.seconds})" case OplogEntry.SetRetryPolicy(p) => s"set-retry(${p.name})" case OplogEntry.RemoveRetryPolicy(p) => s"remove-retry(${p.name})" - case OplogEntry.FilesystemStorageUsageUpdate(p) => s"fs-usage(${p.delta})" case OplogEntry.BeginAtomicRegion(t) => s"begin-atomic(${t.seconds})" case OplogEntry.EndAtomicRegion(p) => s"end-atomic(${p.beginIndex})" case OplogEntry.BeginRemoteWrite(t) => s"begin-rw(${t.seconds})" @@ -158,7 +157,6 @@ object OplogApiCompileSpec extends ZIOSpecDefault { SetRetryPolicyParameters(ts, "default", 0, """{"nodes":[]}""", """{"nodes":[]}""") ), OplogEntry.RemoveRetryPolicy(RemoveRetryPolicyParameters(ts, "default")), - OplogEntry.FilesystemStorageUsageUpdate(FilesystemStorageUsageUpdateParameters(ts, BigInt(4096))), OplogEntry.EndAtomicRegion(EndAtomicRegionParameters(ts, BigInt(1))), OplogEntry.EndRemoteWrite(EndRemoteWriteParameters(ts, BigInt(2))), OplogEntry.GrowMemory(GrowMemoryParameters(ts, BigInt(65536))), @@ -222,9 +220,9 @@ object OplogApiCompileSpec extends ZIOSpecDefault { ) def spec = suite("OplogApiCompileSpec")( - test("all 41 OplogEntry variants constructed") { + test("all 40 OplogEntry variants constructed") { val distinctTags = allEntries.map(describeEntry).map(_.takeWhile(_ != '(')).distinct - assertTrue(distinctTags.size >= 41) + assertTrue(distinctTags.size >= 40) }, test("exhaustive OplogEntry match compiles") { allEntries.foreach(e => Predef.assert(describeEntry(e).nonEmpty)) diff --git a/sdks/scala/core/js/src/test/scala/golem/host/OplogEntryRoundtripSpec.scala b/sdks/scala/core/js/src/test/scala/golem/host/OplogEntryRoundtripSpec.scala index c4820a5fb0..43ea033086 100644 --- a/sdks/scala/core/js/src/test/scala/golem/host/OplogEntryRoundtripSpec.scala +++ b/sdks/scala/core/js/src/test/scala/golem/host/OplogEntryRoundtripSpec.scala @@ -294,21 +294,6 @@ object OplogEntryRoundtripSpec extends ZIOSpecDefault { p.name == "default" ) }, - test("FilesystemStorageUsageUpdate from dynamic") { - val raw = wrapEntry( - "filesystem-storage-usage-update", - js.Dynamic.literal( - timestamp = ts(), - delta = js.BigInt("4096") - ) - ) - val parsed = OplogEntry.fromJs(raw) - val p = parsed.asInstanceOf[OplogEntry.FilesystemStorageUsageUpdate].params - assertTrue( - parsed.isInstanceOf[OplogEntry.FilesystemStorageUsageUpdate], - p.delta == BigInt(4096) - ) - }, test("Log from dynamic") { val raw = wrapEntry( "log", diff --git a/sdks/scala/wit/deps/golem-1.x/golem-oplog.wit b/sdks/scala/wit/deps/golem-1.x/golem-oplog.wit index 28f7d9d514..97ef23469d 100644 --- a/sdks/scala/wit/deps/golem-1.x/golem-oplog.wit +++ b/sdks/scala/wit/deps/golem-1.x/golem-oplog.wit @@ -406,11 +406,6 @@ interface oplog { delta: u64 } - record filesystem-storage-usage-update-parameters { - timestamp: datetime, - delta: s64 - } - type agent-resource-id = u64; record create-resource-parameters { @@ -566,8 +561,6 @@ interface oplog { exceeded-table-limit, exceeded-http-call-limit, exceeded-rpc-call-limit, - node-out-of-filesystem-storage, - agent-exceeded-filesystem-storage-limit, agent-terminated-by-quota(agent-terminated-by-quota-error), ephemeral-sleep-too-long(ephemeral-sleep-too-long), ephemeral-fuel-exhausted(ephemeral-fuel-exhausted), @@ -786,8 +779,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(raw-create-resource-parameters), /// Dropped a resource instance @@ -895,8 +886,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(create-resource-parameters), /// Dropped a resource instance diff --git a/sdks/scala/wit/dts/golem_api_1_5_0_oplog.d.ts b/sdks/scala/wit/dts/golem_api_1_5_0_oplog.d.ts index 206c291637..e987d18077 100644 --- a/sdks/scala/wit/dts/golem_api_1_5_0_oplog.d.ts +++ b/sdks/scala/wit/dts/golem_api_1_5_0_oplog.d.ts @@ -411,10 +411,6 @@ declare module 'golem:api/oplog@1.5.0' { timestamp: Datetime; delta: bigint; }; - export type FilesystemStorageUsageUpdateParameters = { - timestamp: Datetime; - delta: bigint; - }; export type AgentResourceId = bigint; export type CreateResourceParameters = { timestamp: Datetime; @@ -644,12 +640,6 @@ declare module 'golem:api/oplog@1.5.0' { { tag: 'exceeded-rpc-call-limit' } | - { - tag: 'node-out-of-filesystem-storage' - } | - { - tag: 'agent-exceeded-filesystem-storage-limit' - } | { tag: 'agent-terminated-by-quota' val: AgentTerminatedByQuotaError @@ -937,11 +927,6 @@ declare module 'golem:api/oplog@1.5.0' { tag: 'grow-memory' val: GrowMemoryParameters } | - /** Updated filesystem usage by a signed delta */ - { - tag: 'filesystem-storage-usage-update' - val: FilesystemStorageUsageUpdateParameters - } | /** Created a resource instance */ { tag: 'create-resource' @@ -1201,11 +1186,6 @@ declare module 'golem:api/oplog@1.5.0' { tag: 'grow-memory' val: GrowMemoryParameters } | - /** Updated filesystem usage by a signed delta */ - { - tag: 'filesystem-storage-usage-update' - val: FilesystemStorageUsageUpdateParameters - } | /** Created a resource instance */ { tag: 'create-resource' diff --git a/sdks/ts/packages/golem-ts-sdk/src/host/oplog.ts b/sdks/ts/packages/golem-ts-sdk/src/host/oplog.ts index 3850ac14b5..23d8b9b3c3 100644 --- a/sdks/ts/packages/golem-ts-sdk/src/host/oplog.ts +++ b/sdks/ts/packages/golem-ts-sdk/src/host/oplog.ts @@ -38,7 +38,6 @@ import type { SuccessfulUpdateParameters, FailedUpdateParameters, GrowMemoryParameters, - FilesystemStorageUsageUpdateParameters, CreateResourceParameters, DropResourceParameters, LogParameters, @@ -111,7 +110,6 @@ export type { SuccessfulUpdateParameters, FailedUpdateParameters, GrowMemoryParameters, - FilesystemStorageUsageUpdateParameters, AgentResourceId, CreateResourceParameters, DropResourceParameters, @@ -208,7 +206,6 @@ export type PublicOplogEntry = | { tag: 'successful-update'; val: SuccessfulUpdateParameters } | { tag: 'failed-update'; val: FailedUpdateParameters } | { tag: 'grow-memory'; val: GrowMemoryParameters } - | { tag: 'filesystem-storage-usage-update'; val: FilesystemStorageUsageUpdateParameters } | { tag: 'create-resource'; val: CreateResourceParameters } | { tag: 'drop-resource'; val: DropResourceParameters } | { tag: 'log'; val: LogParameters } diff --git a/sdks/ts/packages/golem-ts-sdk/types/golem_api_1_5_0_oplog.d.ts b/sdks/ts/packages/golem-ts-sdk/types/golem_api_1_5_0_oplog.d.ts index 206c291637..e987d18077 100644 --- a/sdks/ts/packages/golem-ts-sdk/types/golem_api_1_5_0_oplog.d.ts +++ b/sdks/ts/packages/golem-ts-sdk/types/golem_api_1_5_0_oplog.d.ts @@ -411,10 +411,6 @@ declare module 'golem:api/oplog@1.5.0' { timestamp: Datetime; delta: bigint; }; - export type FilesystemStorageUsageUpdateParameters = { - timestamp: Datetime; - delta: bigint; - }; export type AgentResourceId = bigint; export type CreateResourceParameters = { timestamp: Datetime; @@ -644,12 +640,6 @@ declare module 'golem:api/oplog@1.5.0' { { tag: 'exceeded-rpc-call-limit' } | - { - tag: 'node-out-of-filesystem-storage' - } | - { - tag: 'agent-exceeded-filesystem-storage-limit' - } | { tag: 'agent-terminated-by-quota' val: AgentTerminatedByQuotaError @@ -937,11 +927,6 @@ declare module 'golem:api/oplog@1.5.0' { tag: 'grow-memory' val: GrowMemoryParameters } | - /** Updated filesystem usage by a signed delta */ - { - tag: 'filesystem-storage-usage-update' - val: FilesystemStorageUsageUpdateParameters - } | /** Created a resource instance */ { tag: 'create-resource' @@ -1201,11 +1186,6 @@ declare module 'golem:api/oplog@1.5.0' { tag: 'grow-memory' val: GrowMemoryParameters } | - /** Updated filesystem usage by a signed delta */ - { - tag: 'filesystem-storage-usage-update' - val: FilesystemStorageUsageUpdateParameters - } | /** Created a resource instance */ { tag: 'create-resource' diff --git a/sdks/ts/wit/deps/golem-1.x/golem-oplog.wit b/sdks/ts/wit/deps/golem-1.x/golem-oplog.wit index 28f7d9d514..97ef23469d 100644 --- a/sdks/ts/wit/deps/golem-1.x/golem-oplog.wit +++ b/sdks/ts/wit/deps/golem-1.x/golem-oplog.wit @@ -406,11 +406,6 @@ interface oplog { delta: u64 } - record filesystem-storage-usage-update-parameters { - timestamp: datetime, - delta: s64 - } - type agent-resource-id = u64; record create-resource-parameters { @@ -566,8 +561,6 @@ interface oplog { exceeded-table-limit, exceeded-http-call-limit, exceeded-rpc-call-limit, - node-out-of-filesystem-storage, - agent-exceeded-filesystem-storage-limit, agent-terminated-by-quota(agent-terminated-by-quota-error), ephemeral-sleep-too-long(ephemeral-sleep-too-long), ephemeral-fuel-exhausted(ephemeral-fuel-exhausted), @@ -786,8 +779,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(raw-create-resource-parameters), /// Dropped a resource instance @@ -895,8 +886,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(create-resource-parameters), /// Dropped a resource instance diff --git a/test-components/initial-file-system/src/p3_parity.rs b/test-components/initial-file-system/src/p3_parity.rs deleted file mode 100644 index 9779ae1e40..0000000000 --- a/test-components/initial-file-system/src/p3_parity.rs +++ /dev/null @@ -1,246 +0,0 @@ -use golem_rust::wasip3::filesystem::preopens as p3_preopens; -use golem_rust::wasip3::filesystem::types as p3_types; -use golem_rust::{agent_definition, agent_implementation}; -use wasi::filesystem::preopens as p2_preopens; -use wasi::filesystem::types as p2_types; - -#[agent_definition] -pub trait P3FileSystem { - fn new(name: String) -> Self; - /// Runs the same filesystem operations through both the WASI 0.2 and WASI 0.3 imports - /// against a read-only and a read-write initial file, and reports `name=value` entries - /// so the host-side test can assert P2/P3 parity. - async fn run(&self) -> Vec; -} - -struct P3FileSystemImpl { - _name: String, -} - -fn p2_err(error: p2_types::ErrorCode) -> String { - match error { - p2_types::ErrorCode::NotPermitted => "not-permitted".to_string(), - other => format!("{other:?}"), - } -} - -fn p3_err(error: p3_types::ErrorCode) -> String { - match error { - p3_types::ErrorCode::NotPermitted => "not-permitted".to_string(), - other => format!("{other:?}"), - } -} - -fn p2_result(result: Result<(), p2_types::ErrorCode>) -> String { - match result { - Ok(()) => "ok".to_string(), - Err(error) => format!("err:{}", p2_err(error)), - } -} - -fn p3_result(result: Result<(), p3_types::ErrorCode>) -> String { - match result { - Ok(()) => "ok".to_string(), - Err(error) => format!("err:{}", p3_err(error)), - } -} - -#[agent_implementation] -impl P3FileSystem for P3FileSystemImpl { - fn new(name: String) -> Self { - Self { _name: name } - } - - async fn run(&self) -> Vec { - let mut results = Vec::new(); - - let (root_p2, _) = p2_preopens::get_directories() - .into_iter() - .next() - .expect("no P2 preopened directory"); - let (root_p3, _) = p3_preopens::get_directories() - .into_iter() - .next() - .expect("no P3 preopened directory"); - - let ro_p2 = root_p2 - .open_at( - p2_types::PathFlags::empty(), - "foo.txt", - p2_types::OpenFlags::empty(), - p2_types::DescriptorFlags::READ, - ) - .expect("P2 open of read-only file failed"); - let ro_p3 = root_p3 - .open_at( - p3_types::PathFlags::empty(), - "foo.txt".to_string(), - p3_types::OpenFlags::empty(), - p3_types::DescriptorFlags::READ, - ) - .await - .expect("P3 open of read-only file failed"); - - // get-flags must mask the write bit for read-only initial files - let ro_flags_p2 = ro_p2.get_flags().expect("P2 get_flags failed"); - let ro_flags_p3 = ro_p3.get_flags().await.expect("P3 get_flags failed"); - results.push(format!( - "ro_flags_p2_write={}", - ro_flags_p2.contains(p2_types::DescriptorFlags::WRITE) - )); - results.push(format!( - "ro_flags_p3_write={}", - ro_flags_p3.contains(p3_types::DescriptorFlags::WRITE) - )); - - // metadata-hash parity between P2 and P3 for the same unchanged file - let ro_hash_p2 = ro_p2.metadata_hash().expect("P2 metadata_hash failed"); - let ro_hash_p3 = ro_p3 - .metadata_hash() - .await - .expect("P3 metadata_hash failed"); - results.push(format!( - "ro_hash_parity={}", - ro_hash_p2.lower == ro_hash_p3.lower && ro_hash_p2.upper == ro_hash_p3.upper - )); - let ro_hash_p3_again = ro_p3 - .metadata_hash() - .await - .expect("P3 metadata_hash (2nd) failed"); - results.push(format!( - "ro_hash_p3_deterministic={}", - ro_hash_p3.lower == ro_hash_p3_again.lower && ro_hash_p3.upper == ro_hash_p3_again.upper - )); - - let ro_hash_at_p2 = root_p2 - .metadata_hash_at(p2_types::PathFlags::empty(), "foo.txt") - .expect("P2 metadata_hash_at failed"); - let ro_hash_at_p3 = root_p3 - .metadata_hash_at(p3_types::PathFlags::empty(), "foo.txt".to_string()) - .await - .expect("P3 metadata_hash_at failed"); - results.push(format!( - "ro_hash_at_parity={}", - ro_hash_at_p2.lower == ro_hash_at_p3.lower && ro_hash_at_p2.upper == ro_hash_at_p3.upper - )); - - // mutations through a read-only file descriptor must be rejected identically - results.push(format!( - "ro_set_times_p2={}", - p2_result(ro_p2.set_times(p2_types::NewTimestamp::Now, p2_types::NewTimestamp::Now)) - )); - results.push(format!( - "ro_set_times_p3={}", - p3_result( - ro_p3 - .set_times(p3_types::NewTimestamp::Now, p3_types::NewTimestamp::Now) - .await - ) - )); - results.push(format!( - "ro_set_times_at_p2={}", - p2_result(ro_p2.set_times_at( - p2_types::PathFlags::empty(), - "x", - p2_types::NewTimestamp::Now, - p2_types::NewTimestamp::Now - )) - )); - results.push(format!( - "ro_set_times_at_p3={}", - p3_result( - ro_p3 - .set_times_at( - p3_types::PathFlags::empty(), - "x".to_string(), - p3_types::NewTimestamp::Now, - p3_types::NewTimestamp::Now - ) - .await - ) - )); - results.push(format!( - "ro_rename_at_p2={}", - p2_result(ro_p2.rename_at("x", &root_p2, "y")) - )); - results.push(format!( - "ro_rename_at_p3={}", - p3_result( - ro_p3 - .rename_at("x".to_string(), &root_p3, "y".to_string()) - .await - ) - )); - results.push(format!( - "ro_symlink_at_p2={}", - p2_result(ro_p2.symlink_at("x", "y")) - )); - results.push(format!( - "ro_symlink_at_p3={}", - p3_result(ro_p3.symlink_at("x".to_string(), "y".to_string()).await) - )); - results.push(format!( - "ro_unlink_file_at_p2={}", - p2_result(ro_p2.unlink_file_at("x")) - )); - results.push(format!( - "ro_unlink_file_at_p3={}", - p3_result(ro_p3.unlink_file_at("x".to_string()).await) - )); - - let rw_p2 = root_p2 - .open_at( - p2_types::PathFlags::empty(), - "bar/baz.txt", - p2_types::OpenFlags::empty(), - p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, - ) - .expect("P2 open of read-write file failed"); - let rw_p3 = root_p3 - .open_at( - p3_types::PathFlags::empty(), - "bar/baz.txt".to_string(), - p3_types::OpenFlags::empty(), - p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, - ) - .await - .expect("P3 open of read-write file failed"); - - let rw_flags_p2 = rw_p2.get_flags().expect("P2 get_flags (rw) failed"); - let rw_flags_p3 = rw_p3.get_flags().await.expect("P3 get_flags (rw) failed"); - results.push(format!( - "rw_flags_p2_write={}", - rw_flags_p2.contains(p2_types::DescriptorFlags::WRITE) - )); - results.push(format!( - "rw_flags_p3_write={}", - rw_flags_p3.contains(p3_types::DescriptorFlags::WRITE) - )); - - let rw_hash_p2 = rw_p2.metadata_hash().expect("P2 metadata_hash (rw) failed"); - let rw_hash_p3 = rw_p3 - .metadata_hash() - .await - .expect("P3 metadata_hash (rw) failed"); - results.push(format!( - "rw_hash_parity={}", - rw_hash_p2.lower == rw_hash_p3.lower && rw_hash_p2.upper == rw_hash_p3.upper - )); - - // set-times on a read-write file must succeed through both versions - results.push(format!( - "rw_set_times_p2={}", - p2_result(rw_p2.set_times(p2_types::NewTimestamp::Now, p2_types::NewTimestamp::Now)) - )); - results.push(format!( - "rw_set_times_p3={}", - p3_result( - rw_p3 - .set_times(p3_types::NewTimestamp::Now, p3_types::NewTimestamp::Now) - .await - ) - )); - - results - } -} diff --git a/test-components/initial-file-system/src/p3_parity/mod.rs b/test-components/initial-file-system/src/p3_parity/mod.rs new file mode 100644 index 0000000000..d4f5529cfd --- /dev/null +++ b/test-components/initial-file-system/src/p3_parity/mod.rs @@ -0,0 +1,160 @@ +use golem_rust::{agent_definition, agent_implementation}; + +mod parity; +mod quota; + +#[agent_definition] +pub trait P3FileSystem { + fn new(name: String) -> Self; + /// Runs the same filesystem operations through both the WASI 0.2 and WASI 0.3 imports + /// against a read-only and a read-write initial file, and reports `name=value` entries + /// so the host-side test can assert P2/P3 parity. + async fn run(&self) -> Vec; + /// Reads the mutable initial file left by `run` through both WASI versions + /// without modifying it. + async fn inspect_run(&self) -> Vec; + /// Runs read-write filesystem operations through both WASI versions against + /// a file created by the agent. + async fn run_writable(&self) -> Vec; + /// Reads the file produced by `run_writable` through both WASI versions + /// without modifying it. + async fn inspect_writable(&self) -> Vec; + /// Applies filesystem mutation histories whose final bytes and topology are + /// inspected after reconstruction. + async fn run_reconstruction_matrix(&self) -> Vec; + /// Inspects the final bytes and topology produced by `run_reconstruction_matrix`. + async fn inspect_reconstruction_matrix(&self) -> Vec; + /// Replaces a file used to verify reconstruction to an exact oplog position. + async fn write_replay_target(&self, value: String); + /// Reads a file without changing it. + async fn inspect_path(&self, path: String) -> Vec; + /// Writes blocks through a P2 direct descriptor until the project quota + /// denies further growth. + async fn run_p2_quota_surface(&self) -> Vec; + /// Streams one block through P3 while project quota enforcement is active. + async fn run_p3_with_quota(&self) -> bool; + /// Attempts to stream more data through P3 than the project limit permits. + async fn exhaust_p3_quota(&self) -> Vec; + /// Attempts a P2 write larger than the available filesystem capacity. + async fn exhaust_p2_quota(&self) -> Vec; + /// Inspects the successfully persisted prefix of the failed P2 write. + async fn inspect_p2_exhaustion(&self) -> Vec; + /// Inspects the successfully persisted prefix of the failed P3 quota write. + async fn inspect_p3_exhaustion(&self) -> Vec; + /// Exercises the storage-affecting P2 filesystem operations under a project quota. + async fn run_p2_quota_matrix(&self) -> Vec; + /// Exercises the storage-affecting P3 filesystem operations under a project quota. + async fn run_p3_quota_matrix(&self) -> Vec; + /// Exhausts the P2 object quota and verifies open-unlinked inode accounting. + async fn run_p2_object_quota(&self) -> Vec; + /// Verifies P2 object capacity returns after the prior invocation closes its handles. + async fn complete_p2_object_quota_release(&self) -> bool; + /// Exhausts the P3 object quota and verifies open-unlinked inode accounting. + async fn run_p3_object_quota(&self) -> Vec; + /// Verifies P3 object capacity returns after the prior invocation closes its handles. + async fn complete_p3_object_quota_release(&self) -> bool; + /// Confirms that the guest invocation started without producing side effects. + async fn confirm_invocation_started(&self) -> String; + /// Verifies that abandoning only a P3 write completion future does not cancel + /// the write still driven by its input stream. + async fn abandon_p3_write_completion(&self) -> bool; +} + +struct P3FileSystemImpl { + _name: String, +} + +#[agent_implementation] +impl P3FileSystem for P3FileSystemImpl { + fn new(name: String) -> Self { + Self { _name: name } + } + + async fn run(&self) -> Vec { + parity::run().await + } + + async fn inspect_run(&self) -> Vec { + parity::inspect_run().await + } + + async fn run_writable(&self) -> Vec { + parity::run_writable().await + } + + async fn inspect_writable(&self) -> Vec { + parity::inspect_writable().await + } + + async fn run_reconstruction_matrix(&self) -> Vec { + parity::run_reconstruction_matrix().await + } + + async fn inspect_reconstruction_matrix(&self) -> Vec { + parity::inspect_reconstruction_matrix().await + } + + async fn write_replay_target(&self, value: String) { + parity::write_replay_target(&value) + } + + async fn inspect_path(&self, path: String) -> Vec { + parity::inspect_file(&path).await + } + + async fn run_p2_quota_surface(&self) -> Vec { + quota::run_p2_quota_surface().await + } + + async fn run_p3_with_quota(&self) -> bool { + quota::run_p3_with_quota().await + } + + async fn exhaust_p3_quota(&self) -> Vec { + quota::exhaust_p3_quota().await + } + + async fn exhaust_p2_quota(&self) -> Vec { + quota::exhaust_p2_quota() + } + + async fn inspect_p2_exhaustion(&self) -> Vec { + quota::inspect_p2_exhaustion() + } + + async fn inspect_p3_exhaustion(&self) -> Vec { + quota::inspect_p3_exhaustion().await + } + + async fn run_p2_quota_matrix(&self) -> Vec { + quota::run_p2_quota_matrix().await + } + + async fn run_p3_quota_matrix(&self) -> Vec { + quota::run_p3_quota_matrix().await + } + + async fn run_p2_object_quota(&self) -> Vec { + quota::run_p2_object_quota().await + } + + async fn complete_p2_object_quota_release(&self) -> bool { + quota::complete_p2_object_quota_release().await + } + + async fn run_p3_object_quota(&self) -> Vec { + quota::run_p3_object_quota().await + } + + async fn complete_p3_object_quota_release(&self) -> bool { + quota::complete_p3_object_quota_release().await + } + + async fn confirm_invocation_started(&self) -> String { + "executed".to_string() + } + + async fn abandon_p3_write_completion(&self) -> bool { + parity::abandon_p3_write_completion().await + } +} diff --git a/test-components/initial-file-system/src/p3_parity/parity.rs b/test-components/initial-file-system/src/p3_parity/parity.rs new file mode 100644 index 0000000000..dbf96d5401 --- /dev/null +++ b/test-components/initial-file-system/src/p3_parity/parity.rs @@ -0,0 +1,1045 @@ +use golem_rust::wasip3::filesystem::preopens as p3_preopens; +use golem_rust::wasip3::filesystem::types as p3_types; +use wasi::filesystem::preopens as p2_preopens; +use wasi::filesystem::types as p2_types; +use wasip3::wit_stream; + +const P2_RECONSTRUCTION_TIMESTAMP_SECONDS: u64 = 946_684_800; +const P3_RECONSTRUCTION_TIMESTAMP_SECONDS: i64 = 978_307_200; + +fn p2_err(error: p2_types::ErrorCode) -> String { + match error { + p2_types::ErrorCode::NotPermitted => "not-permitted".to_string(), + other => format!("{other:?}"), + } +} + +fn p3_err(error: p3_types::ErrorCode) -> String { + match error { + p3_types::ErrorCode::NotPermitted => "not-permitted".to_string(), + other => format!("{other:?}"), + } +} + +fn p2_result(result: Result<(), p2_types::ErrorCode>) -> String { + match result { + Ok(()) => "ok".to_string(), + Err(error) => format!("err:{}", p2_err(error)), + } +} + +fn p3_result(result: Result<(), p3_types::ErrorCode>) -> String { + match result { + Ok(()) => "ok".to_string(), + Err(error) => format!("err:{}", p3_err(error)), + } +} + +pub(crate) async fn run() -> Vec { + let mut results = Vec::new(); + + let (root_p2, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let (root_p3, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + + let ro_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "foo.txt", + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::READ, + ) + .expect("P2 open of read-only file failed"); + let ro_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + "foo.txt".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ, + ) + .await + .expect("P3 open of read-only file failed"); + + // get-flags must mask the write bit for read-only initial files + let ro_flags_p2 = ro_p2.get_flags().expect("P2 get_flags failed"); + let ro_flags_p3 = ro_p3.get_flags().await.expect("P3 get_flags failed"); + results.push(format!( + "ro_flags_p2_write={}", + ro_flags_p2.contains(p2_types::DescriptorFlags::WRITE) + )); + results.push(format!( + "ro_flags_p3_write={}", + ro_flags_p3.contains(p3_types::DescriptorFlags::WRITE) + )); + + // metadata-hash parity between P2 and P3 for the same unchanged file + let ro_hash_p2 = ro_p2.metadata_hash().expect("P2 metadata_hash failed"); + let ro_hash_p3 = ro_p3 + .metadata_hash() + .await + .expect("P3 metadata_hash failed"); + results.push(format!( + "ro_hash_parity={}", + ro_hash_p2.lower == ro_hash_p3.lower && ro_hash_p2.upper == ro_hash_p3.upper + )); + let ro_hash_p3_again = ro_p3 + .metadata_hash() + .await + .expect("P3 metadata_hash (2nd) failed"); + results.push(format!( + "ro_hash_p3_deterministic={}", + ro_hash_p3.lower == ro_hash_p3_again.lower && ro_hash_p3.upper == ro_hash_p3_again.upper + )); + + let ro_hash_at_p2 = root_p2 + .metadata_hash_at(p2_types::PathFlags::empty(), "foo.txt") + .expect("P2 metadata_hash_at failed"); + let ro_hash_at_p3 = root_p3 + .metadata_hash_at(p3_types::PathFlags::empty(), "foo.txt".to_string()) + .await + .expect("P3 metadata_hash_at failed"); + results.push(format!( + "ro_hash_at_parity={}", + ro_hash_at_p2.lower == ro_hash_at_p3.lower && ro_hash_at_p2.upper == ro_hash_at_p3.upper + )); + + // mutations through a read-only file descriptor must be rejected identically + results.push(format!( + "ro_set_times_p2={}", + p2_result(ro_p2.set_times(p2_types::NewTimestamp::Now, p2_types::NewTimestamp::Now)) + )); + results.push(format!( + "ro_set_times_p3={}", + p3_result( + ro_p3 + .set_times(p3_types::NewTimestamp::Now, p3_types::NewTimestamp::Now) + .await + ) + )); + results.push(format!( + "ro_set_times_at_p2={}", + p2_result(ro_p2.set_times_at( + p2_types::PathFlags::empty(), + "x", + p2_types::NewTimestamp::Now, + p2_types::NewTimestamp::Now + )) + )); + results.push(format!( + "ro_set_times_at_p3={}", + p3_result( + ro_p3 + .set_times_at( + p3_types::PathFlags::empty(), + "x".to_string(), + p3_types::NewTimestamp::Now, + p3_types::NewTimestamp::Now + ) + .await + ) + )); + results.push(format!( + "ro_rename_at_p2={}", + p2_result(ro_p2.rename_at("x", &root_p2, "y")) + )); + results.push(format!( + "ro_rename_at_p3={}", + p3_result( + ro_p3 + .rename_at("x".to_string(), &root_p3, "y".to_string()) + .await + ) + )); + results.push(format!( + "ro_symlink_at_p2={}", + p2_result(ro_p2.symlink_at("x", "y")) + )); + results.push(format!( + "ro_symlink_at_p3={}", + p3_result(ro_p3.symlink_at("x".to_string(), "y".to_string()).await) + )); + results.push(format!( + "ro_unlink_file_at_p2={}", + p2_result(ro_p2.unlink_file_at("x")) + )); + results.push(format!( + "ro_unlink_file_at_p3={}", + p3_result(ro_p3.unlink_file_at("x".to_string()).await) + )); + results.push(format!( + "ro_parent_open_write_p2={}", + match root_p2.open_at( + p2_types::PathFlags::empty(), + "foo.txt", + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::WRITE, + ) { + Ok(_) => "ok".to_string(), + Err(error) => format!("err:{}", p2_err(error)), + } + )); + results.push(format!( + "ro_parent_open_write_p3={}", + match root_p3 + .open_at( + p3_types::PathFlags::empty(), + "foo.txt".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::WRITE, + ) + .await + { + Ok(_) => "ok".to_string(), + Err(error) => format!("err:{}", p3_err(error)), + } + )); + results.push(format!( + "ro_parent_unlink_p2={}", + p2_result(root_p2.unlink_file_at("foo.txt")) + )); + results.push(format!( + "ro_parent_unlink_p3={}", + p3_result(root_p3.unlink_file_at("foo.txt".to_string()).await) + )); + results.push(format!( + "ro_parent_rename_p2={}", + p2_result(root_p2.rename_at("foo.txt", &root_p2, "foo-moved.txt")) + )); + results.push(format!( + "ro_parent_rename_p3={}", + p3_result( + root_p3 + .rename_at("foo.txt".to_string(), &root_p3, "foo-moved.txt".to_string(),) + .await + ) + )); + results.push(format!( + "ro_parent_link_p2={}", + p2_result(root_p2.link_at( + p2_types::PathFlags::empty(), + "foo.txt", + &root_p2, + "foo-alias.txt", + )) + )); + results.push(format!( + "ro_parent_link_p3={}", + p3_result( + root_p3 + .link_at( + p3_types::PathFlags::empty(), + "foo.txt".to_string(), + &root_p3, + "foo-alias.txt".to_string(), + ) + .await + ) + )); + results.push(format!( + "ro_alias_create_p2={}", + p2_result(root_p2.symlink_at("foo.txt", "foo-link-p2")) + )); + results.push(format!( + "ro_alias_open_write_p2={}", + match root_p2.open_at( + p2_types::PathFlags::SYMLINK_FOLLOW, + "foo-link-p2", + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::WRITE, + ) { + Ok(_) => "ok".to_string(), + Err(error) => format!("err:{}", p2_err(error)), + } + )); + results.push(format!( + "ro_alias_unlink_p2={}", + p2_result(root_p2.unlink_file_at("foo-link-p2")) + )); + results.push(format!( + "ro_alias_create_p3={}", + p3_result( + root_p3 + .symlink_at("foo.txt".to_string(), "foo-link-p3".to_string()) + .await + ) + )); + results.push(format!( + "ro_alias_open_write_p3={}", + match root_p3 + .open_at( + p3_types::PathFlags::SYMLINK_FOLLOW, + "foo-link-p3".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::WRITE, + ) + .await + { + Ok(_) => "ok".to_string(), + Err(error) => format!("err:{}", p3_err(error)), + } + )); + results.push(format!( + "ro_alias_unlink_p3={}", + p3_result(root_p3.unlink_file_at("foo-link-p3".to_string()).await) + )); + + let rw_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "bar/baz.txt", + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("P2 open of read-write file failed"); + let rw_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + "bar/baz.txt".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("P3 open of read-write file failed"); + + let rw_flags_p2 = rw_p2.get_flags().expect("P2 get_flags (rw) failed"); + let rw_flags_p3 = rw_p3.get_flags().await.expect("P3 get_flags (rw) failed"); + results.push(format!( + "rw_flags_p2_write={}", + rw_flags_p2.contains(p2_types::DescriptorFlags::WRITE) + )); + results.push(format!( + "rw_flags_p3_write={}", + rw_flags_p3.contains(p3_types::DescriptorFlags::WRITE) + )); + + let rw_hash_p2 = rw_p2.metadata_hash().expect("P2 metadata_hash (rw) failed"); + let rw_hash_p3 = rw_p3 + .metadata_hash() + .await + .expect("P3 metadata_hash (rw) failed"); + results.push(format!( + "rw_hash_parity={}", + rw_hash_p2.lower == rw_hash_p3.lower && rw_hash_p2.upper == rw_hash_p3.upper + )); + + // set-times on a read-write file must succeed through both versions + results.push(format!( + "rw_set_times_p2={}", + p2_result(rw_p2.set_times(p2_types::NewTimestamp::Now, p2_types::NewTimestamp::Now)) + )); + results.push(format!( + "rw_set_times_p3={}", + p3_result( + rw_p3 + .set_times(p3_types::NewTimestamp::Now, p3_types::NewTimestamp::Now) + .await + ) + )); + + rw_p2.write(b"p2-to-p3", 0).expect("P2 write failed"); + let (p3_read, p3_read_result) = rw_p3.read_via_stream(0); + let p3_bytes = p3_read.collect().await; + p3_read_result.await.expect("P3 read after P2 write failed"); + results.push(format!( + "p2_write_p3_read={}", + String::from_utf8(p3_bytes).expect("P3 read was not UTF-8") + )); + + let (mut p3_write, p3_write_data) = wit_stream::new(); + let p3_write_result = rw_p3.write_via_stream(p3_write_data, 0); + let unwritten = p3_write.write_all(b"p3-to-p2".to_vec()).await; + assert!(unwritten.is_empty(), "P3 stream did not accept all bytes"); + drop(p3_write); + p3_write_result.await.expect("P3 write failed"); + let (p2_bytes, _) = rw_p2.read(8, 0).expect("P2 read after P3 write failed"); + results.push(format!( + "p3_write_p2_read={}", + String::from_utf8(p2_bytes).expect("P2 read was not UTF-8") + )); + + results +} + +pub(crate) async fn run_writable() -> Vec { + let (root_p2, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let (root_p3, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + + let file_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "managed-parity.txt", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("P2 file creation failed"); + let file_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + "managed-parity.txt".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("P3 file open failed"); + + file_p2.write(b"p2-to-p3", 0).expect("P2 write failed"); + let (p3_read, p3_read_result) = file_p3.read_via_stream(0); + let p3_bytes = p3_read.collect().await; + p3_read_result.await.expect("P3 read after P2 write failed"); + + let (mut p3_write, p3_write_data) = wit_stream::new(); + let p3_write_result = file_p3.write_via_stream(p3_write_data, 0); + let unwritten = p3_write.write_all(b"p3-to-p2".to_vec()).await; + assert!(unwritten.is_empty(), "P3 stream did not accept all bytes"); + drop(p3_write); + p3_write_result.await.expect("P3 write failed"); + let (p2_bytes, _) = file_p2.read(8, 0).expect("P2 read after P3 write failed"); + + vec![ + format!( + "p2_write_p3_read={}", + String::from_utf8(p3_bytes).expect("P3 read was not UTF-8") + ), + format!( + "p3_write_p2_read={}", + String::from_utf8(p2_bytes).expect("P2 read was not UTF-8") + ), + ] +} + +pub(crate) async fn inspect_file(path: &str) -> Vec { + let (root_p2, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let (root_p3, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + let file_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + path, + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::READ, + ) + .expect("P2 file open failed"); + let file_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + path.to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ, + ) + .await + .expect("P3 file open failed"); + + let (p2_bytes, _) = file_p2.read(64, 0).expect("P2 read failed"); + let (p3_read, p3_read_result) = file_p3.read_via_stream(0); + let p3_bytes = p3_read.collect().await; + p3_read_result.await.expect("P3 read failed"); + + vec![ + format!( + "p2_read={}", + String::from_utf8(p2_bytes).expect("P2 read was not UTF-8") + ), + format!( + "p3_read={}", + String::from_utf8(p3_bytes).expect("P3 read was not UTF-8") + ), + ] +} + +pub(crate) async fn inspect_run() -> Vec { + inspect_file("bar/baz.txt").await +} + +pub(crate) async fn inspect_writable() -> Vec { + inspect_file("managed-parity.txt").await +} + +pub(crate) async fn abandon_p3_write_completion() -> bool { + let (root, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + let file = root + .open_at( + p3_types::PathFlags::empty(), + "abandoned-completion.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create abandoned-completion file"); + let (mut writer, data) = wit_stream::new(); + let completion = file.write_via_stream(data, 0); + drop(completion); + let expected = b"input-stream-still-drives-write".to_vec(); + assert!(writer.write_all(expected.clone()).await.is_empty()); + drop(writer); + file.sync_data() + .await + .expect("synchronize abandoned-completion write"); + + let (reader, completion) = file.read_via_stream(0); + let actual = reader.collect().await; + completion.await.expect("read abandoned-completion file"); + actual == expected +} + +async fn write_p3(file: &p3_types::Descriptor, bytes: &[u8], offset: u64) { + let (mut writer, data) = wit_stream::new(); + let result = file.write_via_stream(data, offset); + assert!( + writer.write_all(bytes.to_vec()).await.is_empty(), + "P3 stream did not accept all bytes" + ); + drop(writer); + result.await.expect("P3 write failed"); +} + +pub(crate) async fn run_reconstruction_matrix() -> Vec { + let (root_p2, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let (root_p3, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + + let resized_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-p2-resize.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 resize file"); + resized_p2 + .write(b"abcdefghijkl", 0) + .expect("write P2 resize file"); + resized_p2.set_size(10).expect("resize P2 file"); + resized_p2.set_size(6).expect("truncate P2 file"); + resized_p2 + .set_times( + p2_types::NewTimestamp::Timestamp(wasi::clocks::wall_clock::Datetime { + seconds: P2_RECONSTRUCTION_TIMESTAMP_SECONDS, + nanoseconds: 0, + }), + p2_types::NewTimestamp::Timestamp(wasi::clocks::wall_clock::Datetime { + seconds: P2_RECONSTRUCTION_TIMESTAMP_SECONDS, + nanoseconds: 0, + }), + ) + .expect("set P2 reconstruction timestamps"); + + let appended_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-p2-append.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 append file"); + appended_p2 + .write(b"p2-", 0) + .expect("write P2 append prefix"); + appended_p2 + .append_via_stream() + .expect("open P2 append stream") + .blocking_write_and_flush(b"append") + .expect("append P2 bytes"); + + root_p2 + .create_directory_at("replay-p2-directory") + .expect("create P2 directory"); + root_p2 + .create_directory_at("replay-p2-directory/removed") + .expect("create removable P2 directory"); + root_p2 + .remove_directory_at("replay-p2-directory/removed") + .expect("remove P2 directory"); + + let splice_source = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-splice-source.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 splice source"); + splice_source + .write(b"splice-data", 0) + .expect("write P2 splice source"); + let splice_target = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-splice-target.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 splice target"); + let splice_input = splice_source + .read_via_stream(0) + .expect("open P2 splice input"); + let splice_output = splice_target + .write_via_stream(0) + .expect("open P2 splice output"); + assert_eq!( + splice_output + .blocking_splice(&splice_input, 11) + .expect("splice P2 bytes"), + 11 + ); + splice_output.blocking_flush().expect("flush P2 splice"); + + let hard_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-p2-hard.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 hard-link source"); + hard_p2 + .write(b"hard-p2", 0) + .expect("write P2 hard-link source"); + root_p2 + .link_at( + p2_types::PathFlags::empty(), + "replay-p2-hard.bin", + &root_p2, + "replay-p2-hard-link.bin", + ) + .expect("create P2 hard link"); + root_p2 + .symlink_at("replay-p2-hard.bin", "replay-p2-symlink.bin") + .expect("create P2 symlink"); + + let replacement_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-p2-replacement.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 replacement"); + replacement_p2 + .write(b"old", 0) + .expect("write P2 replacement"); + let replacement_source_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-p2-replacement-source.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 replacement source"); + replacement_source_p2 + .write(b"new-p2", 0) + .expect("write P2 replacement source"); + drop((replacement_p2, replacement_source_p2)); + root_p2 + .rename_at( + "replay-p2-replacement-source.bin", + &root_p2, + "replay-p2-replacement.bin", + ) + .expect("replace P2 file by rename"); + + let unlinked_p2 = root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-p2-unlinked.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 open-unlinked file"); + unlinked_p2 + .write(b"hidden-p2", 0) + .expect("write P2 open-unlinked file"); + root_p2 + .unlink_file_at("replay-p2-unlinked.bin") + .expect("unlink open P2 file"); + assert_eq!( + unlinked_p2 + .read(9, 0) + .expect("read open-unlinked P2 file") + .0, + b"hidden-p2" + ); + drop(unlinked_p2); + + let resized_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + "replay-p3-resize.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 resize file"); + write_p3(&resized_p3, b"uvwxyzABCDEF", 0).await; + resized_p3.set_size(10).await.expect("resize P3 file"); + resized_p3.set_size(6).await.expect("truncate P3 file"); + resized_p3 + .set_times( + p3_types::NewTimestamp::Timestamp(wasip3::clocks::system_clock::Instant { + seconds: P3_RECONSTRUCTION_TIMESTAMP_SECONDS, + nanoseconds: 0, + }), + p3_types::NewTimestamp::Timestamp(wasip3::clocks::system_clock::Instant { + seconds: P3_RECONSTRUCTION_TIMESTAMP_SECONDS, + nanoseconds: 0, + }), + ) + .await + .expect("set P3 reconstruction timestamps"); + + let appended_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + "replay-p3-append.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 append file"); + write_p3(&appended_p3, b"p3-", 0).await; + let (mut append_writer, append_data) = wit_stream::new(); + let append_result = appended_p3.append_via_stream(append_data); + assert!(append_writer.write_all(b"append".to_vec()).await.is_empty()); + drop(append_writer); + append_result.await.expect("append P3 bytes"); + + root_p3 + .create_directory_at("replay-p3-directory".to_string()) + .await + .expect("create P3 directory"); + root_p3 + .create_directory_at("replay-p3-directory/removed".to_string()) + .await + .expect("create removable P3 directory"); + root_p3 + .remove_directory_at("replay-p3-directory/removed".to_string()) + .await + .expect("remove P3 directory"); + + let hard_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + "replay-p3-hard.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 hard-link source"); + write_p3(&hard_p3, b"hard-p3", 0).await; + root_p3 + .link_at( + p3_types::PathFlags::empty(), + "replay-p3-hard.bin".to_string(), + &root_p3, + "replay-p3-hard-link.bin".to_string(), + ) + .await + .expect("create P3 hard link"); + root_p3 + .symlink_at( + "replay-p3-hard.bin".to_string(), + "replay-p3-symlink.bin".to_string(), + ) + .await + .expect("create P3 symlink"); + + let replacement_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + "replay-p3-replacement.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 replacement"); + write_p3(&replacement_p3, b"old", 0).await; + let replacement_source_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + "replay-p3-replacement-source.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 replacement source"); + write_p3(&replacement_source_p3, b"new-p3", 0).await; + drop((replacement_p3, replacement_source_p3)); + root_p3 + .rename_at( + "replay-p3-replacement-source.bin".to_string(), + &root_p3, + "replay-p3-replacement.bin".to_string(), + ) + .await + .expect("replace P3 file by rename"); + + let unlinked_p3 = root_p3 + .open_at( + p3_types::PathFlags::empty(), + "replay-p3-unlinked.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 open-unlinked file"); + write_p3(&unlinked_p3, b"hidden-p3", 0).await; + root_p3 + .unlink_file_at("replay-p3-unlinked.bin".to_string()) + .await + .expect("unlink open P3 file"); + assert_eq!( + unlinked_p3 + .stat() + .await + .expect("stat open-unlinked P3 file") + .size, + 9 + ); + drop(unlinked_p3); + + inspect_reconstruction_matrix().await +} + +pub(crate) async fn inspect_reconstruction_matrix() -> Vec { + let (root_p2, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let (root_p3, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + + let inspect_p2 = |path: &str| { + let file = root_p2 + .open_at( + p2_types::PathFlags::empty(), + path, + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::READ, + ) + .expect("open reconstructed P2 file"); + let stat = file.stat().expect("stat reconstructed P2 file"); + let (bytes, _) = file.read(64, 0).expect("read reconstructed P2 file"); + ( + String::from_utf8(bytes).expect("P2 bytes were not UTF-8"), + stat.link_count, + stat.data_modification_timestamp, + ) + }; + let p2_resize = inspect_p2("replay-p2-resize.bin"); + let p2_append = inspect_p2("replay-p2-append.bin"); + let p2_splice = inspect_p2("replay-splice-target.bin"); + let p2_hard = inspect_p2("replay-p2-hard.bin"); + let p2_hard_link = inspect_p2("replay-p2-hard-link.bin"); + let p2_replacement = inspect_p2("replay-p2-replacement.bin"); + let p2_symlink = root_p2 + .readlink_at("replay-p2-symlink.bin") + .expect("read reconstructed P2 symlink"); + let p2_symlink_bytes = root_p2 + .open_at( + p2_types::PathFlags::SYMLINK_FOLLOW, + "replay-p2-symlink.bin", + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::READ, + ) + .expect("follow reconstructed P2 symlink") + .read(64, 0) + .expect("read reconstructed P2 symlink target") + .0; + + async fn inspect_p3( + root: &p3_types::Descriptor, + path: &str, + ) -> (String, u64, Option) { + let file = root + .open_at( + p3_types::PathFlags::empty(), + path.to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ, + ) + .await + .expect("open reconstructed P3 file"); + let stat = file.stat().await.expect("stat reconstructed P3 file"); + let (reader, result) = file.read_via_stream(0); + let bytes = reader.collect().await; + result.await.expect("read reconstructed P3 file"); + ( + String::from_utf8(bytes).expect("P3 bytes were not UTF-8"), + stat.link_count, + stat.data_modification_timestamp, + ) + } + let p3_resize = inspect_p3(&root_p3, "replay-p3-resize.bin").await; + let p3_append = inspect_p3(&root_p3, "replay-p3-append.bin").await; + let p3_hard = inspect_p3(&root_p3, "replay-p3-hard.bin").await; + let p3_hard_link = inspect_p3(&root_p3, "replay-p3-hard-link.bin").await; + let p3_replacement = inspect_p3(&root_p3, "replay-p3-replacement.bin").await; + let p3_symlink = root_p3 + .readlink_at("replay-p3-symlink.bin".to_string()) + .await + .expect("read reconstructed P3 symlink"); + let p3_symlink_file = root_p3 + .open_at( + p3_types::PathFlags::SYMLINK_FOLLOW, + "replay-p3-symlink.bin".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ, + ) + .await + .expect("follow reconstructed P3 symlink"); + let (p3_symlink_reader, p3_symlink_result) = p3_symlink_file.read_via_stream(0); + let p3_symlink_bytes = p3_symlink_reader.collect().await; + p3_symlink_result + .await + .expect("read reconstructed P3 symlink target"); + + vec![ + format!("p2-resize={}", p2_resize.0), + format!( + "p2-times={}:{}", + p2_resize + .2 + .expect("P2 modification timestamp missing") + .seconds, + p2_resize + .2 + .expect("P2 modification timestamp missing") + .nanoseconds + ), + format!("p2-append={}", p2_append.0), + format!( + "p2-directory={}:removed={}", + root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-p2-directory", + p2_types::OpenFlags::DIRECTORY, + p2_types::DescriptorFlags::READ, + ) + .is_ok(), + root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-p2-directory/removed", + p2_types::OpenFlags::DIRECTORY, + p2_types::DescriptorFlags::READ, + ) + .is_err() + ), + format!("p2-splice={}", p2_splice.0), + format!("p2-hard={}:{}", p2_hard.0, p2_hard.1), + format!("p2-hard-link={}:{}", p2_hard_link.0, p2_hard_link.1), + format!( + "p2-symlink={}:{}", + p2_symlink, + String::from_utf8(p2_symlink_bytes).expect("P2 symlink bytes were not UTF-8") + ), + format!("p2-replacement={}", p2_replacement.0), + format!( + "p2-open-unlinked-absent={}", + root_p2 + .open_at( + p2_types::PathFlags::empty(), + "replay-p2-unlinked.bin", + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::READ, + ) + .is_err() + ), + format!("p3-resize={}", p3_resize.0), + format!( + "p3-times={}:{}", + p3_resize + .2 + .expect("P3 modification timestamp missing") + .seconds, + p3_resize + .2 + .expect("P3 modification timestamp missing") + .nanoseconds + ), + format!("p3-append={}", p3_append.0), + format!( + "p3-directory={}:removed={}", + root_p3 + .open_at( + p3_types::PathFlags::empty(), + "replay-p3-directory".to_string(), + p3_types::OpenFlags::DIRECTORY, + p3_types::DescriptorFlags::READ, + ) + .await + .is_ok(), + root_p3 + .open_at( + p3_types::PathFlags::empty(), + "replay-p3-directory/removed".to_string(), + p3_types::OpenFlags::DIRECTORY, + p3_types::DescriptorFlags::READ, + ) + .await + .is_err() + ), + format!("p3-hard={}:{}", p3_hard.0, p3_hard.1), + format!("p3-hard-link={}:{}", p3_hard_link.0, p3_hard_link.1), + format!( + "p3-symlink={p3_symlink}:{}", + String::from_utf8(p3_symlink_bytes).expect("P3 symlink bytes were not UTF-8") + ), + format!("p3-replacement={}", p3_replacement.0), + format!( + "p3-open-unlinked-absent={}", + root_p3 + .open_at( + p3_types::PathFlags::empty(), + "replay-p3-unlinked.bin".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ, + ) + .await + .is_err() + ), + ] +} + +pub(crate) fn write_replay_target(value: &str) { + let (root, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let file = root + .open_at( + p2_types::PathFlags::empty(), + "replay-target.txt", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::WRITE, + ) + .expect("open replay target"); + file.write(value.as_bytes(), 0) + .expect("write replay target"); +} diff --git a/test-components/initial-file-system/src/p3_parity/quota.rs b/test-components/initial-file-system/src/p3_parity/quota.rs new file mode 100644 index 0000000000..2eb35637aa --- /dev/null +++ b/test-components/initial-file-system/src/p3_parity/quota.rs @@ -0,0 +1,695 @@ +use golem_rust::wasip3::filesystem::preopens as p3_preopens; +use golem_rust::wasip3::filesystem::types as p3_types; +use wasi::filesystem::preopens as p2_preopens; +use wasi::filesystem::types as p2_types; +use wasi::io::streams::StreamError as P2StreamError; +use wasip3::wit_stream; + +fn p3_result(result: Result<(), p3_types::ErrorCode>) -> String { + match result { + Ok(()) => "ok".to_string(), + Err(p3_types::ErrorCode::NotPermitted) => "err:not-permitted".to_string(), + Err(p3_types::ErrorCode::Quota) => "err:quota".to_string(), + Err(p3_types::ErrorCode::InsufficientSpace) => "err:insufficient-space".to_string(), + Err(error) => format!("err:{error:?}"), + } +} + +fn p2_stream_result(result: Result<(), P2StreamError>) -> String { + match result { + Ok(()) => "ok".to_string(), + Err(P2StreamError::Closed) => "err:closed".to_string(), + Err(P2StreamError::LastOperationFailed(error)) => { + match p2_types::filesystem_error_code(&error) { + Some(p2_types::ErrorCode::Quota) => "err:quota".to_string(), + Some(p2_types::ErrorCode::InsufficientSpace) => { + "err:insufficient-space".to_string() + } + Some(error) => format!("err:{error:?}"), + None => "err:unclassified".to_string(), + } + } + } +} + +pub(crate) async fn run_p2_quota_surface() -> Vec { + let (root, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let block = vec![0x5a; 4096]; + let file = root + .open_at( + p2_types::PathFlags::empty(), + "quota-p2.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("P2 quota file creation failed"); + let mut written_blocks = 0; + let mut growth_denied = false; + for index in 0..512 { + match file.write(&block, index * block.len() as u64) { + Ok(written) if written == block.len() as u64 => written_blocks += 1, + Ok(_) | Err(p2_types::ErrorCode::Quota) => { + growth_denied = true; + break; + } + Err(error) => panic!("unexpected P2 quota error: {error:?}"), + } + } + drop(file); + let _ = root.unlink_file_at("quota-p2.bin"); + vec![ + format!("p2_wrote_before_limit={}", written_blocks > 0), + format!("p2_growth_denied={growth_denied}"), + ] +} + +pub(crate) async fn run_p3_with_quota() -> bool { + let (root, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + let block = vec![0x5a; 4096]; + let file = root + .open_at( + p3_types::PathFlags::empty(), + "quota-p3.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("P3 quota file creation failed"); + let (mut stream, data) = wit_stream::new(); + let result = file.write_via_stream(data, 0); + let unwritten = stream.write_all(block).await; + drop(stream); + let written = unwritten.is_empty() && result.await.is_ok(); + drop(file); + let _ = root.unlink_file_at("quota-p3.bin".to_string()).await; + written +} + +pub(crate) async fn exhaust_p3_quota() -> Vec { + let (root, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + let file = root + .open_at( + p3_types::PathFlags::empty(), + "quota-p3-exhaustion.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("P3 quota file creation failed"); + let (mut stream, data) = wit_stream::new(); + let result = file.write_via_stream(data, 0); + let mut written_blocks = 0; + let mut unwritten = Vec::new(); + for _ in 0..512 { + unwritten = stream.write_all(vec![0x5a; 4096]).await; + if unwritten.is_empty() { + written_blocks += 1; + } else { + break; + } + } + drop(stream); + vec![ + format!("completion={}", p3_result(result.await)), + format!("prefix-persisted={}", written_blocks > 0), + format!("unwritten-bytes={}", unwritten.len()), + ] +} + +pub(crate) fn exhaust_p2_quota() -> Vec { + let (root, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let file = root + .open_at( + p2_types::PathFlags::empty(), + "quota-p2-exhaustion.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::WRITE, + ) + .expect("P2 quota file creation failed"); + let output = file + .write_via_stream(0) + .expect("P2 quota output stream creation failed"); + let mut written_blocks = 0; + let completion = loop { + let result = output.blocking_write_and_flush(&vec![0x4a; 4096]); + if result.is_ok() { + written_blocks += 1; + assert!(written_blocks < 512, "P2 project quota was not enforced"); + } else { + break p2_stream_result(result); + } + }; + vec![ + format!("completion={completion}"), + format!("prefix-persisted={}", written_blocks > 0), + ] +} + +pub(crate) fn inspect_p2_exhaustion() -> Vec { + let (root, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let file = root + .open_at( + p2_types::PathFlags::empty(), + "quota-p2-exhaustion.bin", + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::READ, + ) + .expect("open P2 quota exhaustion file"); + let size = file.stat().expect("stat P2 quota exhaustion file").size; + let (bytes, _) = file.read(size, 0).expect("read P2 quota exhaustion file"); + vec![ + format!("size={size}"), + format!( + "prefix-complete={}", + bytes.len() >= 4096 && bytes[..4096].iter().all(|byte| *byte == 0x4a) + ), + format!( + "suffix-bytes={}", + bytes[bytes.len().min(4096)..] + .iter() + .filter(|byte| **byte == 0x6b) + .count() + ), + ] +} + +pub(crate) async fn inspect_p3_exhaustion() -> Vec { + let (root, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + let file = root + .open_at( + p3_types::PathFlags::empty(), + "quota-p3-exhaustion.bin".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ, + ) + .await + .expect("open P3 quota exhaustion file"); + let size = file + .stat() + .await + .expect("stat P3 quota exhaustion file") + .size; + let (reader, result) = file.read_via_stream(0); + let bytes = reader.collect().await; + result.await.expect("read P3 quota exhaustion file"); + vec![ + format!("size={size}"), + format!( + "prefix-complete={}", + bytes.len() >= 4096 && bytes[..4096].iter().all(|byte| *byte == 0x5a) + ), + format!( + "suffix-bytes={}", + bytes[bytes.len().min(4096)..] + .iter() + .filter(|byte| **byte == 0x6c) + .count() + ), + ] +} + +pub(crate) async fn run_p2_quota_matrix() -> Vec { + let (root, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let block = vec![0x2a; 4096]; + let file = root + .open_at( + p2_types::PathFlags::empty(), + "p2-matrix.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 matrix file"); + let direct = file.write(&block, 0).is_ok(); + let positioned = file + .write_via_stream(4096) + .and_then(|stream| { + stream + .blocking_write_and_flush(&block) + .map_err(|_| p2_types::ErrorCode::Io) + }) + .is_ok(); + let appended = file + .append_via_stream() + .and_then(|stream| { + stream + .blocking_write_and_flush(&block) + .map_err(|_| p2_types::ErrorCode::Io) + }) + .is_ok(); + let sparse_resize = file.set_size(512 * 1024).is_ok(); + let overwrite = file.write(&block, 0).is_ok(); + let truncate = file.set_size(8192).is_ok(); + + let source = root + .open_at( + p2_types::PathFlags::empty(), + "p2-splice-source.bin", + p2_types::OpenFlags::CREATE | p2_types::OpenFlags::TRUNCATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 splice source"); + source.write(&block, 0).expect("write P2 splice source"); + let input = source.read_via_stream(0).expect("open P2 splice input"); + let output = file.append_via_stream().expect("open P2 splice output"); + let splice = output.blocking_splice(&input, block.len() as u64).is_ok() + && output.blocking_flush().is_ok(); + + root.link_at( + p2_types::PathFlags::empty(), + "p2-matrix.bin", + &root, + "p2-matrix-link.bin", + ) + .expect("create P2 hard link"); + let hard_link = file.stat().is_ok_and(|stat| stat.link_count == 2); + root.unlink_file_at("p2-matrix.bin") + .expect("unlink first P2 name"); + let first_unlink = root + .open_at( + p2_types::PathFlags::empty(), + "p2-matrix-link.bin", + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::READ, + ) + .and_then(|alias| alias.read(1, 0)) + .is_ok(); + root.unlink_file_at("p2-matrix-link.bin") + .expect("unlink final P2 name"); + let open_unlinked = file.read(1, 0).is_ok(); + + let replacement = root + .open_at( + p2_types::PathFlags::empty(), + "p2-replacement.bin", + p2_types::OpenFlags::CREATE, + p2_types::DescriptorFlags::READ | p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 replacement target"); + replacement.write(b"old", 0).expect("write P2 replacement"); + root.rename_at("p2-splice-source.bin", &root, "p2-replacement.bin") + .expect("replace P2 target by rename"); + let rename_replace = root + .open_at( + p2_types::PathFlags::empty(), + "p2-replacement.bin", + p2_types::OpenFlags::empty(), + p2_types::DescriptorFlags::READ, + ) + .and_then(|renamed| renamed.read(1, 0)) + .is_ok_and(|(bytes, _)| bytes == vec![0x2a]); + root.create_directory_at("p2-metadata-dir") + .expect("create P2 directory"); + root.symlink_at("p2-replacement.bin", "p2-metadata-link") + .expect("create P2 symlink"); + + let growth = root + .open_at( + p2_types::PathFlags::empty(), + "p2-growth.bin", + p2_types::OpenFlags::CREATE, + p2_types::DescriptorFlags::WRITE, + ) + .expect("create P2 growth file"); + let mut grew = false; + let mut growth_denied = false; + for index in 0..512 { + match growth.write(&block, index * block.len() as u64) { + Ok(written) if written == block.len() as u64 => { + grew = true; + growth.sync_data().expect("settle P2 matrix quota usage"); + } + Ok(written) => { + grew |= written > 0; + growth_denied = true; + break; + } + Err(p2_types::ErrorCode::Quota) => { + growth_denied = true; + break; + } + Err(error) => panic!("unexpected P2 matrix growth error: {error:?}"), + } + } + vec![ + format!("direct={direct}"), + format!("positioned-stream={positioned}"), + format!("append={appended}"), + format!("sparse-resize={sparse_resize}"), + format!("overwrite={overwrite}"), + format!("truncate={truncate}"), + format!("splice={splice}"), + format!("hard-link={hard_link}"), + format!("first-unlink={first_unlink}"), + format!("open-unlinked={open_unlinked}"), + format!("rename-replace={rename_replace}"), + format!("grew={grew}"), + format!("growth-denied={growth_denied}"), + ] +} + +pub(crate) async fn run_p3_quota_matrix() -> Vec { + let (root, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + let block = vec![0x3b; 4096]; + let file = root + .open_at( + p3_types::PathFlags::empty(), + "p3-matrix.bin".to_string(), + p3_types::OpenFlags::CREATE | p3_types::OpenFlags::TRUNCATE, + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 matrix file"); + let (mut positioned_tx, positioned_data) = wit_stream::new(); + let positioned_result = file.write_via_stream(positioned_data, 0); + let positioned = positioned_tx.write_all(block.clone()).await.is_empty(); + drop(positioned_tx); + let positioned = positioned && positioned_result.await.is_ok(); + let (mut append_tx, append_data) = wit_stream::new(); + let append_result = file.append_via_stream(append_data); + let appended = append_tx.write_all(block.clone()).await.is_empty(); + drop(append_tx); + let appended = appended && append_result.await.is_ok(); + let sparse_resize = file.set_size(512 * 1024).await.is_ok(); + let (mut overwrite_tx, overwrite_data) = wit_stream::new(); + let overwrite_result = file.write_via_stream(overwrite_data, 0); + let overwrite = overwrite_tx.write_all(block.clone()).await.is_empty(); + drop(overwrite_tx); + let overwrite = overwrite && overwrite_result.await.is_ok(); + let truncate = file.set_size(8192).await.is_ok(); + + root.link_at( + p3_types::PathFlags::empty(), + "p3-matrix.bin".to_string(), + &root, + "p3-matrix-link.bin".to_string(), + ) + .await + .expect("create P3 hard link"); + let hard_link = file.stat().await.is_ok_and(|stat| stat.link_count == 2); + root.unlink_file_at("p3-matrix.bin".to_string()) + .await + .expect("unlink first P3 name"); + let first_unlink = root + .open_at( + p3_types::PathFlags::empty(), + "p3-matrix-link.bin".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ, + ) + .await + .is_ok(); + root.unlink_file_at("p3-matrix-link.bin".to_string()) + .await + .expect("unlink final P3 name"); + let open_unlinked = file.stat().await.is_ok(); + + let source = root + .open_at( + p3_types::PathFlags::empty(), + "p3-source.bin".to_string(), + p3_types::OpenFlags::CREATE, + p3_types::DescriptorFlags::READ | p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 rename source"); + let (mut source_tx, source_data) = wit_stream::new(); + let source_result = source.write_via_stream(source_data, 0); + assert!(source_tx.write_all(block.clone()).await.is_empty()); + drop(source_tx); + source_result.await.expect("write P3 rename source"); + let replacement = root + .open_at( + p3_types::PathFlags::empty(), + "p3-replacement.bin".to_string(), + p3_types::OpenFlags::CREATE, + p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 replacement target"); + drop(replacement); + root.rename_at( + "p3-source.bin".to_string(), + &root, + "p3-replacement.bin".to_string(), + ) + .await + .expect("replace P3 target by rename"); + let rename_replace = root + .open_at( + p3_types::PathFlags::empty(), + "p3-replacement.bin".to_string(), + p3_types::OpenFlags::empty(), + p3_types::DescriptorFlags::READ, + ) + .await + .is_ok(); + root.create_directory_at("p3-metadata-dir".to_string()) + .await + .expect("create P3 directory"); + root.symlink_at( + "p3-replacement.bin".to_string(), + "p3-metadata-link".to_string(), + ) + .await + .expect("create P3 symlink"); + + let growth = root + .open_at( + p3_types::PathFlags::empty(), + "p3-growth.bin".to_string(), + p3_types::OpenFlags::CREATE, + p3_types::DescriptorFlags::WRITE, + ) + .await + .expect("create P3 growth file"); + let (mut growth_tx, growth_data) = wit_stream::new(); + let growth_result = growth.write_via_stream(growth_data, 0); + let mut grew = false; + let mut unwritten = Vec::new(); + for _ in 0..512 { + unwritten = growth_tx.write_all(block.clone()).await; + if unwritten.is_empty() { + grew = true; + } else { + break; + } + } + drop(growth_tx); + let growth_denied = matches!( + growth_result.await, + Err(p3_types::ErrorCode::Quota) + ) && grew + && !unwritten.is_empty(); + vec![ + format!("positioned-stream={positioned}"), + format!("append={appended}"), + format!("sparse-resize={sparse_resize}"), + format!("overwrite={overwrite}"), + format!("truncate={truncate}"), + format!("hard-link={hard_link}"), + format!("first-unlink={first_unlink}"), + format!("open-unlinked={open_unlinked}"), + format!("rename-replace={rename_replace}"), + format!("growth-denied={growth_denied}"), + ] +} + +pub(crate) async fn run_p2_object_quota() -> Vec { + let (root, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + let held = root + .open_at( + p2_types::PathFlags::empty(), + "p2-held", + p2_types::OpenFlags::CREATE, + p2_types::DescriptorFlags::READ, + ) + .expect("create P2 held object"); + root.link_at( + p2_types::PathFlags::empty(), + "p2-held", + &root, + "p2-held-link", + ) + .expect("hard link must not consume an object"); + let hard_link_same_inode = held.stat().is_ok_and(|stat| stat.link_count == 2); + let mut created = 0; + for index in 0..64 { + match root.open_at( + p2_types::PathFlags::empty(), + &format!("p2-object-{index}"), + p2_types::OpenFlags::CREATE, + p2_types::DescriptorFlags::READ, + ) { + Ok(file) => { + created += 1; + drop(file); + } + Err(p2_types::ErrorCode::Quota) => break, + Err(error) => panic!("unexpected P2 object creation error: {error:?}"), + } + } + let object_denied = created < 64; + let directory_denied = matches!( + root.create_directory_at("p2-object-directory"), + Err(p2_types::ErrorCode::Quota) + ); + let symlink_denied = matches!( + root.symlink_at("p2-object-0", "p2-object-symlink"), + Err(p2_types::ErrorCode::Quota) + ); + root.unlink_file_at("p2-held").expect("unlink P2 held name"); + root.unlink_file_at("p2-held-link") + .expect("unlink P2 held link"); + let denied_while_open = matches!( + root.open_at( + p2_types::PathFlags::empty(), + "p2-before-close", + p2_types::OpenFlags::CREATE, + p2_types::DescriptorFlags::READ, + ), + Err(p2_types::ErrorCode::Quota) + ); + drop(held); + vec![ + format!("hard-link-same-inode={hard_link_same_inode}"), + format!("object-denied={object_denied}"), + format!("directory-denied={directory_denied}"), + format!("symlink-denied={symlink_denied}"), + format!("denied-while-open={denied_while_open}"), + ] +} + +pub(crate) async fn complete_p2_object_quota_release() -> bool { + let (root, _) = p2_preopens::get_directories() + .into_iter() + .next() + .expect("no P2 preopened directory"); + root.open_at( + p2_types::PathFlags::empty(), + "p2-after-close", + p2_types::OpenFlags::CREATE, + p2_types::DescriptorFlags::READ, + ) + .is_ok() +} + +pub(crate) async fn run_p3_object_quota() -> Vec { + let (root, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + let held = root + .open_at( + p3_types::PathFlags::empty(), + "p3-held".to_string(), + p3_types::OpenFlags::CREATE, + p3_types::DescriptorFlags::READ, + ) + .await + .expect("create P3 held object"); + root.link_at( + p3_types::PathFlags::empty(), + "p3-held".to_string(), + &root, + "p3-held-link".to_string(), + ) + .await + .expect("hard link must not consume an object"); + let hard_link_same_inode = held.stat().await.is_ok_and(|stat| stat.link_count == 2); + let mut created = 0; + for index in 0..64 { + match root + .open_at( + p3_types::PathFlags::empty(), + format!("p3-object-{index}"), + p3_types::OpenFlags::CREATE, + p3_types::DescriptorFlags::READ, + ) + .await + { + Ok(file) => { + created += 1; + drop(file); + } + Err(p3_types::ErrorCode::Quota) => break, + Err(error) => panic!("unexpected P3 object creation error: {error:?}"), + } + } + let object_denied = created < 64; + let directory_denied = matches!( + root.create_directory_at("p3-object-directory".to_string()) + .await, + Err(p3_types::ErrorCode::Quota) + ); + let symlink_denied = matches!( + root.symlink_at("p3-object-0".to_string(), "p3-object-symlink".to_string()) + .await, + Err(p3_types::ErrorCode::Quota) + ); + root.unlink_file_at("p3-held".to_string()) + .await + .expect("unlink P3 held name"); + root.unlink_file_at("p3-held-link".to_string()) + .await + .expect("unlink P3 held link"); + let denied_while_open = matches!( + root.open_at( + p3_types::PathFlags::empty(), + "p3-before-close".to_string(), + p3_types::OpenFlags::CREATE, + p3_types::DescriptorFlags::READ, + ) + .await, + Err(p3_types::ErrorCode::Quota) + ); + drop(held); + vec![ + format!("hard-link-same-inode={hard_link_same_inode}"), + format!("object-denied={object_denied}"), + format!("directory-denied={directory_denied}"), + format!("symlink-denied={symlink_denied}"), + format!("denied-while-open={denied_while_open}"), + ] +} + +pub(crate) async fn complete_p3_object_quota_release() -> bool { + let (root, _) = p3_preopens::get_directories() + .into_iter() + .next() + .expect("no P3 preopened directory"); + root.open_at( + p3_types::PathFlags::empty(), + "p3-after-close".to_string(), + p3_types::OpenFlags::CREATE, + p3_types::DescriptorFlags::READ, + ) + .await + .is_ok() +} diff --git a/wit/deps/golem-1.x/golem-oplog.wit b/wit/deps/golem-1.x/golem-oplog.wit index 28f7d9d514..97ef23469d 100644 --- a/wit/deps/golem-1.x/golem-oplog.wit +++ b/wit/deps/golem-1.x/golem-oplog.wit @@ -406,11 +406,6 @@ interface oplog { delta: u64 } - record filesystem-storage-usage-update-parameters { - timestamp: datetime, - delta: s64 - } - type agent-resource-id = u64; record create-resource-parameters { @@ -566,8 +561,6 @@ interface oplog { exceeded-table-limit, exceeded-http-call-limit, exceeded-rpc-call-limit, - node-out-of-filesystem-storage, - agent-exceeded-filesystem-storage-limit, agent-terminated-by-quota(agent-terminated-by-quota-error), ephemeral-sleep-too-long(ephemeral-sleep-too-long), ephemeral-fuel-exhausted(ephemeral-fuel-exhausted), @@ -786,8 +779,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(raw-create-resource-parameters), /// Dropped a resource instance @@ -895,8 +886,6 @@ interface oplog { failed-update(failed-update-parameters), /// Increased total linear memory size grow-memory(grow-memory-parameters), - /// Updated filesystem usage by a signed delta - filesystem-storage-usage-update(filesystem-storage-usage-update-parameters), /// Created a resource instance create-resource(create-resource-parameters), /// Dropped a resource instance